mirror of
https://github.com/orangecoding/fredy.git
synced 2026-06-16 12:31:07 +00:00
Compare commits
17 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fbad4456d7 | ||
|
|
deec626feb | ||
|
|
88c6641485 | ||
|
|
f4eedda658 | ||
|
|
d2b80561f8 | ||
|
|
3bda88a075 | ||
|
|
86465e0076 | ||
|
|
d947dad488 | ||
|
|
23ef434fe1 | ||
|
|
5e6d92c5be | ||
|
|
4ba098e0b6 | ||
|
|
2d1a9a0452 | ||
|
|
6fbee3e7c6 | ||
|
|
46775c3662 | ||
|
|
1feb5bfda1 | ||
|
|
3ec9ed3b2a | ||
|
|
75a536d5ab |
@@ -81,10 +81,10 @@ yarn run test
|
|||||||
# Architecture
|
# Architecture
|
||||||

|

|
||||||
|
|
||||||
### Immoscout
|
### Immoscout / Immonet
|
||||||
I have added **experimental** support for Immoscout. Immoscout is somewhat special, because they have decided to secure their service from bots using Re-Capture. Finding a way around this is barely possible. For _Fredy_ to be able to bypass this check, I'm using a service called [ScrapingAnt](https://scrapingant.com/). The trick is to use a headless browser, rotating proxies and (once successfully validated) to re-send the cookies each time.
|
I have added **experimental** support for Immoscout and Immonet. They both are somewhat special, because they have decided to secure their service from bots using Re-Capture. Finding a way around this is barely possible. For _Fredy_ to be able to bypass this check, I'm using a service called [ScrapingAnt](https://scrapingant.com/). The trick is to use a headless browser, rotating proxies and (once successfully validated) to re-send the cookies each time.
|
||||||
|
|
||||||
To be able to use Immoscout, you need to create an account at ScrapingAnt. Configure the API key in the "General Settings" tab (visible when logged in as administrator).
|
To be able to use Immoscout / Immonet, you need to create an account at ScrapingAnt. Configure the API key in the "General Settings" tab (visible when logged in as administrator).
|
||||||
The rest will be handled by _Fredy_. Keep in mind, the support is experimental. There might be bugs and you might not always pass the re-capture check, but most of the time it works rather well :)
|
The rest will be handled by _Fredy_. Keep in mind, the support is experimental. There might be bugs and you might not always pass the re-capture check, but most of the time it works rather well :)
|
||||||
|
|
||||||
If you need more than the 1000 API calls allowed per month, I'd suggest opting for a paid account... ScrapingAnt loves OpenSource, therefore they have decided to give all _Fredy_ users a 10% discount by using the code **FREDY10** (Disclaimer: I do not earn any money for recommending their service).
|
If you need more than the 1000 API calls allowed per month, I'd suggest opting for a paid account... ScrapingAnt loves OpenSource, therefore they have decided to give all _Fredy_ users a 10% discount by using the code **FREDY10** (Disclaimer: I do not earn any money for recommending their service).
|
||||||
|
|||||||
@@ -45,15 +45,15 @@ class FredyRuntime {
|
|||||||
_getListings(url) {
|
_getListings(url) {
|
||||||
return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
const id = this._providerId;
|
const id = this._providerId;
|
||||||
if (scrapingAnt.isImmoscout(id) && !scrapingAnt.isScrapingAntApiKeySet()) {
|
if (scrapingAnt.needScrapingAnt(id) && !scrapingAnt.isScrapingAntApiKeySet()) {
|
||||||
const error = 'Immoscout can only be used with if you have set an apikey for scrapingAnt.';
|
const error = 'Immoscout or Immonet can only be used with if you have set an apikey for scrapingAnt.';
|
||||||
/* eslint-disable no-console */
|
/* eslint-disable no-console */
|
||||||
console.log(error);
|
console.log(error);
|
||||||
/* eslint-enable no-console */
|
/* eslint-enable no-console */
|
||||||
reject(error);
|
reject(error);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const u = scrapingAnt.isImmoscout(id) ? scrapingAnt.transformUrlForScrapingAnt(url, id) : url;
|
const u = scrapingAnt.needScrapingAnt(id) ? scrapingAnt.transformUrlForScrapingAnt(url, id) : url;
|
||||||
try {
|
try {
|
||||||
if (this._providerConfig.paginate != null) {
|
if (this._providerConfig.paginate != null) {
|
||||||
xray(u, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields])
|
xray(u, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields])
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ import files from 'serve-static';
|
|||||||
import path from 'path';
|
import path from 'path';
|
||||||
import { getDirName } from '../utils.js';
|
import { getDirName } from '../utils.js';
|
||||||
const service = restana();
|
const service = restana();
|
||||||
const staticService = files(path.join(getDirName(), '../../ui/public'));
|
const staticService = files(path.join(getDirName(), '../ui/public'));
|
||||||
const PORT = config.port || 9998;
|
const PORT = config.port || 9998;
|
||||||
|
|
||||||
service.use(bodyParser.json());
|
service.use(bodyParser.json());
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ jobRouter.get('/processingTimes', async (req, res) => {
|
|||||||
let scrapingAntData = null;
|
let scrapingAntData = null;
|
||||||
if (config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0) {
|
if (config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0) {
|
||||||
try {
|
try {
|
||||||
const response = await fetch(`https://api.scrapingant.com/v1/usage?x-api-key=${config.scrapingAnt.apiKey}`);
|
const response = await fetch(`https://api.scrapingant.com/v2/usage?x-api-key=${config.scrapingAnt.apiKey}`);
|
||||||
scrapingAntData = await response.json();
|
scrapingAntData = await response.json();
|
||||||
} catch (Exception) {
|
} catch (Exception) {
|
||||||
console.error('Could not query plan data from scraping ant.', Exception);
|
console.error('Could not query plan data from scraping ant.', Exception);
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ const template = fs.readFileSync(path.resolve(__dirname + '/notification/emailTe
|
|||||||
const emailTemplate = Handlebars.compile(template);
|
const emailTemplate = Handlebars.compile(template);
|
||||||
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||||
const { apiPublicKey, apiPrivateKey, receiver, from } = notificationConfig.find(
|
const { apiPublicKey, apiPrivateKey, receiver, from } = notificationConfig.find(
|
||||||
(adapter) => adapter.id === 'mailJet'
|
(adapter) => adapter.id === 'mailjet'
|
||||||
).fields;
|
).fields;
|
||||||
const to = receiver
|
const to = receiver
|
||||||
.trim()
|
.trim()
|
||||||
@@ -18,7 +18,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
|
|||||||
Email: r.trim(),
|
Email: r.trim(),
|
||||||
}));
|
}));
|
||||||
return mailjet
|
return mailjet
|
||||||
.connect(apiPublicKey, apiPrivateKey)
|
.apiConnect(apiPublicKey, apiPrivateKey)
|
||||||
.post('send', { version: 'v3.1' })
|
.post('send', { version: 'v3.1' })
|
||||||
.request({
|
.request({
|
||||||
Messages: [
|
Messages: [
|
||||||
|
|||||||
51
lib/notification/adapter/ntfy.js
Normal file
51
lib/notification/adapter/ntfy.js
Normal file
@@ -0,0 +1,51 @@
|
|||||||
|
import { markdown2Html } from '../../services/markdown.js';
|
||||||
|
import { getJob } from '../../services/storage/jobStorage.js';
|
||||||
|
import fetch from 'node-fetch';
|
||||||
|
|
||||||
|
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||||
|
const { priority, server, topic } = notificationConfig.find((adapter) => adapter.id === 'ntfy').fields;
|
||||||
|
const job = getJob(jobKey);
|
||||||
|
const jobName = job == null ? jobKey : job.name;
|
||||||
|
const promises = newListings.map((newListing) => {
|
||||||
|
const message = `Address: ${newListing.address} Size: ${newListing.size.replace(/2m/g, '$m^2$')} Price: ${
|
||||||
|
newListing.price
|
||||||
|
}`;
|
||||||
|
return fetch(server, {
|
||||||
|
method: 'POST',
|
||||||
|
body: JSON.stringify({
|
||||||
|
topic: topic,
|
||||||
|
message: message,
|
||||||
|
title: newListing.title,
|
||||||
|
tags: [serviceName, jobName],
|
||||||
|
priority: parseInt(priority),
|
||||||
|
click: newListing.link,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
return Promise.all(promises);
|
||||||
|
};
|
||||||
|
export const config = {
|
||||||
|
id: 'ntfy',
|
||||||
|
name: 'ntfy',
|
||||||
|
readme: markdown2Html('lib/notification/adapter/ntfy.md'),
|
||||||
|
description: 'Fredy will send new listings to your ntfy.',
|
||||||
|
fields: {
|
||||||
|
priority: {
|
||||||
|
type: 'number',
|
||||||
|
label: 'Priority',
|
||||||
|
description: 'The priority of the send notification.',
|
||||||
|
},
|
||||||
|
server: {
|
||||||
|
type: 'text',
|
||||||
|
label: 'Server-URL',
|
||||||
|
description: 'The server url to the send the notification to.',
|
||||||
|
},
|
||||||
|
topic: {
|
||||||
|
type: 'text',
|
||||||
|
label: 'topic',
|
||||||
|
description:
|
||||||
|
'The topic where fredy should send notifications to. The topic is a secret, only known to you, make sure it is something not generic.',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
5
lib/notification/adapter/ntfy.md
Normal file
5
lib/notification/adapter/ntfy.md
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
### ntfy Adapter
|
||||||
|
|
||||||
|
For ntfy, you need to create a topic on your preferred ntfy instance. This is pretty easy. Please visit the steps in the [docs](https://docs.ntfy.sh/publish/) and follow the instructions.
|
||||||
|
|
||||||
|
As a result, you get the URL for configuration in fredy. In addition, the priority must be defined.
|
||||||
@@ -1,14 +1,12 @@
|
|||||||
import utils from '../utils.js';
|
import utils from '../utils.js';
|
||||||
let appliedBlackList = [];
|
let appliedBlackList = [];
|
||||||
function normalize(o) {
|
function normalize(o) {
|
||||||
const id = parseInt(o.id.substring(o.id.indexOf('_') + 1, o.id.length));
|
const id = o.id.substring(o.id.lastIndexOf('/') + 1, o.id.length);
|
||||||
const size = o.size != null ? o.size.replace('Wohnfläche ', '') : 'N/A m²';
|
const size = o.size != null ? o.size.replace('Wohnfläche ', '') : 'N/A m²';
|
||||||
const price = o.price.replace('Kaufpreis ', '');
|
const price = o.price.replace('Kaufpreis ', '');
|
||||||
const address = o.address.split(' • ')[o.address.split(' • ').length - 1];
|
const address = o.address.split(' • ')[o.address.split(' • ').length - 1];
|
||||||
const title = o.title || 'No title available';
|
const title = o.title || 'No title available';
|
||||||
//normally we would just read the link from the source, but immonet decided to trick user by adding a click listener instead of
|
const link = o.id;
|
||||||
//a href to do some weird reporting. (Very user friendly for handicaped ppl... not)
|
|
||||||
const link = `https://www.immonet.de/angebot/${id}`;
|
|
||||||
return Object.assign(o, { id, address, price, size, title, link });
|
return Object.assign(o, { id, address, price, size, title, link });
|
||||||
}
|
}
|
||||||
function applyBlacklist(o) {
|
function applyBlacklist(o) {
|
||||||
@@ -18,14 +16,14 @@ function applyBlacklist(o) {
|
|||||||
}
|
}
|
||||||
const config = {
|
const config = {
|
||||||
url: null,
|
url: null,
|
||||||
crawlContainer: '#result-list-stage .item',
|
crawlContainer: '.content-wrapper-tiles .ng-star-inserted',
|
||||||
sortByDateParam: 'sortby=19',
|
sortByDateParam: 'sortby=19',
|
||||||
crawlFields: {
|
crawlFields: {
|
||||||
id: '@id',
|
id: '.card a@href',
|
||||||
price: 'div[id*="selPrice_"] | trim',
|
title: '.card h3 |trim',
|
||||||
size: 'div[id*="selArea_"] | trim',
|
price: '.card .has-font-300 .is-bold | trim',
|
||||||
title: '.item a img@title',
|
size: '.card .has-font-300 .ml-100 | trim',
|
||||||
address: '.item .box-25 .ellipsis .text-100 | removeNewline | trim',
|
address: '.card span:nth-child(2) | trim',
|
||||||
},
|
},
|
||||||
paginate: '#idResultList .margin-bottom-6.margin-bottom-sm-12 .panel a.pull-right@href',
|
paginate: '#idResultList .margin-bottom-6.margin-bottom-sm-12 .panel a.pull-right@href',
|
||||||
normalize: normalize,
|
normalize: normalize,
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ const config = {
|
|||||||
};
|
};
|
||||||
export const metaInformation = {
|
export const metaInformation = {
|
||||||
name: 'Ebay Kleinanzeigen',
|
name: 'Ebay Kleinanzeigen',
|
||||||
baseUrl: 'https://www.ebay-kleinanzeigen.de/',
|
baseUrl: 'https://www.kleinanzeigen.de/',
|
||||||
id: 'kleinanzeigen',
|
id: 'kleinanzeigen',
|
||||||
};
|
};
|
||||||
export const init = (sourceConfig, blacklist, blacklistedDistricts) => {
|
export const init = (sourceConfig, blacklist, blacklistedDistricts) => {
|
||||||
|
|||||||
@@ -1,11 +1,16 @@
|
|||||||
import fetch from 'node-fetch';
|
import fetch from 'node-fetch';
|
||||||
import { config } from '../utils.js';
|
import { config } from '../utils.js';
|
||||||
import { makeUrlResidential } from './scrapingAnt.js';
|
import { makeUrlResidential } from './scrapingAnt.js';
|
||||||
|
import https from 'https';
|
||||||
//if ScrapingAnt got blocked, this http status is returned
|
//if ScrapingAnt got blocked, this http status is returned
|
||||||
const BLOCKED_HTTP_STATUS = 423;
|
const BLOCKED_HTTP_STATUS = 423;
|
||||||
const NOT_FOUND_HTTP_STATUS = 404;
|
const NOT_FOUND_HTTP_STATUS = 404;
|
||||||
const MAX_RETRIES_SCRAPING_ANT = 10;
|
const MAX_RETRIES_SCRAPING_ANT = 10;
|
||||||
const EXPECTED_STATUS_CODES = [BLOCKED_HTTP_STATUS, NOT_FOUND_HTTP_STATUS];
|
const EXPECTED_STATUS_CODES = [BLOCKED_HTTP_STATUS, NOT_FOUND_HTTP_STATUS];
|
||||||
|
const agent = new https.Agent({
|
||||||
|
rejectUnauthorized: false,
|
||||||
|
});
|
||||||
|
|
||||||
function makeDriver(headers = {}) {
|
function makeDriver(headers = {}) {
|
||||||
let cookies = '';
|
let cookies = '';
|
||||||
async function scrapingAntDriver(context, callback, retryCounter = 0) {
|
async function scrapingAntDriver(context, callback, retryCounter = 0) {
|
||||||
@@ -41,9 +46,10 @@ function makeDriver(headers = {}) {
|
|||||||
/* eslint-enable no-console */
|
/* eslint-enable no-console */
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The regular request driver is taking care of everyting, that doesn't need to be scraped by ScrapingAnt (which is
|
* The regular request driver is taking care of everyting, that doesn't need to be scraped by ScrapingAnt (which is
|
||||||
* everything != Immoscout as of writing this)
|
* everything != Immoscout & Immonet as of writing this)
|
||||||
*/
|
*/
|
||||||
return async function driver(context, callback) {
|
return async function driver(context, callback) {
|
||||||
if (context.url.toLowerCase().indexOf('scrapingant') !== -1) {
|
if (context.url.toLowerCase().indexOf('scrapingant') !== -1) {
|
||||||
@@ -55,6 +61,7 @@ function makeDriver(headers = {}) {
|
|||||||
...headers,
|
...headers,
|
||||||
Cookie: cookies,
|
Cookie: cookies,
|
||||||
},
|
},
|
||||||
|
agent,
|
||||||
});
|
});
|
||||||
const result = await response.text();
|
const result = await response.text();
|
||||||
callback(null, result);
|
callback(null, result);
|
||||||
|
|||||||
@@ -1,12 +1,22 @@
|
|||||||
import { metaInformation } from '../provider/immoscout.js';
|
import { metaInformation as immoScoutInfo } from '../provider/immoscout.js';
|
||||||
|
import { metaInformation as immoNetInfo } from '../provider/immonet.js';
|
||||||
import { config } from '../utils.js';
|
import { config } from '../utils.js';
|
||||||
const isImmoscout = (id) => {
|
|
||||||
return id.toLowerCase() === metaInformation.id;
|
const additionalImmonetUrlParams = `&wait_for_selector=.content-wrapper-tiles&js_snippet=${Buffer.from(
|
||||||
|
'window.scrollTo(0,document.body.scrollHeight);'
|
||||||
|
).toString('base64')}`;
|
||||||
|
|
||||||
|
const needScrapingAnt = (id) => {
|
||||||
|
return id.toLowerCase() === immoScoutInfo.id || id.toLowerCase() === immoNetInfo.id;
|
||||||
};
|
};
|
||||||
export const transformUrlForScrapingAnt = (url, id) => {
|
export const transformUrlForScrapingAnt = (url, id) => {
|
||||||
if (isImmoscout(id)) {
|
let urlParams = '';
|
||||||
//only do calls to scrapingAnt when dealing with Immoscout
|
if (needScrapingAnt(id)) {
|
||||||
url = `https://api.scrapingant.com/v1/general?url=${encodeURIComponent(url)}&proxy_type=datacenter`;
|
if (id.toLowerCase() === immoNetInfo.id) {
|
||||||
|
urlParams = additionalImmonetUrlParams;
|
||||||
|
}
|
||||||
|
//only do calls to scrapingAnt when dealing with Immoscout/Immonet
|
||||||
|
url = `https://api.scrapingant.com/v2/general?url=${encodeURIComponent(url)}&proxy_type=datacenter${urlParams}`;
|
||||||
}
|
}
|
||||||
return url;
|
return url;
|
||||||
};
|
};
|
||||||
@@ -16,4 +26,4 @@ export const isScrapingAntApiKeySet = () => {
|
|||||||
export const makeUrlResidential = (url) => {
|
export const makeUrlResidential = (url) => {
|
||||||
return url.replace('datacenter', 'residential');
|
return url.replace('datacenter', 'residential');
|
||||||
};
|
};
|
||||||
export { isImmoscout };
|
export { needScrapingAnt };
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
import lodash from 'lodash';
|
import lodash from 'lodash';
|
||||||
import { LowSync } from 'lowdb';
|
import { LowSync } from 'lowdb';
|
||||||
export default class LowdashAdapter extends LowSync {
|
export default class LowdashAdapter extends LowSync {
|
||||||
constructor(adapter) {
|
constructor(adapter, defaultData = {}) {
|
||||||
super(adapter);
|
super(adapter, defaultData);
|
||||||
this.chain = lodash.chain(this).get('data');
|
this.chain = lodash.chain(this).get('data');
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,11 +7,10 @@ import LowdashAdapter from './LowDashAdapter.js';
|
|||||||
|
|
||||||
const file = path.join(getDirName(), '../', 'db/jobs.json');
|
const file = path.join(getDirName(), '../', 'db/jobs.json');
|
||||||
const adapter = new JSONFileSync(file);
|
const adapter = new JSONFileSync(file);
|
||||||
const db = new LowdashAdapter(adapter);
|
const db = new LowdashAdapter(adapter, { jobs: [] });
|
||||||
|
|
||||||
db.read();
|
db.read();
|
||||||
|
|
||||||
db.data ||= { jobs: [] };
|
|
||||||
|
|
||||||
export const upsertJob = ({ jobId, name, blacklist = [], enabled = true, provider, notificationAdapter, userId }) => {
|
export const upsertJob = ({ jobId, name, blacklist = [], enabled = true, provider, notificationAdapter, userId }) => {
|
||||||
const currentJob =
|
const currentJob =
|
||||||
|
|||||||
@@ -5,12 +5,10 @@ import LowdashAdapter from './LowDashAdapter.js';
|
|||||||
|
|
||||||
const file = path.join(getDirName(), '../', 'db/jobListingData.json');
|
const file = path.join(getDirName(), '../', 'db/jobListingData.json');
|
||||||
const adapter = new JSONFileSync(file);
|
const adapter = new JSONFileSync(file);
|
||||||
const db = new LowdashAdapter(adapter);
|
const db = new LowdashAdapter(adapter, {});
|
||||||
|
|
||||||
db.read();
|
db.read();
|
||||||
|
|
||||||
db.data ||= {};
|
|
||||||
|
|
||||||
const buildKey = (jobKey, providerId, endpoint) => {
|
const buildKey = (jobKey, providerId, endpoint) => {
|
||||||
let key = `${jobKey}`;
|
let key = `${jobKey}`;
|
||||||
if (jobKey == null && endpoint == null) {
|
if (jobKey == null && endpoint == null) {
|
||||||
|
|||||||
@@ -6,24 +6,24 @@ import * as jobStorage from './jobStorage.js';
|
|||||||
import path from 'path';
|
import path from 'path';
|
||||||
import LowdashAdapter from './LowDashAdapter.js';
|
import LowdashAdapter from './LowDashAdapter.js';
|
||||||
|
|
||||||
|
const defaultData = {
|
||||||
|
user: [
|
||||||
|
//you probably want to change the default password ;)
|
||||||
|
{
|
||||||
|
id: nanoid(),
|
||||||
|
lastLogin: Date.now(),
|
||||||
|
username: 'admin',
|
||||||
|
password: hasher.hash('admin'),
|
||||||
|
isAdmin: true,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
const file = path.join(getDirName(), '../', 'db/users.json');
|
const file = path.join(getDirName(), '../', 'db/users.json');
|
||||||
const adapter = new JSONFileSync(file);
|
const adapter = new JSONFileSync(file);
|
||||||
const db = new LowdashAdapter(adapter);
|
const db = new LowdashAdapter(adapter, defaultData);
|
||||||
|
|
||||||
db.read();
|
db.read();
|
||||||
db.data ||= {
|
|
||||||
user: [
|
|
||||||
//you probably want to change the default password ;)
|
|
||||||
{
|
|
||||||
id: nanoid(),
|
|
||||||
lastLogin: Date.now(),
|
|
||||||
username: 'admin',
|
|
||||||
password: hasher.hash('admin'),
|
|
||||||
isAdmin: true,
|
|
||||||
isDemo: false,
|
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
|
||||||
|
|
||||||
export const getUsers = (withPassword) => {
|
export const getUsers = (withPassword) => {
|
||||||
const jobs = jobStorage.getJobs();
|
const jobs = jobStorage.getJobs();
|
||||||
|
|||||||
50
package.json
50
package.json
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "fredy",
|
"name": "fredy",
|
||||||
"version": "7.1.0",
|
"version": "7.3.2",
|
||||||
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
|
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"start": "node index.js",
|
"start": "node index.js",
|
||||||
@@ -55,27 +55,27 @@
|
|||||||
"Firefox ESR"
|
"Firefox ESR"
|
||||||
],
|
],
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@douyinfe/semi-ui": "2.31.0",
|
"@douyinfe/semi-ui": "2.42.4",
|
||||||
"@rematch/core": "2.2.0",
|
"@rematch/core": "2.2.0",
|
||||||
"@rematch/loading": "2.1.2",
|
"@rematch/loading": "2.1.2",
|
||||||
"@sendgrid/mail": "7.7.0",
|
"@sendgrid/mail": "7.7.0",
|
||||||
"@vitejs/plugin-react": "3.1.0",
|
"@vitejs/plugin-react": "4.0.4",
|
||||||
"better-sqlite3": "8.2.0",
|
"better-sqlite3": "8.6.0",
|
||||||
"body-parser": "1.20.2",
|
"body-parser": "1.20.2",
|
||||||
"cookie-session": "2.0.0",
|
"cookie-session": "2.0.0",
|
||||||
"handlebars": "4.7.7",
|
"handlebars": "4.7.8",
|
||||||
"highcharts": "10.3.3",
|
"highcharts": "11.1.0",
|
||||||
"highcharts-react-official": "3.2.0",
|
"highcharts-react-official": "3.2.1",
|
||||||
"lodash": "4.17.21",
|
"lodash": "4.17.21",
|
||||||
"lowdb": "5.1.0",
|
"lowdb": "6.0.1",
|
||||||
"markdown": "^0.5.0",
|
"markdown": "^0.5.0",
|
||||||
"nanoid": "4.0.1",
|
"nanoid": "4.0.2",
|
||||||
"node-fetch": "3.3.1",
|
"node-fetch": "3.3.2",
|
||||||
"node-mailjet": "6.0.2",
|
"node-mailjet": "6.0.4",
|
||||||
"query-string": "8.1.0",
|
"query-string": "8.1.0",
|
||||||
"react": "18.2.0",
|
"react": "18.2.0",
|
||||||
"react-dom": "18.2.0",
|
"react-dom": "18.2.0",
|
||||||
"react-redux": "8.0.5",
|
"react-redux": "8.1.2",
|
||||||
"react-router": "5.2.1",
|
"react-router": "5.2.1",
|
||||||
"react-router-dom": "5.3.0",
|
"react-router-dom": "5.3.0",
|
||||||
"redux": "4.2.1",
|
"redux": "4.2.1",
|
||||||
@@ -84,25 +84,25 @@
|
|||||||
"serve-static": "1.15.0",
|
"serve-static": "1.15.0",
|
||||||
"slack": "11.0.2",
|
"slack": "11.0.2",
|
||||||
"string-similarity": "^4.0.4",
|
"string-similarity": "^4.0.4",
|
||||||
"vite": "4.2.0",
|
"vite": "4.4.9",
|
||||||
"x-ray": "2.3.4"
|
"x-ray": "2.3.4"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@babel/core": "7.21.3",
|
"@babel/core": "7.22.15",
|
||||||
"@babel/eslint-parser": "7.21.3",
|
"@babel/eslint-parser": "7.22.15",
|
||||||
"@babel/preset-env": "7.20.2",
|
"@babel/preset-env": "7.22.15",
|
||||||
"@babel/preset-react": "7.18.6",
|
"@babel/preset-react": "7.22.15",
|
||||||
"chai": "4.3.7",
|
"chai": "4.3.8",
|
||||||
"eslint": "8.36.0",
|
"eslint": "8.48.0",
|
||||||
"eslint-config-prettier": "8.7.0",
|
"eslint-config-prettier": "8.8.0",
|
||||||
"eslint-plugin-react": "7.32.2",
|
"eslint-plugin-react": "7.33.2",
|
||||||
"esmock": "2.1.0",
|
"esmock": "2.4.0",
|
||||||
"history": "5.3.0",
|
"history": "5.3.0",
|
||||||
"husky": "4.3.8",
|
"husky": "4.3.8",
|
||||||
"less": "4.1.3",
|
"less": "4.2.0",
|
||||||
"lint-staged": "13.2.0",
|
"lint-staged": "13.2.2",
|
||||||
"mocha": "10.2.0",
|
"mocha": "10.2.0",
|
||||||
"prettier": "2.8.5",
|
"prettier": "2.8.8",
|
||||||
"redux-logger": "3.0.6"
|
"redux-logger": "3.0.6"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import { get } from '../mocks/mockNotification.js';
|
|||||||
import { mockFredy, providerConfig } from '../utils.js';
|
import { mockFredy, providerConfig } from '../utils.js';
|
||||||
import chai from 'chai';
|
import chai from 'chai';
|
||||||
import * as provider from '../../lib/provider/immonet.js';
|
import * as provider from '../../lib/provider/immonet.js';
|
||||||
|
import * as scrapingAnt from '../../lib/services/scrapingAnt.js';
|
||||||
const expect = chai.expect;
|
const expect = chai.expect;
|
||||||
describe('#immonet testsuite()', () => {
|
describe('#immonet testsuite()', () => {
|
||||||
after(() => {
|
after(() => {
|
||||||
@@ -12,6 +13,13 @@ describe('#immonet testsuite()', () => {
|
|||||||
it('should test immonet provider', async () => {
|
it('should test immonet provider', async () => {
|
||||||
const Fredy = await mockFredy();
|
const Fredy = await mockFredy();
|
||||||
return await new Promise((resolve) => {
|
return await new Promise((resolve) => {
|
||||||
|
if (!scrapingAnt.isScrapingAntApiKeySet()) {
|
||||||
|
/* eslint-disable no-console */
|
||||||
|
console.info('Skipping Immonet test as ScrapingAnt Api Key is not set.');
|
||||||
|
/* eslint-enable no-console */
|
||||||
|
resolve();
|
||||||
|
return;
|
||||||
|
}
|
||||||
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immonet', similarityCache);
|
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immonet', similarityCache);
|
||||||
fredy.execute().then((listing) => {
|
fredy.execute().then((listing) => {
|
||||||
expect(listing).to.be.a('array');
|
expect(listing).to.be.a('array');
|
||||||
@@ -20,17 +28,17 @@ describe('#immonet testsuite()', () => {
|
|||||||
expect(notificationObj.serviceName).to.equal('immonet');
|
expect(notificationObj.serviceName).to.equal('immonet');
|
||||||
notificationObj.payload.forEach((notify) => {
|
notificationObj.payload.forEach((notify) => {
|
||||||
/** check the actual structure **/
|
/** check the actual structure **/
|
||||||
expect(notify.id).to.be.a('number');
|
expect(notify.id).to.be.a('string');
|
||||||
expect(notify.price).to.be.a('string');
|
expect(notify.price).to.be.a('string');
|
||||||
expect(notify.size).to.be.a('string');
|
expect(notify.size).to.be.a('string');
|
||||||
expect(notify.title).to.be.a('string');
|
expect(notify.title).to.be.a('string');
|
||||||
expect(notify.link).to.be.a('string');
|
expect(notify.link).to.be.a('string');
|
||||||
expect(notify.address).to.be.a('string');
|
expect(notify.address).to.be.a('string');
|
||||||
|
|
||||||
/** check the values if possible **/
|
/** check the values if possible **/
|
||||||
expect(notify.price).that.does.include('€');
|
expect(notify.price).that.does.include('€');
|
||||||
expect(notify.size).that.does.include('m²');
|
expect(notify.size).that.does.include('m²');
|
||||||
expect(notify.title).to.be.not.empty;
|
expect(notify.title).to.be.not.empty;
|
||||||
expect(notify.link).that.does.include('https://www.immonet.de');
|
|
||||||
expect(notify.address).to.be.not.empty;
|
expect(notify.address).to.be.not.empty;
|
||||||
});
|
});
|
||||||
resolve();
|
resolve();
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ describe('#kleinanzeigen testsuite()', () => {
|
|||||||
expect(notify.address).to.be.a('string');
|
expect(notify.address).to.be.a('string');
|
||||||
/** check the values if possible **/
|
/** check the values if possible **/
|
||||||
expect(notify.title).to.be.not.empty;
|
expect(notify.title).to.be.not.empty;
|
||||||
expect(notify.link).that.does.include('https://www.ebay-kleinanzeigen.de');
|
expect(notify.link).that.does.include('https://www.kleinanzeigen.de');
|
||||||
expect(notify.address).to.be.not.empty;
|
expect(notify.address).to.be.not.empty;
|
||||||
});
|
});
|
||||||
resolve();
|
resolve();
|
||||||
|
|||||||
@@ -9,7 +9,7 @@
|
|||||||
"enabled": true
|
"enabled": true
|
||||||
},
|
},
|
||||||
"immonet": {
|
"immonet": {
|
||||||
"url": "https://www.immonet.de/immobiliensuche/sel.do?pageoffset=1&listsize=100&objecttype=1&locationname=Düsseldorf&acid=&actype=&district=8717&district=8718&district=8719&district=8720&district=8721&district=8723&district=8724&district=8725&district=8727&district=8728&district=8729&district=8730&district=8731&district=8732&district=8733&district=8737&district=8738&district=8741&district=8745&district=8747&district=8750&district=8752&district=8754&district=8755&district=8756&district=8759&district=8760&district=8761&district=8763&district=8764&district=8765&ajaxIsRadiusActive=false&sortby=19&suchart=1&radius=0&pcatmtypes=1_1&pCatMTypeStoragefield=&parentcat=1&marketingtype=1&fromprice=&toprice=420000&fromarea=90&toarea=&fromplotarea=&toplotarea=&fromrooms=3&torooms=&objectcat=225&objectcat=18&objectcat=17&objectcat=12&objectcat=16&objectcat=181&objectcat=14&objectcat=15&objectcat=226&objectcat=13&wbs=-1&fromyear=&toyear=",
|
"url": "https://www.immonet.de/immobiliensuche/beta?pageoffset=1&listsize=100&objecttype=1&locationname=D%C3%BCsseldorf&acid=&actype=&district=8717&district=8718&district=8719&district=8720&district=8721&district=8723&district=8724&district=8725&district=8727&district=8728&district=8729&district=8730&district=8731&district=8732&district=8733&district=8737&district=8738&district=8741&district=8745&district=8747&district=8750&district=8752&district=8754&district=8755&district=8756&district=8759&district=8760&district=8761&district=8763&district=8764&district=8765&ajaxIsRadiusActive=false&sortby=19&suchart=1&radius=0&pcatmtypes=1_1&pCatMTypeStoragefield=&parentcat=1&marketingtype=1&fromprice=&toprice=420000&fromarea=90&toarea=&fromplotarea=&toplotarea=&fromrooms=3&torooms=&objectcat=225&objectcat=18&objectcat=17&objectcat=12&objectcat=16&objectcat=181&objectcat=14&objectcat=15&objectcat=226&objectcat=13&wbs=-1&fromyear=&toyear=",
|
||||||
"enabled": true
|
"enabled": true
|
||||||
},
|
},
|
||||||
"immowelt": {
|
"immowelt": {
|
||||||
@@ -29,7 +29,7 @@
|
|||||||
"enabled": true
|
"enabled": true
|
||||||
},
|
},
|
||||||
"kleinanzeigen": {
|
"kleinanzeigen": {
|
||||||
"url": "https://www.ebay-kleinanzeigen.de/s-immobilien/duesseldorf/anzeige:angebote/wohnung/k0c195l2068r5",
|
"url": "https://www.kleinanzeigen.de/s-immobilien/duesseldorf/anzeige:angebote/wohnung/k0c195l2068r5",
|
||||||
"enabled": true
|
"enabled": true
|
||||||
},
|
},
|
||||||
"neubauKompass": {
|
"neubauKompass": {
|
||||||
|
|||||||
@@ -23,7 +23,6 @@ describe('#wgGesucht testsuite()', () => {
|
|||||||
expect(notify.id).to.be.a('string');
|
expect(notify.id).to.be.a('string');
|
||||||
expect(notify.title).to.be.a('string');
|
expect(notify.title).to.be.a('string');
|
||||||
expect(notify.details).to.be.a('string');
|
expect(notify.details).to.be.a('string');
|
||||||
expect(notify.size).to.be.a('string');
|
|
||||||
expect(notify.price).to.be.a('string');
|
expect(notify.price).to.be.a('string');
|
||||||
expect(notify.link).to.be.a('string');
|
expect(notify.link).to.be.a('string');
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -47,7 +47,7 @@ export default function ProcessingTimes({ processingTimes }) {
|
|||||||
Credits: {processingTimes.scrapingAntData.remained_credits}/
|
Credits: {processingTimes.scrapingAntData.remained_credits}/
|
||||||
{processingTimes.scrapingAntData.plan_total_credits} (250 credits per call)
|
{processingTimes.scrapingAntData.plan_total_credits} (250 credits per call)
|
||||||
</p>
|
</p>
|
||||||
If you want to scrape Immoscout more often, you have to purchase a premium account of{' '}
|
If you want to scrape Immoscout or Immonet more often, you have to purchase a premium account of{' '}
|
||||||
<a href="https://scrapingant.com/" target="_blank" rel="noreferrer">
|
<a href="https://scrapingant.com/" target="_blank" rel="noreferrer">
|
||||||
ScrapingAnt
|
ScrapingAnt
|
||||||
</a>
|
</a>
|
||||||
|
|||||||
@@ -25,9 +25,12 @@ const validate = (selectedAdapter) => {
|
|||||||
results.push('All fields are mandatory and must be set.');
|
results.push('All fields are mandatory and must be set.');
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (uiElement.type === 'number' && (typeof uiElement.value !== 'number' || uiElement.value < 0)) {
|
if (uiElement.type === 'number') {
|
||||||
results.push('A number field cannot contain anything else and must be > 0.');
|
const numberValue = parseFloat(uiElement.value);
|
||||||
continue;
|
if(isNaN(numberValue) || numberValue < 0) {
|
||||||
|
results.push('A number field cannot contain anything else and must be > 0.');
|
||||||
|
continue;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (uiElement.type === 'boolean' && typeof uiElement.value !== 'boolean') {
|
if (uiElement.type === 'boolean' && typeof uiElement.value !== 'boolean') {
|
||||||
results.push('A boolean field cannot be of a different type.');
|
results.push('A boolean field cannot be of a different type.');
|
||||||
|
|||||||
@@ -101,7 +101,7 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
|
|||||||
description={
|
description={
|
||||||
<div>
|
<div>
|
||||||
<p>
|
<p>
|
||||||
If you chose Immoscout as a provider, make sure to also add the scrapingAnt apiKey to the config.json.
|
If you chose Immoscout or Immonet as a provider, make sure to also add the scrapingAnt apiKey to the config.json.
|
||||||
(See readme)
|
(See readme)
|
||||||
</p>
|
</p>
|
||||||
<p>
|
<p>
|
||||||
|
|||||||
Reference in New Issue
Block a user