mirror of
https://github.com/orangecoding/fredy.git
synced 2026-06-16 12:31:07 +00:00
Compare commits
27 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
06c4ebb975 | ||
|
|
b075e09ac2 | ||
|
|
f215ab53db | ||
|
|
4ed92b246f | ||
|
|
4a9b60633a | ||
|
|
2123c1024b | ||
|
|
35767e6774 | ||
|
|
bf77ba2667 | ||
|
|
827c7e7321 | ||
|
|
7b63dc72cb | ||
|
|
fd42b57010 | ||
|
|
f5917af8f3 | ||
|
|
a85400d570 | ||
|
|
8ce6668c78 | ||
|
|
2d8121a708 | ||
|
|
172c039c79 | ||
|
|
4ab1fd9294 | ||
|
|
50b3fde075 | ||
|
|
1a3fc6f94d | ||
|
|
26ed42230a | ||
|
|
6f4defdc1b | ||
|
|
f798aed342 | ||
|
|
27e098c244 | ||
|
|
37948be0d3 | ||
|
|
cc7bbb77c4 | ||
|
|
96da0b7892 | ||
|
|
72993312c7 |
2
.github/workflows/test.yml
vendored
2
.github/workflows/test.yml
vendored
@@ -15,7 +15,7 @@ jobs:
|
||||
- name: Setup node
|
||||
uses: actions/setup-node@v2.5.1
|
||||
with:
|
||||
node-version: 16
|
||||
node-version: 18
|
||||
cache: 'yarn'
|
||||
- run: yarn install
|
||||
- run: yarn run test
|
||||
|
||||
22
Dockerfile
22
Dockerfile
@@ -1,18 +1,20 @@
|
||||
# syntax=docker/dockerfile:1.3
|
||||
FROM node:16-alpine AS builder
|
||||
COPY --chown=1000:1000 . /fredy
|
||||
FROM node:18
|
||||
|
||||
WORKDIR /fredy
|
||||
USER 1000
|
||||
|
||||
COPY . /fredy
|
||||
|
||||
RUN yarn install
|
||||
|
||||
RUN yarn global add pm2
|
||||
|
||||
RUN yarn run prod
|
||||
|
||||
FROM node:16-alpine
|
||||
COPY --from=builder --chown=1000:1000 /fredy /fredy
|
||||
RUN mkdir /db /conf && \
|
||||
chown 1000:1000 /db /conf && \
|
||||
chmod 777 -R /db/ && \
|
||||
ln -s /db /fredy/db && ln -s /conf /fredy/conf
|
||||
|
||||
EXPOSE 9998
|
||||
USER 1000
|
||||
VOLUME [ "/conf", "/db" ]
|
||||
WORKDIR /fredy
|
||||
CMD node index.js --no-daemon
|
||||
|
||||
CMD pm2-runtime index.js
|
||||
|
||||
2
LICENSE
2
LICENSE
@@ -1,6 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2021 Christian Kellner
|
||||
Copyright (c) 2024 Christian Kellner
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
12
README.md
12
README.md
@@ -17,7 +17,7 @@ _Fredy_ is supported by JetBrains under Open Source Support Program
|
||||
|
||||
## Usage
|
||||
|
||||
- Make sure to use Node.js 16 or above
|
||||
- Make sure to use Node.js 18 or above
|
||||
- Run the following commands:
|
||||
```ssh
|
||||
yarn (or npm install)
|
||||
@@ -33,9 +33,6 @@ _Fredy_ will start with the default port, set to `9998`. You can access _Fredy_
|
||||
|
||||
<img alt="Job Overview" width="30%" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot_3.png">
|
||||
</p>
|
||||
<p align="center">
|
||||
|
||||
</p>
|
||||
|
||||
## Understanding the fundamentals
|
||||
There are 3 important parts in Fredy, that you need to understand to leverage the full power of _Fredy_.
|
||||
@@ -89,7 +86,12 @@ The rest will be handled by _Fredy_. Keep in mind, the support is experimental.
|
||||
|
||||
If you need more than the 1000 API calls allowed per month, I'd suggest opting for a paid account... ScrapingAnt loves OpenSource, therefore they have decided to give all _Fredy_ users a 10% discount by using the code **FREDY10** (Disclaimer: I do not earn any money for recommending their service).
|
||||
|
||||
### Contribution guidelines
|
||||
### 👐 Contributing
|
||||
Thanks to all the people who already contributed!
|
||||
|
||||
<a href="https://github.com/orangecoding/fredy/graphs/contributors">
|
||||
<img src="https://contrib.rocks/image?repo=orangecoding/fredy" />
|
||||
</a>
|
||||
|
||||
See [Contributing](https://github.com/orangecoding/fredy/blob/master/CONTRIBUTING.md)
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '3.3'
|
||||
version: '3.8'
|
||||
services:
|
||||
fredy:
|
||||
container_name: fredy
|
||||
@@ -13,3 +13,4 @@ services:
|
||||
- ./db:/db
|
||||
ports:
|
||||
- 9998:9998
|
||||
restart: unless-stopped
|
||||
|
||||
36
lib/notification/adapter/apprise.js
Normal file
36
lib/notification/adapter/apprise.js
Normal file
@@ -0,0 +1,36 @@
|
||||
import { markdown2Html } from '../../services/markdown.js';
|
||||
import { getJob } from '../../services/storage/jobStorage.js';
|
||||
import fetch from 'node-fetch';
|
||||
|
||||
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||
const { server } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
|
||||
const job = getJob(jobKey);
|
||||
const jobName = job == null ? jobKey : job.name;
|
||||
const promises = newListings.map((newListing) => {
|
||||
const title = `${jobName} at ${serviceName}: ${newListing.title}`;
|
||||
const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\Link: ${newListing.link}`;
|
||||
return fetch(server, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
body: message,
|
||||
title: title,
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
return Promise.all(promises);
|
||||
};
|
||||
export const config = {
|
||||
id: 'apprise',
|
||||
name: 'Apprise',
|
||||
readme: markdown2Html('lib/notification/adapter/apprise.md'),
|
||||
description: 'Fredy will send new listings to your Apprise instance.',
|
||||
fields: {
|
||||
server: {
|
||||
type: 'text',
|
||||
label: 'Server',
|
||||
description: 'The server URL to send the notification to.',
|
||||
},
|
||||
},
|
||||
};
|
||||
3
lib/notification/adapter/apprise.md
Normal file
3
lib/notification/adapter/apprise.md
Normal file
@@ -0,0 +1,3 @@
|
||||
### Apprise Adapter
|
||||
|
||||
Refer to the [instructions](https://github.com/caronc/apprise-api#installation) on how to set up an Apprise instance and how to configure your preferred notification service.
|
||||
50
lib/notification/adapter/pushover.js
Normal file
50
lib/notification/adapter/pushover.js
Normal file
@@ -0,0 +1,50 @@
|
||||
import { markdown2Html } from '../../services/markdown.js';
|
||||
import { getJob } from '../../services/storage/jobStorage.js';
|
||||
import fetch from 'node-fetch';
|
||||
|
||||
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||
const { token, user, device } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
|
||||
const job = getJob(jobKey);
|
||||
const jobName = job == null ? jobKey : job.name;
|
||||
const promises = newListings.map((newListing) => {
|
||||
const title = `${jobName} at ${serviceName}: ${newListing.title}`;
|
||||
const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nLink: ${newListing.link}`;
|
||||
return fetch('https://api.pushover.net/1/messages.json', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
token: token,
|
||||
user: user,
|
||||
message: message,
|
||||
device: device,
|
||||
title: title,
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
return Promise.all(promises);
|
||||
};
|
||||
|
||||
export const config = {
|
||||
id: 'pushover',
|
||||
name: 'Pushover',
|
||||
readme: markdown2Html('lib/notification/adapter/pushover.md'),
|
||||
description: 'Fredy will send new listings to your mobile using Pushover.',
|
||||
fields: {
|
||||
token: {
|
||||
type: 'text',
|
||||
label: 'API token',
|
||||
description: 'Your application\'s API token.',
|
||||
},
|
||||
user: {
|
||||
type: 'text',
|
||||
label: 'User key',
|
||||
description: 'Your user/group key.',
|
||||
},
|
||||
device: {
|
||||
type: 'text',
|
||||
label: 'Device name',
|
||||
description: 'The device name to send your notification to. Messages may be addressed to multiple specific devices by joining them with a comma.',
|
||||
},
|
||||
},
|
||||
};
|
||||
5
lib/notification/adapter/pushover.md
Normal file
5
lib/notification/adapter/pushover.md
Normal file
@@ -0,0 +1,5 @@
|
||||
### Pushover Adapter
|
||||
|
||||
Refer to the [instructions](https://support.pushover.net/i7-what-is-pushover-and-how-do-i-use-it) to set up your Pushover application.
|
||||
|
||||
After setting up the application, please enter both your newly created User key and API token.
|
||||
@@ -1,25 +0,0 @@
|
||||
import { markdown2Html } from '../../services/markdown.js';
|
||||
import Database from 'better-sqlite3';
|
||||
export const send = ({ serviceName, newListings, jobKey }) => {
|
||||
const db = new Database('db/listings.db');
|
||||
const fields = ['serviceName', 'jobKey', 'id', 'size', 'rooms', 'price', 'address', 'title', 'link', 'description'];
|
||||
db.prepare(`CREATE TABLE IF NOT EXISTS listing (${fields.join(' TEXT, ')} TEXT);`).run();
|
||||
const insert = db.prepare(`INSERT INTO listing (${fields.join(', ')}) VALUES (@${fields.join(', @')})`);
|
||||
newListings.map((listing) => {
|
||||
let insertListing = {};
|
||||
fields.map((field) => {
|
||||
insertListing[field] = listing[field];
|
||||
});
|
||||
insertListing.serviceName = serviceName;
|
||||
insertListing.jobKey = jobKey;
|
||||
insert.run(insertListing);
|
||||
});
|
||||
return Promise.resolve();
|
||||
};
|
||||
export const config = {
|
||||
id: 'sqlite',
|
||||
name: 'Sqlite',
|
||||
description: 'This adapter stores listings in a local sqlite3 database.',
|
||||
config: {},
|
||||
readme: markdown2Html('lib/notification/adapter/sqlite.md'),
|
||||
};
|
||||
@@ -1,3 +0,0 @@
|
||||
### Sqlite Adapter
|
||||
|
||||
This adapter stores search results in an sqlite database in db/listings.db
|
||||
@@ -6,7 +6,7 @@ function nullOrEmpty(val) {
|
||||
function normalize(o) {
|
||||
const title = nullOrEmpty(o.title) ? 'NO TITLE FOUND' : o.title.replace('NEU', '');
|
||||
const address = nullOrEmpty(o.address) ? 'NO ADDRESS FOUND' : (o.address || '').replace(/\(.*\),.*$/, '').trim();
|
||||
const link = `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`;
|
||||
const link = nullOrEmpty(o.address) ? 'NO LINK' : `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`;
|
||||
return Object.assign(o, { title, address, link });
|
||||
}
|
||||
function applyBlacklist(o) {
|
||||
@@ -20,7 +20,7 @@ const config = {
|
||||
id: '.result-list-entry@data-obid | int',
|
||||
price: '.result-list-entry .result-list-entry__criteria .grid-item:first-child dd | removeNewline | trim',
|
||||
size: '.result-list-entry .result-list-entry__criteria .grid-item:nth-child(2) dd | removeNewline | trim',
|
||||
title: '.result-list-entry .result-list-entry__brand-title-container h5 | removeNewline | trim',
|
||||
title: '.result-list-entry .result-list-entry__brand-title-container h2 | removeNewline | trim',
|
||||
link: '.result-list-entry .result-list-entry__brand-title-container@href',
|
||||
address: '.result-list-entry .result-list-entry__map-link',
|
||||
},
|
||||
|
||||
@@ -1,44 +1,47 @@
|
||||
import utils from '../utils.js';
|
||||
|
||||
let appliedBlackList = [];
|
||||
|
||||
function normalize(o) {
|
||||
const id = o.id.substring(o.id.indexOf('-') + 1, o.id.length);
|
||||
const size = o.size || 'N/A m²';
|
||||
const price = (o.price || '--- €').replace('Preis auf Anfrage', '--- €');
|
||||
const address = o.address || 'No address available';
|
||||
const title = o.title || 'No title available';
|
||||
const link = `https://immo.swp.de/immobilien/${id}`;
|
||||
const description = o.description;
|
||||
return Object.assign(o, { id, address, price, size, title, link, description });
|
||||
const id = o.id.substring(o.id.indexOf('-') + 1, o.id.length);
|
||||
const size = o.size || 'N/A m²';
|
||||
const price = (o.price || '--- €').replace('Preis auf Anfrage', '--- €');
|
||||
const title = o.title || 'No title available';
|
||||
const link = `https://immo.swp.de/immobilien/${id}`;
|
||||
const description = o.description;
|
||||
return Object.assign(o, {id, price, size, title, link, description});
|
||||
}
|
||||
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||
return titleNotBlacklisted && descNotBlacklisted;
|
||||
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||
return titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
|
||||
const config = {
|
||||
url: null,
|
||||
crawlContainer: '.js-serp-item',
|
||||
sortByDateParam: 's=most_recently_updated_first',
|
||||
crawlFields: {
|
||||
id: '@id',
|
||||
price: 'div.item__spec.item-spec-price | trim',
|
||||
size: 'div.item__spec.item-spec-area | trim',
|
||||
title: 'a.js-item-title-link@title',
|
||||
address: 'div.item__locality | removeNewline | trim',
|
||||
description: 'div.item__main-info-points.clearfix p small | removeNewline | trim',
|
||||
},
|
||||
paginate: 'li.page-item.pagination__item a.page-link@href',
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist,
|
||||
url: null,
|
||||
crawlContainer: '.js-serp-item',
|
||||
sortByDateParam: 's=most_recently_updated_first',
|
||||
crawlFields: {
|
||||
id: '.js-bookmark-btn@data-id',
|
||||
price: 'div.align-items-start div:first-child | trim',
|
||||
size: 'div.align-items-start div:nth-child(3) | trim',
|
||||
title: '.card-title h2 | trim',
|
||||
link: '.ci-search-result__link@href',
|
||||
description: '.js-show-more-item-sm | removeNewline | trim',
|
||||
},
|
||||
paginate: 'li.page-item.pagination__item a.page-link@href',
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist,
|
||||
};
|
||||
export const init = (sourceConfig, blacklist) => {
|
||||
config.enabled = sourceConfig.enabled;
|
||||
config.url = sourceConfig.url;
|
||||
appliedBlackList = blacklist || [];
|
||||
config.enabled = sourceConfig.enabled;
|
||||
config.url = sourceConfig.url;
|
||||
appliedBlackList = blacklist || [];
|
||||
};
|
||||
export const metaInformation = {
|
||||
name: 'Immo Südwest Presse',
|
||||
baseUrl: 'https://immo.swp.de/',
|
||||
id: 'immoswp',
|
||||
name: 'Immo Südwest Presse',
|
||||
baseUrl: 'https://immo.swp.de/',
|
||||
id: 'immoswp',
|
||||
};
|
||||
export { config };
|
||||
export {config};
|
||||
|
||||
@@ -19,7 +19,7 @@ const config = {
|
||||
sortByDateParam: null,
|
||||
crawlFields: {
|
||||
id: '.aditem@data-adid | int',
|
||||
price: '.aditem-main--middle--price | removeNewline | trim',
|
||||
price: '.aditem-main--middle--price-shipping--price | removeNewline | trim',
|
||||
size: '.aditem-main .text-module-end span:nth-child(2) | removeNewline | trim',
|
||||
title: '.aditem-main .text-module-begin a | removeNewline | trim',
|
||||
link: '.aditem-main .text-module-begin a@href | removeNewline | trim',
|
||||
|
||||
52
package.json
52
package.json
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "fredy",
|
||||
"version": "7.4.0",
|
||||
"version": "8.0.6",
|
||||
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
|
||||
"scripts": {
|
||||
"start": "node index.js",
|
||||
@@ -55,54 +55,54 @@
|
||||
"Firefox ESR"
|
||||
],
|
||||
"dependencies": {
|
||||
"@douyinfe/semi-ui": "2.44.0",
|
||||
"@douyinfe/semi-ui": "2.60.0",
|
||||
"@rematch/core": "2.2.0",
|
||||
"@rematch/loading": "2.1.2",
|
||||
"@sendgrid/mail": "7.7.0",
|
||||
"@vitejs/plugin-react": "4.1.0",
|
||||
"@sendgrid/mail": "8.1.3",
|
||||
"@vitejs/plugin-react": "4.3.1",
|
||||
"better-sqlite3": "8.6.0",
|
||||
"body-parser": "1.20.2",
|
||||
"cookie-session": "2.0.0",
|
||||
"cookie-session": "2.1.0",
|
||||
"handlebars": "4.7.8",
|
||||
"highcharts": "11.1.0",
|
||||
"highcharts": "11.4.3",
|
||||
"highcharts-react-official": "3.2.1",
|
||||
"lodash": "4.17.21",
|
||||
"lowdb": "6.0.1",
|
||||
"markdown": "^0.5.0",
|
||||
"nanoid": "4.0.2",
|
||||
"nanoid": "5.0.7",
|
||||
"node-fetch": "3.3.2",
|
||||
"node-mailjet": "6.0.4",
|
||||
"query-string": "8.1.0",
|
||||
"react": "18.2.0",
|
||||
"react-dom": "18.2.0",
|
||||
"react-redux": "8.1.2",
|
||||
"node-mailjet": "6.0.5",
|
||||
"query-string": "8.2.0",
|
||||
"react": "18.3.1",
|
||||
"react-dom": "18.3.1",
|
||||
"react-redux": "9.1.2",
|
||||
"react-router": "5.2.1",
|
||||
"react-router-dom": "5.3.0",
|
||||
"redux": "4.2.1",
|
||||
"redux-thunk": "2.4.2",
|
||||
"restana": "4.9.7",
|
||||
"redux": "5.0.1",
|
||||
"redux-thunk": "3.1.0",
|
||||
"restana": "4.9.9",
|
||||
"serve-static": "1.15.0",
|
||||
"slack": "11.0.2",
|
||||
"string-similarity": "^4.0.4",
|
||||
"vite": "4.4.9",
|
||||
"vite": "5.2.13",
|
||||
"x-ray": "2.3.4"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@babel/core": "7.23.0",
|
||||
"@babel/eslint-parser": "7.22.15",
|
||||
"@babel/preset-env": "7.22.20",
|
||||
"@babel/preset-react": "7.22.15",
|
||||
"chai": "4.3.9",
|
||||
"eslint": "8.50.0",
|
||||
"@babel/core": "7.24.7",
|
||||
"@babel/eslint-parser": "7.24.7",
|
||||
"@babel/preset-env": "7.24.7",
|
||||
"@babel/preset-react": "7.24.7",
|
||||
"chai": "5.1.1",
|
||||
"eslint": "8.56.0",
|
||||
"eslint-config-prettier": "8.8.0",
|
||||
"eslint-plugin-react": "7.33.2",
|
||||
"esmock": "2.5.1",
|
||||
"eslint-plugin-react": "7.34.2",
|
||||
"esmock": "2.6.5",
|
||||
"history": "5.3.0",
|
||||
"husky": "4.3.8",
|
||||
"less": "4.2.0",
|
||||
"lint-staged": "13.2.2",
|
||||
"mocha": "10.2.0",
|
||||
"prettier": "3.0.3",
|
||||
"mocha": "10.4.0",
|
||||
"prettier": "3.3.2",
|
||||
"redux-logger": "3.0.6"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
|
||||
import { get } from '../mocks/mockNotification.js';
|
||||
import { providerConfig, mockFredy } from '../utils.js';
|
||||
import chai from 'chai';
|
||||
import { expect } from 'chai';
|
||||
import * as provider from '../../lib/provider/einsAImmobilien.js';
|
||||
|
||||
const expect = chai.expect;
|
||||
|
||||
describe('#einsAImmobilien testsuite()', () => {
|
||||
after(() => {
|
||||
similarityCache.stopCacheCleanup();
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
|
||||
import { get } from '../mocks/mockNotification.js';
|
||||
import { providerConfig, mockFredy } from '../utils.js';
|
||||
import chai from 'chai';
|
||||
import { expect } from 'chai';
|
||||
import * as provider from '../../lib/provider/immobilienDe.js';
|
||||
const expect = chai.expect;
|
||||
|
||||
describe('#immobilien.de testsuite()', () => {
|
||||
after(() => {
|
||||
similarityCache.stopCacheCleanup();
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
|
||||
import { get } from '../mocks/mockNotification.js';
|
||||
import { mockFredy, providerConfig } from '../utils.js';
|
||||
import chai from 'chai';
|
||||
import { expect } from 'chai';
|
||||
import * as provider from '../../lib/provider/immonet.js';
|
||||
import * as scrapingAnt from '../../lib/services/scrapingAnt.js';
|
||||
const expect = chai.expect;
|
||||
|
||||
describe('#immonet testsuite()', () => {
|
||||
after(() => {
|
||||
similarityCache.stopCacheCleanup();
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
|
||||
import { get } from '../mocks/mockNotification.js';
|
||||
import { mockFredy, providerConfig } from '../utils.js';
|
||||
import chai from 'chai';
|
||||
import { expect } from 'chai';
|
||||
import * as provider from '../../lib/provider/immoscout.js';
|
||||
import * as scrapingAnt from '../../lib/services/scrapingAnt.js';
|
||||
const expect = chai.expect;
|
||||
|
||||
describe('#immoscout testsuite()', () => {
|
||||
after(() => {
|
||||
similarityCache.stopCacheCleanup();
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
|
||||
import { get } from '../mocks/mockNotification.js';
|
||||
import { mockFredy, providerConfig } from '../utils.js';
|
||||
import chai from 'chai';
|
||||
import { expect } from 'chai';
|
||||
import * as provider from '../../lib/provider/immoswp.js';
|
||||
const expect = chai.expect;
|
||||
|
||||
describe('#immoswp testsuite()', () => {
|
||||
after(() => {
|
||||
similarityCache.stopCacheCleanup();
|
||||
@@ -25,12 +25,10 @@ describe('#immoswp testsuite()', () => {
|
||||
expect(notify.size).to.be.a('string');
|
||||
expect(notify.title).to.be.a('string');
|
||||
expect(notify.link).to.be.a('string');
|
||||
expect(notify.address).to.be.a('string');
|
||||
/** check the values if possible **/
|
||||
expect(notify.price).that.does.include('€');
|
||||
expect(notify.title).to.be.not.empty;
|
||||
expect(notify.link).that.does.include('https://immo.swp.de');
|
||||
expect(notify.address).to.be.not.empty;
|
||||
});
|
||||
resolve();
|
||||
});
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
|
||||
import { get } from '../mocks/mockNotification.js';
|
||||
import { mockFredy, providerConfig } from '../utils.js';
|
||||
import chai from 'chai';
|
||||
import { expect } from 'chai';
|
||||
import * as provider from '../../lib/provider/immowelt.js';
|
||||
const expect = chai.expect;
|
||||
|
||||
describe('#immowelt testsuite()', () => {
|
||||
after(() => {
|
||||
similarityCache.stopCacheCleanup();
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
|
||||
import { get } from '../mocks/mockNotification.js';
|
||||
import { mockFredy, providerConfig } from '../utils.js';
|
||||
import chai from 'chai';
|
||||
import { expect } from 'chai';
|
||||
import * as provider from '../../lib/provider/kleinanzeigen.js';
|
||||
const expect = chai.expect;
|
||||
|
||||
describe('#kleinanzeigen testsuite()', () => {
|
||||
after(() => {
|
||||
similarityCache.stopCacheCleanup();
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
|
||||
import { get } from '../mocks/mockNotification.js';
|
||||
import { mockFredy, providerConfig } from '../utils.js';
|
||||
import chai from 'chai';
|
||||
import { expect } from 'chai';
|
||||
import * as provider from '../../lib/provider/neubauKompass.js';
|
||||
const expect = chai.expect;
|
||||
|
||||
describe('#neubauKompass testsuite()', () => {
|
||||
after(() => {
|
||||
similarityCache.stopCacheCleanup();
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import utils from '../../lib/utils.js';
|
||||
import assert from 'assert';
|
||||
import chai from 'chai';
|
||||
const expect = chai.expect;
|
||||
import { expect } from 'chai';
|
||||
|
||||
const fakeWorkingHoursConfig = (from, to) => ({
|
||||
workingHours: {
|
||||
to,
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
|
||||
import { get } from '../mocks/mockNotification.js';
|
||||
import { mockFredy, providerConfig } from '../utils.js';
|
||||
import chai from 'chai';
|
||||
import { expect } from 'chai';
|
||||
import * as provider from '../../lib/provider/wgGesucht.js';
|
||||
const expect = chai.expect;
|
||||
|
||||
describe('#wgGesucht testsuite()', () => {
|
||||
after(() => {
|
||||
similarityCache.stopCacheCleanup();
|
||||
|
||||
@@ -1,16 +1,15 @@
|
||||
import fs from 'fs';
|
||||
import chai from 'chai';
|
||||
import { expect } from 'chai';
|
||||
import { readFile } from 'fs/promises';
|
||||
import mutator from '../../lib/services/queryStringMutator.js';
|
||||
import queryString from 'query-string';
|
||||
const expect = chai.expect;
|
||||
|
||||
const data = await readFile(new URL('./testData.json', import.meta.url));
|
||||
|
||||
const testData = JSON.parse(data);
|
||||
|
||||
let _provider = await Promise.all(
|
||||
fs.readdirSync('./lib/provider/').map(async (integPath) => await import(`../../lib/provider/${integPath}`))
|
||||
fs.readdirSync('./lib/provider/').map(async (integPath) => await import(`../../lib/provider/${integPath}`)),
|
||||
);
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import SimilarityCacheEntry from '../../lib/services/similarity-check/SimilarityCacheEntry.js';
|
||||
import chai from 'chai';
|
||||
const expect = chai.expect;
|
||||
import { expect } from 'chai';
|
||||
|
||||
describe('similarityCheck', () => {
|
||||
describe('#similarityCheck()', () => {
|
||||
it('should be false', () => {
|
||||
@@ -29,10 +29,10 @@ describe('similarityCheck', () => {
|
||||
it('should be false', () => {
|
||||
const check = new SimilarityCacheEntry(0);
|
||||
check.setCacheEntry(
|
||||
'The index is known by several other names, especially Sørensen–Dice index,[3] Sørensen index and Dice\'s coefficient. Other variations include the "similarity coefficient" or "index", such as Dice similarity coefficient (DSC). Common alternate spellings for Sørensen are Sorenson, Soerenson and Sörenson, and all three can also be seen with the –sen ending.'
|
||||
'The index is known by several other names, especially Sørensen–Dice index,[3] Sørensen index and Dice\'s coefficient. Other variations include the "similarity coefficient" or "index", such as Dice similarity coefficient (DSC). Common alternate spellings for Sørensen are Sorenson, Soerenson and Sörenson, and all three can also be seen with the –sen ending.',
|
||||
);
|
||||
check.setCacheEntry(
|
||||
'where |X| and |Y| are the cardinalities of the two sets (i.e. the number of elements in each set). The Sørensen index equals twice the number of elements common to both sets divided by the sum of the number of elements in each set.'
|
||||
'where |X| and |Y| are the cardinalities of the two sets (i.e. the number of elements in each set). The Sørensen index equals twice the number of elements common to both sets divided by the sum of the number of elements in each set.',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -184,8 +184,7 @@ const GeneralSettings = function GeneralSettings() {
|
||||
more likely to fail, but they are cheaper. A call with a datacenter proxy cost 10 credits.
|
||||
<h4>Residential-Proxy</h4>
|
||||
High-quality proxy server located in one of the real people houses across the world. Datacenter
|
||||
proxies are faster and more likely to success, but they are more expensive. A call with a datacenter
|
||||
proxy cost 250 credits.
|
||||
proxies are faster and more likely to success, but they are more expensive.
|
||||
<br />
|
||||
<br />
|
||||
<b>
|
||||
|
||||
@@ -45,7 +45,7 @@ export default function ProcessingTimes({ processingTimes }) {
|
||||
{format(new Date(processingTimes.scrapingAntData.end_date))}
|
||||
<br />
|
||||
Credits: {processingTimes.scrapingAntData.remained_credits}/
|
||||
{processingTimes.scrapingAntData.plan_total_credits} (250 credits per call)
|
||||
{processingTimes.scrapingAntData.plan_total_credits}
|
||||
</p>
|
||||
If you want to scrape Immoscout or Immonet more often, you have to purchase a premium account of{' '}
|
||||
<a href="https://scrapingant.com/" target="_blank" rel="noreferrer">
|
||||
|
||||
Reference in New Issue
Block a user