mirror of
https://github.com/orangecoding/fredy.git
synced 2026-06-16 12:31:07 +00:00
Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a138dafc31 | ||
|
|
c6bb3c44d4 | ||
|
|
a3471a091a |
@@ -1,48 +1,48 @@
|
|||||||
import utils, {buildHash} from '../utils.js';
|
import utils, { buildHash } from '../utils.js';
|
||||||
|
|
||||||
let appliedBlackList = [];
|
let appliedBlackList = [];
|
||||||
|
|
||||||
function normalize(o) {
|
function normalize(o) {
|
||||||
const size = o.size || 'N/A m²';
|
const size = o.size || 'N/A m²';
|
||||||
const price = (o.price || '--- €').replace('Preis auf Anfrage', '--- €');
|
const price = (o.price || '--- €').replace('Preis auf Anfrage', '--- €');
|
||||||
const title = o.title || 'No title available';
|
const title = o.title || 'No title available';
|
||||||
const immoId = o.id.substring(o.id.indexOf('-') + 1, o.id.length);
|
const immoId = o.id.substring(o.id.indexOf('-') + 1, o.id.length);
|
||||||
const link = `https://immo.swp.de/immobilien/${immoId}`;
|
const link = `https://immo.swp.de/immobilien/${immoId}`;
|
||||||
const description = o.description;
|
const description = o.description;
|
||||||
const id = buildHash(immoId, price);
|
const id = buildHash(immoId, price);
|
||||||
return Object.assign(o, {id, price, size, title, link, description});
|
return Object.assign(o, { id, price, size, title, link, description });
|
||||||
}
|
}
|
||||||
|
|
||||||
function applyBlacklist(o) {
|
function applyBlacklist(o) {
|
||||||
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||||
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||||
return titleNotBlacklisted && descNotBlacklisted;
|
return titleNotBlacklisted && descNotBlacklisted;
|
||||||
}
|
}
|
||||||
|
|
||||||
const config = {
|
const config = {
|
||||||
url: null,
|
url: null,
|
||||||
crawlContainer: '.js-serp-item',
|
crawlContainer: '.js-serp-item',
|
||||||
sortByDateParam: 's=most_recently_updated_first',
|
sortByDateParam: 's=most_recently_updated_first',
|
||||||
waitForSelector: 'body',
|
waitForSelector: 'body',
|
||||||
crawlFields: {
|
crawlFields: {
|
||||||
id: '.js-bookmark-btn@data-id',
|
id: '.js-bookmark-btn@data-id',
|
||||||
price: 'div.align-items-start div:first-child | trim',
|
price: 'div.align-items-start div:first-child | trim',
|
||||||
size: 'div.align-items-start div:nth-child(3) | trim',
|
size: 'div.align-items-start div:nth-child(3) | trim',
|
||||||
title: '.card-title h2 | trim',
|
title: '.js-item-title-link@title | trim',
|
||||||
link: '.ci-search-result__link@href',
|
link: '.ci-search-result__link@href',
|
||||||
description: '.js-show-more-item-sm | removeNewline | trim',
|
description: '.js-show-more-item-sm | removeNewline | trim',
|
||||||
},
|
},
|
||||||
normalize: normalize,
|
normalize: normalize,
|
||||||
filter: applyBlacklist,
|
filter: applyBlacklist,
|
||||||
};
|
};
|
||||||
export const init = (sourceConfig, blacklist) => {
|
export const init = (sourceConfig, blacklist) => {
|
||||||
config.enabled = sourceConfig.enabled;
|
config.enabled = sourceConfig.enabled;
|
||||||
config.url = sourceConfig.url;
|
config.url = sourceConfig.url;
|
||||||
appliedBlackList = blacklist || [];
|
appliedBlackList = blacklist || [];
|
||||||
};
|
};
|
||||||
export const metaInformation = {
|
export const metaInformation = {
|
||||||
name: 'Immo Südwest Presse',
|
name: 'Immo Südwest Presse',
|
||||||
baseUrl: 'https://immo.swp.de/',
|
baseUrl: 'https://immo.swp.de/',
|
||||||
id: 'immoswp',
|
id: 'immoswp',
|
||||||
};
|
};
|
||||||
export {config};
|
export { config };
|
||||||
|
|||||||
26
package.json
26
package.json
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "fredy",
|
"name": "fredy",
|
||||||
"version": "11.0.3",
|
"version": "11.0.5",
|
||||||
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
|
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"start": "node prod.js",
|
"start": "node prod.js",
|
||||||
@@ -50,12 +50,12 @@
|
|||||||
"Firefox ESR"
|
"Firefox ESR"
|
||||||
],
|
],
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@douyinfe/semi-ui": "2.73.0",
|
"@douyinfe/semi-ui": "2.75.0",
|
||||||
"@rematch/core": "2.2.0",
|
"@rematch/core": "2.2.0",
|
||||||
"@rematch/loading": "2.1.2",
|
"@rematch/loading": "2.1.2",
|
||||||
"@sendgrid/mail": "8.1.4",
|
"@sendgrid/mail": "8.1.4",
|
||||||
"@vitejs/plugin-react": "4.3.4",
|
"@vitejs/plugin-react": "4.3.4",
|
||||||
"better-sqlite3": "^11.8.0",
|
"better-sqlite3": "^11.8.1",
|
||||||
"body-parser": "1.20.3",
|
"body-parser": "1.20.3",
|
||||||
"cheerio": "^1.0.0",
|
"cheerio": "^1.0.0",
|
||||||
"cookie-session": "2.1.0",
|
"cookie-session": "2.1.0",
|
||||||
@@ -66,11 +66,11 @@
|
|||||||
"lowdb": "6.0.1",
|
"lowdb": "6.0.1",
|
||||||
"markdown": "^0.5.0",
|
"markdown": "^0.5.0",
|
||||||
"mixpanel": "^0.18.0",
|
"mixpanel": "^0.18.0",
|
||||||
"nanoid": "5.0.9",
|
"nanoid": "5.1.2",
|
||||||
"node-fetch": "3.3.2",
|
"node-fetch": "3.3.2",
|
||||||
"node-mailjet": "6.0.6",
|
"node-mailjet": "6.0.6",
|
||||||
"package-up": "^5.0.0",
|
"package-up": "^5.0.0",
|
||||||
"puppeteer": "^24.1.0",
|
"puppeteer": "^24.2.1",
|
||||||
"puppeteer-extra": "^3.3.6",
|
"puppeteer-extra": "^3.3.6",
|
||||||
"puppeteer-extra-plugin-stealth": "^2.11.2",
|
"puppeteer-extra-plugin-stealth": "^2.11.2",
|
||||||
"query-string": "9.1.1",
|
"query-string": "9.1.1",
|
||||||
@@ -88,21 +88,21 @@
|
|||||||
"vite": "5.4.11"
|
"vite": "5.4.11"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@babel/core": "7.26.0",
|
"@babel/core": "7.26.9",
|
||||||
"@babel/eslint-parser": "7.26.5",
|
"@babel/eslint-parser": "7.26.8",
|
||||||
"@babel/preset-env": "7.26.0",
|
"@babel/preset-env": "7.26.9",
|
||||||
"@babel/preset-react": "7.26.3",
|
"@babel/preset-react": "7.26.3",
|
||||||
"chai": "5.1.2",
|
"chai": "5.2.0",
|
||||||
"eslint": "8.56.0",
|
"eslint": "8.56.0",
|
||||||
"eslint-config-prettier": "8.8.0",
|
"eslint-config-prettier": "8.8.0",
|
||||||
"eslint-plugin-react": "7.37.4",
|
"eslint-plugin-react": "7.37.4",
|
||||||
"esmock": "2.6.9",
|
"esmock": "2.7.0",
|
||||||
"history": "5.3.0",
|
"history": "5.3.0",
|
||||||
"husky": "9.1.7",
|
"husky": "9.1.7",
|
||||||
"less": "4.2.1",
|
"less": "4.2.2",
|
||||||
"lint-staged": "15.4.1",
|
"lint-staged": "15.4.3",
|
||||||
"mocha": "10.8.2",
|
"mocha": "10.8.2",
|
||||||
"prettier": "3.4.2",
|
"prettier": "3.5.2",
|
||||||
"redux-logger": "3.0.6"
|
"redux-logger": "3.0.6"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,40 +1,38 @@
|
|||||||
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
|
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
|
||||||
import { get } from '../mocks/mockNotification.js';
|
import {get} from '../mocks/mockNotification.js';
|
||||||
import { mockFredy, providerConfig } from '../utils.js';
|
import {mockFredy, providerConfig} from '../utils.js';
|
||||||
import { expect } from 'chai';
|
import {expect} from 'chai';
|
||||||
import * as provider from '../../lib/provider/immonet.js';
|
import * as provider from '../../lib/provider/immonet.js';
|
||||||
|
|
||||||
describe('#immonet testsuite()', () => {
|
describe('#immonet testsuite()', () => {
|
||||||
after(() => {
|
after(() => {
|
||||||
similarityCache.stopCacheCleanup();
|
similarityCache.stopCacheCleanup();
|
||||||
});
|
});
|
||||||
provider.init(providerConfig.immonet, [], []);
|
provider.init(providerConfig.immonet, [], []);
|
||||||
it('should test immonet provider', async () => {
|
it('should test immonet provider', async () => {
|
||||||
const Fredy = await mockFredy();
|
const Fredy = await mockFredy();
|
||||||
return await new Promise((resolve) => {
|
return await new Promise((resolve) => {
|
||||||
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immonet', similarityCache);
|
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immonet', similarityCache);
|
||||||
fredy.execute().then((listing) => {
|
fredy.execute().then((listing) => {
|
||||||
expect(listing).to.be.a('array');
|
expect(listing).to.be.a('array');
|
||||||
const notificationObj = get();
|
const notificationObj = get();
|
||||||
expect(notificationObj).to.be.a('object');
|
expect(notificationObj).to.be.a('object');
|
||||||
expect(notificationObj.serviceName).to.equal('immonet');
|
expect(notificationObj.serviceName).to.equal('immonet');
|
||||||
notificationObj.payload.forEach((notify) => {
|
notificationObj.payload.forEach((notify) => {
|
||||||
/** check the actual structure **/
|
/** check the actual structure **/
|
||||||
expect(notify.id).to.be.a('string');
|
expect(notify.id).to.be.a('string');
|
||||||
expect(notify.price).to.be.a('string');
|
expect(notify.price).to.be.a('string');
|
||||||
expect(notify.size).to.be.a('string');
|
expect(notify.size).to.be.a('string');
|
||||||
expect(notify.title).to.be.a('string');
|
expect(notify.title).to.be.a('string');
|
||||||
expect(notify.link).to.be.a('string');
|
expect(notify.link).to.be.a('string');
|
||||||
expect(notify.address).to.be.a('string');
|
expect(notify.address).to.be.a('string');
|
||||||
|
|
||||||
/** check the values if possible **/
|
expect(notify.size).that.does.include('m²');
|
||||||
expect(notify.price).that.does.include('€');
|
expect(notify.title).to.be.not.empty;
|
||||||
expect(notify.size).that.does.include('m²');
|
expect(notify.address).to.be.not.empty;
|
||||||
expect(notify.title).to.be.not.empty;
|
});
|
||||||
expect(notify.address).to.be.not.empty;
|
resolve();
|
||||||
});
|
});
|
||||||
resolve();
|
});
|
||||||
});
|
|
||||||
});
|
});
|
||||||
});
|
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user