Compare commits

...

3 Commits

Author SHA1 Message Date
Christian Kellner
a138dafc31 fixing immoweltsp title 2025-03-31 18:38:18 +02:00
weakmap@gmail.com
c6bb3c44d4 upgrade dependencies, fixing tests 2025-02-23 17:14:39 +01:00
weakmap@gmail.com
a3471a091a upgrade dependencies, fixing tests 2025-02-23 17:13:08 +01:00
4 changed files with 449 additions and 402 deletions

View File

@@ -1,48 +1,48 @@
import utils, {buildHash} from '../utils.js'; import utils, { buildHash } from '../utils.js';
let appliedBlackList = []; let appliedBlackList = [];
function normalize(o) { function normalize(o) {
const size = o.size || 'N/A m²'; const size = o.size || 'N/A m²';
const price = (o.price || '--- €').replace('Preis auf Anfrage', '--- €'); const price = (o.price || '--- €').replace('Preis auf Anfrage', '--- €');
const title = o.title || 'No title available'; const title = o.title || 'No title available';
const immoId = o.id.substring(o.id.indexOf('-') + 1, o.id.length); const immoId = o.id.substring(o.id.indexOf('-') + 1, o.id.length);
const link = `https://immo.swp.de/immobilien/${immoId}`; const link = `https://immo.swp.de/immobilien/${immoId}`;
const description = o.description; const description = o.description;
const id = buildHash(immoId, price); const id = buildHash(immoId, price);
return Object.assign(o, {id, price, size, title, link, description}); return Object.assign(o, { id, price, size, title, link, description });
} }
function applyBlacklist(o) { function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList); const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList); const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted; return titleNotBlacklisted && descNotBlacklisted;
} }
const config = { const config = {
url: null, url: null,
crawlContainer: '.js-serp-item', crawlContainer: '.js-serp-item',
sortByDateParam: 's=most_recently_updated_first', sortByDateParam: 's=most_recently_updated_first',
waitForSelector: 'body', waitForSelector: 'body',
crawlFields: { crawlFields: {
id: '.js-bookmark-btn@data-id', id: '.js-bookmark-btn@data-id',
price: 'div.align-items-start div:first-child | trim', price: 'div.align-items-start div:first-child | trim',
size: 'div.align-items-start div:nth-child(3) | trim', size: 'div.align-items-start div:nth-child(3) | trim',
title: '.card-title h2 | trim', title: '.js-item-title-link@title | trim',
link: '.ci-search-result__link@href', link: '.ci-search-result__link@href',
description: '.js-show-more-item-sm | removeNewline | trim', description: '.js-show-more-item-sm | removeNewline | trim',
}, },
normalize: normalize, normalize: normalize,
filter: applyBlacklist, filter: applyBlacklist,
}; };
export const init = (sourceConfig, blacklist) => { export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled; config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url; config.url = sourceConfig.url;
appliedBlackList = blacklist || []; appliedBlackList = blacklist || [];
}; };
export const metaInformation = { export const metaInformation = {
name: 'Immo Südwest Presse', name: 'Immo Südwest Presse',
baseUrl: 'https://immo.swp.de/', baseUrl: 'https://immo.swp.de/',
id: 'immoswp', id: 'immoswp',
}; };
export {config}; export { config };

View File

@@ -1,6 +1,6 @@
{ {
"name": "fredy", "name": "fredy",
"version": "11.0.3", "version": "11.0.5",
"description": "[F]ind [R]eal [E]states [d]amn eas[y].", "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
"scripts": { "scripts": {
"start": "node prod.js", "start": "node prod.js",
@@ -50,12 +50,12 @@
"Firefox ESR" "Firefox ESR"
], ],
"dependencies": { "dependencies": {
"@douyinfe/semi-ui": "2.73.0", "@douyinfe/semi-ui": "2.75.0",
"@rematch/core": "2.2.0", "@rematch/core": "2.2.0",
"@rematch/loading": "2.1.2", "@rematch/loading": "2.1.2",
"@sendgrid/mail": "8.1.4", "@sendgrid/mail": "8.1.4",
"@vitejs/plugin-react": "4.3.4", "@vitejs/plugin-react": "4.3.4",
"better-sqlite3": "^11.8.0", "better-sqlite3": "^11.8.1",
"body-parser": "1.20.3", "body-parser": "1.20.3",
"cheerio": "^1.0.0", "cheerio": "^1.0.0",
"cookie-session": "2.1.0", "cookie-session": "2.1.0",
@@ -66,11 +66,11 @@
"lowdb": "6.0.1", "lowdb": "6.0.1",
"markdown": "^0.5.0", "markdown": "^0.5.0",
"mixpanel": "^0.18.0", "mixpanel": "^0.18.0",
"nanoid": "5.0.9", "nanoid": "5.1.2",
"node-fetch": "3.3.2", "node-fetch": "3.3.2",
"node-mailjet": "6.0.6", "node-mailjet": "6.0.6",
"package-up": "^5.0.0", "package-up": "^5.0.0",
"puppeteer": "^24.1.0", "puppeteer": "^24.2.1",
"puppeteer-extra": "^3.3.6", "puppeteer-extra": "^3.3.6",
"puppeteer-extra-plugin-stealth": "^2.11.2", "puppeteer-extra-plugin-stealth": "^2.11.2",
"query-string": "9.1.1", "query-string": "9.1.1",
@@ -88,21 +88,21 @@
"vite": "5.4.11" "vite": "5.4.11"
}, },
"devDependencies": { "devDependencies": {
"@babel/core": "7.26.0", "@babel/core": "7.26.9",
"@babel/eslint-parser": "7.26.5", "@babel/eslint-parser": "7.26.8",
"@babel/preset-env": "7.26.0", "@babel/preset-env": "7.26.9",
"@babel/preset-react": "7.26.3", "@babel/preset-react": "7.26.3",
"chai": "5.1.2", "chai": "5.2.0",
"eslint": "8.56.0", "eslint": "8.56.0",
"eslint-config-prettier": "8.8.0", "eslint-config-prettier": "8.8.0",
"eslint-plugin-react": "7.37.4", "eslint-plugin-react": "7.37.4",
"esmock": "2.6.9", "esmock": "2.7.0",
"history": "5.3.0", "history": "5.3.0",
"husky": "9.1.7", "husky": "9.1.7",
"less": "4.2.1", "less": "4.2.2",
"lint-staged": "15.4.1", "lint-staged": "15.4.3",
"mocha": "10.8.2", "mocha": "10.8.2",
"prettier": "3.4.2", "prettier": "3.5.2",
"redux-logger": "3.0.6" "redux-logger": "3.0.6"
} }
} }

View File

@@ -1,40 +1,38 @@
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js'; import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
import { get } from '../mocks/mockNotification.js'; import {get} from '../mocks/mockNotification.js';
import { mockFredy, providerConfig } from '../utils.js'; import {mockFredy, providerConfig} from '../utils.js';
import { expect } from 'chai'; import {expect} from 'chai';
import * as provider from '../../lib/provider/immonet.js'; import * as provider from '../../lib/provider/immonet.js';
describe('#immonet testsuite()', () => { describe('#immonet testsuite()', () => {
after(() => { after(() => {
similarityCache.stopCacheCleanup(); similarityCache.stopCacheCleanup();
}); });
provider.init(providerConfig.immonet, [], []); provider.init(providerConfig.immonet, [], []);
it('should test immonet provider', async () => { it('should test immonet provider', async () => {
const Fredy = await mockFredy(); const Fredy = await mockFredy();
return await new Promise((resolve) => { return await new Promise((resolve) => {
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immonet', similarityCache); const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immonet', similarityCache);
fredy.execute().then((listing) => { fredy.execute().then((listing) => {
expect(listing).to.be.a('array'); expect(listing).to.be.a('array');
const notificationObj = get(); const notificationObj = get();
expect(notificationObj).to.be.a('object'); expect(notificationObj).to.be.a('object');
expect(notificationObj.serviceName).to.equal('immonet'); expect(notificationObj.serviceName).to.equal('immonet');
notificationObj.payload.forEach((notify) => { notificationObj.payload.forEach((notify) => {
/** check the actual structure **/ /** check the actual structure **/
expect(notify.id).to.be.a('string'); expect(notify.id).to.be.a('string');
expect(notify.price).to.be.a('string'); expect(notify.price).to.be.a('string');
expect(notify.size).to.be.a('string'); expect(notify.size).to.be.a('string');
expect(notify.title).to.be.a('string'); expect(notify.title).to.be.a('string');
expect(notify.link).to.be.a('string'); expect(notify.link).to.be.a('string');
expect(notify.address).to.be.a('string'); expect(notify.address).to.be.a('string');
/** check the values if possible **/ expect(notify.size).that.does.include('m²');
expect(notify.price).that.does.include('€'); expect(notify.title).to.be.not.empty;
expect(notify.size).that.does.include('m²'); expect(notify.address).to.be.not.empty;
expect(notify.title).to.be.not.empty; });
expect(notify.address).to.be.not.empty; resolve();
}); });
resolve(); });
});
}); });
});
}); });

693
yarn.lock

File diff suppressed because it is too large Load Diff