upgrading dependencies, fixing immowelt, using hash of price and id as unique identifier for listings

This commit is contained in:
Christian Kellner
2024-09-05 13:34:14 +02:00
parent 1bf012f13e
commit 35feb772d7
15 changed files with 680 additions and 446 deletions

View File

@@ -1,18 +1,22 @@
import utils from '../utils.js';
import utils, {buildHash} from '../utils.js';
let appliedBlackList = [];
function normalize(o) {
let size = `${o.size.replace(' Wohnfläche ', '').trim()}`;
if (o.rooms != null) {
size += ` / / ${o.rooms.trim()}`;
}
const link = `https://www.1a-immobilienmarkt.de/expose/${o.id}.html`;
return Object.assign(o, { size, link });
const id = buildHash(o.id, o.price);
return Object.assign(o, { id, size, link });
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted;
}
const config = {
url: null,
crawlContainer: '.tabelle',