mirror of
https://github.com/orangecoding/fredy.git
synced 2026-06-16 12:31:07 +00:00
upgrading dependencies, fixing immowelt, using hash of price and id as unique identifier for listings
This commit is contained in:
@@ -1,18 +1,22 @@
|
||||
import utils from '../utils.js';
|
||||
import utils, {buildHash} from '../utils.js';
|
||||
let appliedBlackList = [];
|
||||
|
||||
function normalize(o) {
|
||||
let size = `${o.size.replace(' Wohnfläche ', '').trim()}`;
|
||||
if (o.rooms != null) {
|
||||
size += ` / / ${o.rooms.trim()}`;
|
||||
}
|
||||
const link = `https://www.1a-immobilienmarkt.de/expose/${o.id}.html`;
|
||||
return Object.assign(o, { size, link });
|
||||
const id = buildHash(o.id, o.price);
|
||||
return Object.assign(o, { id, size, link });
|
||||
}
|
||||
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||
return titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
|
||||
const config = {
|
||||
url: null,
|
||||
crawlContainer: '.tabelle',
|
||||
|
||||
Reference in New Issue
Block a user