mirror of
https://github.com/orangecoding/fredy.git
synced 2026-06-16 12:31:07 +00:00
Feature/spec filter (#276)
* feat(): create map component, add area filtering to the job config * feat(): filter listings by area filter * chore(): cleanup * feat(): solve feedback * feat(): solve most providers * feat(): solve maybe other providers * feat(): add specFilter config, also add rooms to listing * feat(): change tests * feat(): fix kleinanzeigen parser * feat(): add spec filter switch for listing overviiews * feat(): add rooms and size to the overview and detail of a listing * feat(): rem label * feat(): add types, update providers, they now return specs as numbers * feat(): add jsonconfig to enable type checks * feat: add type for prividerConfig, add fieldNames per provider * feat: fix tests, provider, add formatListing * chore: remov duplicates * feat(): fix tests * feat: fix immoscout * chore: geojson typing * feat: solve requested changes
This commit is contained in:
@@ -46,6 +46,10 @@ import {
|
||||
convertWebToMobile,
|
||||
} from '../services/immoscout/immoscout-web-translator.js';
|
||||
import logger from '../services/logger.js';
|
||||
import { extractNumber } from '../utils/extract-number.js';
|
||||
/** @import { ParsedListing } from '../types/listing.js' */
|
||||
/** @import { ProviderConfig } from '../types/providerConfig.js' */
|
||||
|
||||
let appliedBlackList = [];
|
||||
|
||||
async function getListings(url) {
|
||||
@@ -168,22 +172,44 @@ async function isListingActive(link) {
|
||||
function nullOrEmpty(val) {
|
||||
return val == null || val.length === 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {any} o
|
||||
* @returns {ParsedListing}
|
||||
*/
|
||||
function normalize(o) {
|
||||
const title = nullOrEmpty(o.title) ? 'NO TITLE FOUND' : o.title.replace('NEU', '');
|
||||
const title = (o.title || '').replace('NEU', '').trim();
|
||||
const address = nullOrEmpty(o.address) ? 'NO ADDRESS FOUND' : (o.address || '').replace(/\(.*\),.*$/, '').trim();
|
||||
const id = buildHash(o.id, o.price);
|
||||
return Object.assign(o, { id, title, address });
|
||||
return {
|
||||
id,
|
||||
link: o.link,
|
||||
title,
|
||||
price: extractNumber(o.price),
|
||||
size: extractNumber(o.size),
|
||||
rooms: extractNumber(o.rooms),
|
||||
address,
|
||||
image: o.image,
|
||||
description: o.description,
|
||||
};
|
||||
}
|
||||
/**
|
||||
* @param {ParsedListing} o
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function applyBlacklist(o) {
|
||||
return !isOneOf(o.title, appliedBlackList);
|
||||
}
|
||||
/** @type {ProviderConfig} */
|
||||
const config = {
|
||||
fieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'],
|
||||
url: null,
|
||||
crawlFields: {
|
||||
id: 'id',
|
||||
title: 'title',
|
||||
price: 'price',
|
||||
size: 'size',
|
||||
rooms: 'rooms',
|
||||
link: 'link',
|
||||
address: 'address',
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user