External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

building multi platform docker images (#101)

Browse Source 
* building multi platform docker images

* upgrading dependencies | using scraping ant for neubaukompass
This commit is contained in:
Christian Kellner
2024-07-24 09:32:21 +02:00
committed by GitHub
parent 8ba3a53779
commit 92db8219b4
9 changed files with 1651 additions and 413 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
lib/provider/immoscout.js
View File

@@ -6,7 +6,7 @@ function nullOrEmpty(val) {
function normalize(o) {
const title = nullOrEmpty(o.title) ? 'NO TITLE FOUND' : o.title.replace('NEU', '');
const address = nullOrEmpty(o.address) ? 'NO ADDRESS FOUND' : (o.address || '').replace(/\(.*\),.*$/, '').trim();
const link = nullOrEmpty(o.address) ? 'NO LINK' : `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`;
const link = nullOrEmpty(o.link) ? 'NO LINK' : `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`;
return Object.assign(o, { title, address, link });
}
function applyBlacklist(o) {

6
lib/provider/neubauKompass.js
View File

@@ -1,7 +1,11 @@
import utils from '../utils.js';
let appliedBlackList = [];
function nullOrEmpty(val) {
return val == null || val.length === 0;
}
function normalize(o) {
return o;
const link = nullOrEmpty(o.link) ? 'NO LINK' : `https://www.neubaukompass.de${o.link.substring(o.link.indexOf('/neubau'))}`;
return {...o, link};
}
function applyBlacklist(o) {
return !utils.isOneOf(o.title, appliedBlackList);

3
lib/services/scrapingAnt.js
View File

@@ -1,5 +1,6 @@
import { metaInformation as immoScoutInfo } from '../provider/immoscout.js';
import { metaInformation as immoNetInfo } from '../provider/immonet.js';
import { metaInformation as neuBauCompassInfo } from '../provider/neubauKompass.js';
import { config } from '../utils.js';
const additionalImmonetUrlParams = `&wait_for_selector=.content-wrapper-tiles&js_snippet=${Buffer.from(
@@ -7,7 +8,7 @@ const additionalImmonetUrlParams = `&wait_for_selector=.content-wrapper-tiles&js
).toString('base64')}`;
const needScrapingAnt = (id) => {
return id.toLowerCase() === immoScoutInfo.id || id.toLowerCase() === immoNetInfo.id;
return id.toLowerCase() === immoScoutInfo.id || id.toLowerCase() === immoNetInfo.id || id.toLowerCase() === neuBauCompassInfo.id.toLowerCase();
};
export const transformUrlForScrapingAnt = (url, id) => {
let urlParams = '';