External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Check if a listing is still active (#184)

Browse Source 
* check if a listing is still active

* upgrade dependencies
This commit is contained in:
Christian Kellner
2025-09-22 09:57:50 +02:00
committed by GitHub
parent 28eddc5d7f
commit c839f3abc9
22 changed files with 487 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
lib/provider/einsAImmobilien.js
View File

@@ -1,4 +1,5 @@
import utils, { buildHash } from '../utils.js';
import { buildHash, isOneOf } from '../utils.js';
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
let appliedBlackList = [];
function normalize(o) {
@@ -29,8 +30,8 @@ function normalizePrice(price) {
return result[0];
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted;
}
@@ -49,6 +50,7 @@ const config = {
},
normalize: normalize,
filter: applyBlacklist,
activeTester: checkIfListingIsActive,
};
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;

8
lib/provider/immobilienDe.js
View File

@@ -1,4 +1,5 @@
import utils, { buildHash } from '../utils.js';
import { buildHash, isOneOf } from '../utils.js';
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
let appliedBlackList = [];
@@ -24,8 +25,8 @@ function normalize(o) {
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted;
}
@@ -46,6 +47,7 @@ const config = {
},
normalize: normalize,
filter: applyBlacklist,
activeTester: checkIfListingIsActive,
};
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;

8
lib/provider/immonet.js
View File

@@ -1,4 +1,5 @@
import utils, { buildHash } from '../utils.js';
import { isOneOf, buildHash } from '../utils.js';
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
let appliedBlackList = [];
function normalize(o) {
@@ -11,8 +12,8 @@ function normalize(o) {
return Object.assign(o, { id, address, price, size, title, link });
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted;
}
const config = {
@@ -31,6 +32,7 @@ const config = {
},
normalize: normalize,
filter: applyBlacklist,
activeTester: checkIfListingIsActive,
};
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;

29
lib/provider/immoscout.js
View File

@@ -35,8 +35,11 @@
*
*/
import utils, { buildHash } from '../utils.js';
import { convertWebToMobile } from '../services/immoscout/immoscout-web-translator.js';
import { buildHash, isOneOf } from '../utils.js';
import {
convertImmoscoutListingToMobileListing,
convertWebToMobile,
} from '../services/immoscout/immoscout-web-translator.js';
import logger from '../services/logger.js';
let appliedBlackList = [];
@@ -77,6 +80,25 @@ async function getListings(url) {
});
}
async function isListingActive(link) {
const result = await fetch(convertImmoscoutListingToMobileListing(link), {
headers: {
'User-Agent': 'ImmoScout_27.3_26.0_._',
},
});
if (result.status === 200) {
return 1;
}
if (result.status === 404) {
return 0;
}
logger.warn('Unknown status for immoscout listing', link);
return -1;
}
function nullOrEmpty(val) {
return val == null || val.length === 0;
}
@@ -87,7 +109,7 @@ function normalize(o) {
return Object.assign(o, { id, title, address });
}
function applyBlacklist(o) {
return !utils.isOneOf(o.title, appliedBlackList);
return !isOneOf(o.title, appliedBlackList);
}
const config = {
url: null,
@@ -104,6 +126,7 @@ const config = {
normalize: normalize,
filter: applyBlacklist,
getListings: getListings,
activeTester: isListingActive,
};
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;

8
lib/provider/immoswp.js
View File

@@ -1,4 +1,5 @@
import utils, { buildHash } from '../utils.js';
import { isOneOf, buildHash } from '../utils.js';
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
let appliedBlackList = [];
@@ -14,8 +15,8 @@ function normalize(o) {
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted;
}
@@ -35,6 +36,7 @@ const config = {
},
normalize: normalize,
filter: applyBlacklist,
activeTester: checkIfListingIsActive,
};
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;

8
lib/provider/immowelt.js
View File

@@ -1,4 +1,5 @@
import utils, { buildHash } from '../utils.js';
import { buildHash, isOneOf } from '../utils.js';
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
let appliedBlackList = [];
@@ -8,8 +9,8 @@ function normalize(o) {
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted;
}
@@ -30,6 +31,7 @@ const config = {
},
normalize: normalize,
filter: applyBlacklist,
activeTester: checkIfListingIsActive,
};
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;

10
lib/provider/kleinanzeigen.js
View File

@@ -1,4 +1,5 @@
import utils, { buildHash } from '../utils.js';
import { buildHash, isOneOf } from '../utils.js';
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
let appliedBlackList = [];
let appliedBlacklistedDistricts = [];
@@ -11,10 +12,10 @@ function normalize(o) {
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
const isBlacklistedDistrict =
appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.description, appliedBlacklistedDistricts);
appliedBlacklistedDistricts.length === 0 ? false : isOneOf(o.description, appliedBlacklistedDistricts);
return o.title != null && !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
}
@@ -36,6 +37,7 @@ const config = {
},
normalize: normalize,
filter: applyBlacklist,
activeTester: checkIfListingIsActive,
};
export const metaInformation = {
name: 'Ebay Kleinanzeigen',

6
lib/provider/neubauKompass.js
View File

@@ -1,4 +1,5 @@
import utils, { buildHash } from '../utils.js';
import { isOneOf, buildHash } from '../utils.js';
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
let appliedBlackList = [];
@@ -15,7 +16,7 @@ function normalize(o) {
}
function applyBlacklist(o) {
return !utils.isOneOf(o.title, appliedBlackList);
return !isOneOf(o.title, appliedBlackList);
}
const config = {
@@ -33,6 +34,7 @@ const config = {
},
normalize: normalize,
filter: applyBlacklist,
activeTester: checkIfListingIsActive,
};
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;

8
lib/provider/wgGesucht.js
View File

@@ -1,4 +1,5 @@
import utils, { buildHash } from '../utils.js';
import { isOneOf, buildHash } from '../utils.js';
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
let appliedBlackList = [];
@@ -10,8 +11,8 @@ function normalize(o) {
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
return o.id != null && titleNotBlacklisted && descNotBlacklisted;
}
@@ -31,6 +32,7 @@ const config = {
},
normalize: normalize,
filter: applyBlacklist,
activeTester: checkIfListingIsActive,
};
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;