diff --git a/lib/FredyPipelineExecutioner.js b/lib/FredyPipelineExecutioner.js index c7e79f1..5495cd2 100755 --- a/lib/FredyPipelineExecutioner.js +++ b/lib/FredyPipelineExecutioner.js @@ -261,7 +261,7 @@ class FredyPipelineExecutioner { * @returns {ParsedListing[]} Filtered listings that pass validation and provider filter. */ _filter(listings) { - const requiredKeys = this._providerConfig.fieldNames; + const requiredKeys = this._providerConfig.requiredFieldNames; const requireValues = ['id', 'link', 'title']; const filteredListings = listings diff --git a/lib/provider/einsAImmobilien.js b/lib/provider/einsAImmobilien.js index dbc2d86..771ae8e 100755 --- a/lib/provider/einsAImmobilien.js +++ b/lib/provider/einsAImmobilien.js @@ -64,7 +64,7 @@ function applyBlacklist(o) { /** @type {ProviderConfig} */ const config = { - fieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'], + requiredFieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'], url: null, crawlContainer: '.tabelle', sortByDateParam: 'sort_type=newest', diff --git a/lib/provider/immobilienDe.js b/lib/provider/immobilienDe.js index 2de2ae1..40fbb2d 100644 --- a/lib/provider/immobilienDe.js +++ b/lib/provider/immobilienDe.js @@ -105,7 +105,7 @@ function applyBlacklist(o) { /** @type {ProviderConfig} */ const config = { - fieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'], + requiredFieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'], url: null, crawlContainer: 'a.lr-card', sortByDateParam: 'sort_col=*created_ts&sort_dir=desc', diff --git a/lib/provider/immoscout.js b/lib/provider/immoscout.js index 9aa0312..8eaccf3 100644 --- a/lib/provider/immoscout.js +++ b/lib/provider/immoscout.js @@ -202,7 +202,7 @@ function applyBlacklist(o) { } /** @type {ProviderConfig} */ const config = { - fieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'], + requiredFieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'], url: null, crawlFields: { id: 'id', diff --git a/lib/provider/immoswp.js b/lib/provider/immoswp.js index 4798df6..e01b853 100755 --- a/lib/provider/immoswp.js +++ b/lib/provider/immoswp.js @@ -44,7 +44,7 @@ function applyBlacklist(o) { /** @type {ProviderConfig} */ const config = { - fieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'], + requiredFieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'], url: null, crawlContainer: '.js-serp-item', sortByDateParam: 's=most_recently_updated_first', diff --git a/lib/provider/immowelt.js b/lib/provider/immowelt.js index 1e99787..38a509d 100755 --- a/lib/provider/immowelt.js +++ b/lib/provider/immowelt.js @@ -82,7 +82,7 @@ function applyBlacklist(o) { /** @type {ProviderConfig} */ const config = { - fieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'], + requiredFieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'], url: null, crawlContainer: 'div[data-testid="serp-core-scrollablelistview-testid"]:not(div[data-testid="serp-enlargementlist-testid"] div[data-testid="serp-card-testid"]) div[data-testid="serp-core-classified-card-testid"]', diff --git a/lib/provider/kleinanzeigen.js b/lib/provider/kleinanzeigen.js index a49542f..a132d11 100755 --- a/lib/provider/kleinanzeigen.js +++ b/lib/provider/kleinanzeigen.js @@ -186,7 +186,7 @@ function applyBlacklist(o) { /** @type {ProviderConfig} */ const config = { - fieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'], + requiredFieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'], url: null, crawlContainer: '#srchrslt-adtable .ad-listitem ', //sort by date is standard oO diff --git a/lib/provider/mcMakler.js b/lib/provider/mcMakler.js index 9d8ba41..3e133dd 100755 --- a/lib/provider/mcMakler.js +++ b/lib/provider/mcMakler.js @@ -44,7 +44,7 @@ function applyBlacklist(o) { } /** @type {ProviderConfig} */ const config = { - fieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'], + requiredFieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'], url: null, crawlContainer: 'article[data-testid="propertyCard"]', sortByDateParam: 'sortBy=DATE&sortOn=DESC', diff --git a/lib/provider/neubauKompass.js b/lib/provider/neubauKompass.js index ab05bdf..96cd5cb 100755 --- a/lib/provider/neubauKompass.js +++ b/lib/provider/neubauKompass.js @@ -47,7 +47,7 @@ function applyBlacklist(o) { /** @type {ProviderConfig} */ const config = { - fieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'], + requiredFieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'], url: null, crawlContainer: '.col-12.mb-4', sortByDateParam: 'Sortierung=Id&Richtung=DESC', diff --git a/lib/provider/ohneMakler.js b/lib/provider/ohneMakler.js index 9eb4603..6f8eba7 100755 --- a/lib/provider/ohneMakler.js +++ b/lib/provider/ohneMakler.js @@ -41,7 +41,7 @@ function applyBlacklist(o) { } /** @type {ProviderConfig} */ const config = { - fieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'], + requiredFieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'], url: null, crawlContainer: 'div[data-livecomponent-id*="search/property_list"] .grid > div', sortByDateParam: null, diff --git a/lib/provider/regionalimmobilien24.js b/lib/provider/regionalimmobilien24.js index aef3171..739cb9d 100755 --- a/lib/provider/regionalimmobilien24.js +++ b/lib/provider/regionalimmobilien24.js @@ -45,7 +45,7 @@ function applyBlacklist(o) { } /** @type {ProviderConfig} */ const config = { - fieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'], + requiredFieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'], url: null, crawlContainer: '.listentry-content', sortByDateParam: null, // sort by date is standard diff --git a/lib/provider/sparkasse.js b/lib/provider/sparkasse.js index 75db6e7..71c77c4 100755 --- a/lib/provider/sparkasse.js +++ b/lib/provider/sparkasse.js @@ -90,7 +90,7 @@ function applyBlacklist(o) { } /** @type {ProviderConfig} */ const config = { - fieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'], + requiredFieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'], url: null, crawlContainer: 'div[data-testid="estate-link"]', sortByDateParam: 'sortBy=date_desc', diff --git a/lib/provider/wgGesucht.js b/lib/provider/wgGesucht.js index f355627..cf1257b 100755 --- a/lib/provider/wgGesucht.js +++ b/lib/provider/wgGesucht.js @@ -84,7 +84,7 @@ const config = { image: '.img-responsive@src', description: '.row .noprint .col-xs-11 |removeNewline |trim', }, - fieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'], + requiredFieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'], normalize: normalize, filter: applyBlacklist, fetchDetails, diff --git a/lib/provider/wohnungsboerse.js b/lib/provider/wohnungsboerse.js index 4581029..3c24302 100644 --- a/lib/provider/wohnungsboerse.js +++ b/lib/provider/wohnungsboerse.js @@ -43,7 +43,7 @@ function applyBlacklist(o) { /** @type {ProviderConfig} */ const config = { - fieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'], + requiredFieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'], url: null, sortByDateParam: null, waitForSelector: 'body', diff --git a/lib/types/providerConfig.js b/lib/types/providerConfig.js index 5f891f6..7b6f77f 100644 --- a/lib/types/providerConfig.js +++ b/lib/types/providerConfig.js @@ -11,7 +11,7 @@ * @property {string} [sortByDateParam] Query parameter used to enforce sorting by date. * @property {string} [waitForSelector] CSS selector to wait for before parsing content. * @property {Object.} crawlFields Mapping of field names to selectors/paths. - * @property {string[]} fieldNames List of field names that this provider supports. + * @property {string[]} requiredFieldNames List of field names that this provider supports. * @property {string} [crawlContainer] CSS selector for the container holding listing items. * @property {(raw: any) => ParsedListing} normalize Function to convert raw scraped data into a ParsedListing shape. * @property {(listing: ParsedListing) => boolean} filter Function to filter out unwanted listings. diff --git a/package.json b/package.json index 464f16c..237a729 100755 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "fredy", - "version": "20.2.1", + "version": "20.3.0", "description": "[F]ind [R]eal [E]states [d]amn eas[y].", "scripts": { "prepare": "husky", diff --git a/test/pipeline_filtering.test.js b/test/pipeline_filtering.test.js index 3b624a3..2a22519 100644 --- a/test/pipeline_filtering.test.js +++ b/test/pipeline_filtering.test.js @@ -22,7 +22,7 @@ describe('Issue reproduction: listings filtered by similarity or area should be normalize: (l) => l, filter: () => true, crawlFields: { id: 'id', title: 'title', address: 'address', price: 'price' }, - fieldNames: ['id', 'title', 'address', 'price'], + requiredFieldNames: ['id', 'title', 'address', 'price'], }; const mockedJob = { @@ -97,7 +97,7 @@ describe('Issue reproduction: listings filtered by similarity or area should be normalize: (l) => l, filter: () => true, crawlFields: { id: 'id', title: 'title', address: 'address', price: 'price' }, - fieldNames: ['id', 'title', 'address', 'price'], + requiredFieldNames: ['id', 'title', 'address', 'price'], }; const fredy = new Fredy(providerConfig, mockedJob, 'test-provider', mockSimilarityCache, undefined); diff --git a/vitest.gh.config.js b/vitest.gh.config.js index 5632320..2889fd7 100644 --- a/vitest.gh.config.js +++ b/vitest.gh.config.js @@ -15,6 +15,7 @@ export default mergeConfig( 'test/provider/immonet.test.js', 'test/provider/immobilienDe.test.js', 'test/provider/immowelt.test.js', + 'test/provider/sparkasse.test.js', ], }, }),