From 7fd8be07a2cbd5f1a029142ae7b1c858210b878c Mon Sep 17 00:00:00 2001 From: orangecoding Date: Fri, 9 Jan 2026 11:37:03 +0100 Subject: [PATCH] adding wohnungsboerse provider --- lib/provider/wohnungsboerse.js | 56 +++++++++++ package.json | 14 +-- test/provider/testProvider.json | 4 + test/provider/wohnungsboerse.test.js | 45 +++++++++ yarn.lock | 136 +++++++++++++-------------- 5 files changed, 180 insertions(+), 75 deletions(-) create mode 100644 lib/provider/wohnungsboerse.js create mode 100644 test/provider/wohnungsboerse.test.js diff --git a/lib/provider/wohnungsboerse.js b/lib/provider/wohnungsboerse.js new file mode 100644 index 0000000..615146e --- /dev/null +++ b/lib/provider/wohnungsboerse.js @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2026 by Christian Kellner. + * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause + */ + +import * as utils from '../utils.js'; + +let appliedBlackList = []; + +function normalize(o) { + const id = o.link.split('/').pop(); + const price = o.price; + const size = o.size; + const rooms = o.rooms; + const [city = '', part = ''] = (o.description || '').split('-').map((v) => v.trim()); + const address = `${part}, ${city}`; + return Object.assign(o, { id, price, size, rooms, address }); +} + +function applyBlacklist(o) { + const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList); + const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList); + return o.id != null && o.title != null && titleNotBlacklisted && descNotBlacklisted && o.link.startsWith(o.link); +} + +const config = { + url: null, + sortByDateParam: null, + waitForSelector: 'body', + crawlContainer: '.search_result_container > a', + crawlFields: { + id: '*', + title: 'h3 | trim', + price: 'dl:nth-of-type(1) dd | removeNewline | trim', + rooms: 'dl:nth-of-type(2) dd | removeNewline | trim', + size: 'dl:nth-of-type(3) dd | removeNewline | trim', + description: 'div.before\\:icon-location_marker | trim', + link: '@href', + imageUrl: 'img@src', + }, + normalize: normalize, + filter: applyBlacklist, +}; + +export const init = (sourceConfig, blacklistTerms) => { + config.url = sourceConfig.url; + appliedBlackList = blacklistTerms || []; +}; + +export const metaInformation = { + name: 'Wohnungsboerse', + baseUrl: 'https://www.wohnungsboerse.net', + id: 'wohnungsboerse', +}; + +export { config }; diff --git a/package.json b/package.json index 06c3485..37884fe 100755 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "fredy", - "version": "17.0.2", + "version": "17.1.0", "description": "[F]ind [R]eal [E]states [d]amn eas[y].", "scripts": { "prepare": "husky", @@ -60,12 +60,12 @@ ], "dependencies": { "adm-zip": "^0.5.16", - "@douyinfe/semi-icons": "^2.90.3", - "@douyinfe/semi-ui": "2.90.3", + "@douyinfe/semi-icons": "^2.90.10", + "@douyinfe/semi-ui": "2.90.10", "@sendgrid/mail": "8.1.6", "@vitejs/plugin-react": "5.1.2", "better-sqlite3": "^12.5.0", - "body-parser": "2.2.1", + "body-parser": "2.2.2", "chart.js": "^4.5.1", "cheerio": "^1.1.2", "cookie-session": "2.1.1", @@ -84,13 +84,13 @@ "react": "18.3.1", "react-chartjs-2": "^5.3.1", "react-dom": "18.3.1", - "react-router": "7.11.0", - "react-router-dom": "7.11.0", + "react-router": "7.12.0", + "react-router-dom": "7.12.0", "restana": "5.1.0", "semver": "^7.7.3", "serve-static": "2.2.1", "slack": "11.0.2", - "vite": "7.3.0", + "vite": "7.3.1", "x-var": "^3.0.1", "zustand": "^5.0.9" }, diff --git a/test/provider/testProvider.json b/test/provider/testProvider.json index 4560a92..39a7a5b 100644 --- a/test/provider/testProvider.json +++ b/test/provider/testProvider.json @@ -51,5 +51,9 @@ "wgGesucht": { "url": "https://www.wg-gesucht.de/wg-zimmer-in-Duesseldorf.30.0.1.0.html", "enabled": true + }, + "wohnungsboerse": { + "url": "https://www.wohnungsboerse.net/searches/index?estate_marketing_types=kauf%2C1&marketing_type=kauf&estate_types%5B0%5D=1&is_rendite=0&estate_id=&zipcodes%5B%5D=&cities%5B%5D=Duesseldorf&districts%5B%5D=&term=D%C3%BCsseldorf&umkreiskm=&pricetext=&minprice=&maxprice=&sizetext=&minsize=&maxsize=&roomstext=&minrooms=&maxrooms=", + "IsActive": true } } diff --git a/test/provider/wohnungsboerse.test.js b/test/provider/wohnungsboerse.test.js new file mode 100644 index 0000000..ca66649 --- /dev/null +++ b/test/provider/wohnungsboerse.test.js @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2026 by Christian Kellner. + * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause + */ + +/* + * Copyright (c) 2025 by Christian Kellner. + * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause + */ + +import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js'; +import { get } from '../mocks/mockNotification.js'; +import { providerConfig, mockFredy } from '../utils.js'; +import { expect } from 'chai'; +import * as provider from '../../lib/provider/wohnungsboerse.js'; + +describe('#wohnungsboerse testsuite()', () => { + provider.init(providerConfig.wohnungsboerse, [], []); + it('should test wohnungsboerse provider', async () => { + const Fredy = await mockFredy(); + return await new Promise((resolve) => { + const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'wohnungsboerse', similarityCache); + fredy.execute().then((listings) => { + expect(listings).to.be.a('array'); + const notificationObj = get(); + expect(notificationObj).to.be.a('object'); + expect(notificationObj.serviceName).to.equal('wohnungsboerse'); + notificationObj.payload.forEach((notify) => { + /** check the actual structure **/ + expect(notify.id).to.be.a('string'); + expect(notify.price).to.be.a('string'); + expect(notify.size).to.be.a('string'); + expect(notify.title).to.be.a('string'); + expect(notify.link).to.be.a('string'); + expect(notify.address).to.be.a('string'); + /** check the values if possible **/ + expect(notify.size).to.be.not.empty; + expect(notify.title).to.be.not.empty; + expect(notify.link).that.does.include('https://www.wohnungsboerse.net'); + }); + resolve(); + }); + }); + }); +}); diff --git a/yarn.lock b/yarn.lock index 9398f6b..49f2812 100644 --- a/yarn.lock +++ b/yarn.lock @@ -997,34 +997,34 @@ dependencies: tslib "^2.0.0" -"@douyinfe/semi-animation-react@2.90.3": - version "2.90.3" - resolved "https://registry.yarnpkg.com/@douyinfe/semi-animation-react/-/semi-animation-react-2.90.3.tgz#dc55e082febfdae38c42caefb9928b1bb853647a" - integrity sha512-iivpAVysJqKUmpRp4gqXb/+x7A4DOWXNv8lNM841CC2EVpCnFh4yOgXvxYCELD1V70RcXcXMTOGwAukCO1HkCA== +"@douyinfe/semi-animation-react@2.90.10": + version "2.90.10" + resolved "https://registry.yarnpkg.com/@douyinfe/semi-animation-react/-/semi-animation-react-2.90.10.tgz#46c489052cdb4219a21f7d4c4dca44dd40fb6f5d" + integrity sha512-xajEqbF1nZnFsocYJGhhOFPn9w1lOy7J6GBITKsMKpuBKq03TSKvZr9Sj7cMALEbPb8Ebp2y44t3e/9fQYpwlw== dependencies: - "@douyinfe/semi-animation" "2.90.3" - "@douyinfe/semi-animation-styled" "2.90.3" + "@douyinfe/semi-animation" "2.90.10" + "@douyinfe/semi-animation-styled" "2.90.10" classnames "^2.2.6" -"@douyinfe/semi-animation-styled@2.90.3": - version "2.90.3" - resolved "https://registry.yarnpkg.com/@douyinfe/semi-animation-styled/-/semi-animation-styled-2.90.3.tgz#6a5758af4540b482fd9a5db4960cac0511fb1f27" - integrity sha512-PQfNfOWOKiKHcViJOQipPRYYLFQNhzdFn9O84Rky9KzhrkH5/6NsNPMKIPfIUNN15qNI1gQMPP2mQxRAXJQTNg== +"@douyinfe/semi-animation-styled@2.90.10": + version "2.90.10" + resolved "https://registry.yarnpkg.com/@douyinfe/semi-animation-styled/-/semi-animation-styled-2.90.10.tgz#4d1cbebf90c60b1e79394dd87a79372114760cad" + integrity sha512-o1Oq/yMdX7DfLr1KiK/aDEEK/ZNa/WNmOh79wAyQe+jGVBBOFuyygfwZhYM1Gjp+vy0m/D+wb2S9L40+8JeWSw== -"@douyinfe/semi-animation@2.90.3": - version "2.90.3" - resolved "https://registry.yarnpkg.com/@douyinfe/semi-animation/-/semi-animation-2.90.3.tgz#197a12e1b100e8d15dbf7c7e858ce828938837cc" - integrity sha512-NcLXV2KDpgxHbqS+/d1KAkYJVmSlV0n1gRwB3diM6EgOaGLZHGNXMCVBuN7mss9f2sF6GTTJRwcjH3VdcV5E9g== +"@douyinfe/semi-animation@2.90.10": + version "2.90.10" + resolved "https://registry.yarnpkg.com/@douyinfe/semi-animation/-/semi-animation-2.90.10.tgz#b8f1f61ea8b3f548fc7c74cd6c93f196b3b9b0fb" + integrity sha512-eHG9LLtPtQfZP3shCkDXvixx8aHQ9wAeN8rrOFEMjJbApuAYtpArcm9nwmjTfYCWce1GZS64jrSt4apuSIKCHA== dependencies: bezier-easing "^2.1.0" -"@douyinfe/semi-foundation@2.90.3": - version "2.90.3" - resolved "https://registry.yarnpkg.com/@douyinfe/semi-foundation/-/semi-foundation-2.90.3.tgz#f75b51bd6f7bf3d8da3d9a3ed99aa0e44a2d3626" - integrity sha512-p9dtZbvkLtMKQturVwxtlJqfIOzHHN8ujaVgR/T9bS1ojzW+3Nua22yTkl7baLKUk/fvS/841BP3aHNXUmk71A== +"@douyinfe/semi-foundation@2.90.10": + version "2.90.10" + resolved "https://registry.yarnpkg.com/@douyinfe/semi-foundation/-/semi-foundation-2.90.10.tgz#4aeecd3afd675ba9fc64bb14592de130e1896e5e" + integrity sha512-4CARsSG9tghXkdBNyHT1G207vyIr+Y2RZS9zm37LpgrUezesw+znB8Z6mQIfxKrTDb7KezutubGJB/VUFNgj1w== dependencies: - "@douyinfe/semi-animation" "2.90.3" - "@douyinfe/semi-json-viewer-core" "2.90.3" + "@douyinfe/semi-animation" "2.90.10" + "@douyinfe/semi-json-viewer-core" "2.90.10" "@mdx-js/mdx" "^3.0.1" async-validator "^3.5.0" classnames "^2.2.6" @@ -1038,44 +1038,44 @@ remark-gfm "^4.0.0" scroll-into-view-if-needed "^2.2.24" -"@douyinfe/semi-icons@2.90.3", "@douyinfe/semi-icons@^2.90.3": - version "2.90.3" - resolved "https://registry.yarnpkg.com/@douyinfe/semi-icons/-/semi-icons-2.90.3.tgz#9dc40a743951aed7ea5a709adc248f89a97ec3d9" - integrity sha512-1zeBiBtpnRTblJo1OWTRv1W0ULYdAb4XAF9K4AF7YSnYw+jvHn0zulaQyBqGsuCKeJlfOaCKSui/5HjDt1RgJg== +"@douyinfe/semi-icons@2.90.10", "@douyinfe/semi-icons@^2.90.10": + version "2.90.10" + resolved "https://registry.yarnpkg.com/@douyinfe/semi-icons/-/semi-icons-2.90.10.tgz#7115167226ea2c7c70aad030f3df674668878c7c" + integrity sha512-8QY24bjsqW3Vpm3RCfsmYo8NBLa/9mJmRe2ycPazHq6YRM2xVgWyqkd7HWlJiRJy/4HDGVKxIwPlujDEHyDWLg== dependencies: classnames "^2.2.6" -"@douyinfe/semi-illustrations@2.90.3": - version "2.90.3" - resolved "https://registry.yarnpkg.com/@douyinfe/semi-illustrations/-/semi-illustrations-2.90.3.tgz#03cb5f2edb909fe0f11ef907f4061597053ca7d3" - integrity sha512-RXTVyAwRNiFEa9jXSGANCpm8aN2K4iU1knffmVNaRirGmMc5h6b8Uq7ZkVezizegSW6ETtucpFa2NmJX53MpPg== +"@douyinfe/semi-illustrations@2.90.10": + version "2.90.10" + resolved "https://registry.yarnpkg.com/@douyinfe/semi-illustrations/-/semi-illustrations-2.90.10.tgz#c74cbc2cc96370dba4784b1870000aff0b1abe86" + integrity sha512-xZ3MJLrW4YcdSyps/h83AEvrxggcXOZnc5ZwGE+K3IvCV7jjye/0L7dMG7iJEoOMaSDNF2MqQJSVUoDnHV+jBA== -"@douyinfe/semi-json-viewer-core@2.90.3": - version "2.90.3" - resolved "https://registry.yarnpkg.com/@douyinfe/semi-json-viewer-core/-/semi-json-viewer-core-2.90.3.tgz#eb38c59377dd2954194f8474938bcea106e6e267" - integrity sha512-18d2TuNdwXPt08b+ODiBSehyuKrvdZuF+2hVYnL3R7KRDCdd/BMLbhqPmr9IA92aP3u/TQKjRjIKhgpqw6XCQw== +"@douyinfe/semi-json-viewer-core@2.90.10": + version "2.90.10" + resolved "https://registry.yarnpkg.com/@douyinfe/semi-json-viewer-core/-/semi-json-viewer-core-2.90.10.tgz#659bcbb485ac1bd42e45e462b38abf4acc4ead03" + integrity sha512-0sLzUqR9JF5AxVWDjfpeZDqAi+olcb7ZM/hVKCQxPZEcF4VTQwB/cseg55Zbx3oMkC+xtLjicZy4ymfCLgW22Q== dependencies: jsonc-parser "^3.3.1" -"@douyinfe/semi-theme-default@2.90.3": - version "2.90.3" - resolved "https://registry.yarnpkg.com/@douyinfe/semi-theme-default/-/semi-theme-default-2.90.3.tgz#a2ebde9d37bccb8ceec3ed9fdde0d18b573515f7" - integrity sha512-DfHJ/3jHaTMJfcuer8LTcrh84Vxixberpk0DyvzgwVD1T+zAOppElwWq0tGDc5lnDHMgCY4R9W4b0aUcLCB2qA== +"@douyinfe/semi-theme-default@2.90.10": + version "2.90.10" + resolved "https://registry.yarnpkg.com/@douyinfe/semi-theme-default/-/semi-theme-default-2.90.10.tgz#b1f3e0fb3b5bf8d52c2492cd4393a68df0b07c70" + integrity sha512-q8liKStjQBuhGcADO5b86rvSW/ZxOhi5Ydgcr7D2/HyTsvkZA7R4spROAfwVGKGkc8juJvGqn2NirU2zwxAuMA== -"@douyinfe/semi-ui@2.90.3": - version "2.90.3" - resolved "https://registry.yarnpkg.com/@douyinfe/semi-ui/-/semi-ui-2.90.3.tgz#465b0d94ae5316a57c8dd12a0c8d4c47061e2329" - integrity sha512-Ojrj2y6G/79A85khqP8IinZaPFIIC3kWO263ppYQm/3Vb+yvg1ih1XIFNkPrhqha4Hybs1g+1LxKhjdvkhURRg== +"@douyinfe/semi-ui@2.90.10": + version "2.90.10" + resolved "https://registry.yarnpkg.com/@douyinfe/semi-ui/-/semi-ui-2.90.10.tgz#486c1551a7616a8a073b4343756be25147c73ab0" + integrity sha512-BclA5Ir1OEzZpS965T/y51KLIxEwRpJlyV/E2Pljsm98keSpc/kkoh9fKFiPMqTafgiuuey8XWStyC88tYZGEg== dependencies: "@dnd-kit/core" "^6.0.8" "@dnd-kit/sortable" "^7.0.2" "@dnd-kit/utilities" "^3.2.1" - "@douyinfe/semi-animation" "2.90.3" - "@douyinfe/semi-animation-react" "2.90.3" - "@douyinfe/semi-foundation" "2.90.3" - "@douyinfe/semi-icons" "2.90.3" - "@douyinfe/semi-illustrations" "2.90.3" - "@douyinfe/semi-theme-default" "2.90.3" + "@douyinfe/semi-animation" "2.90.10" + "@douyinfe/semi-animation-react" "2.90.10" + "@douyinfe/semi-foundation" "2.90.10" + "@douyinfe/semi-icons" "2.90.10" + "@douyinfe/semi-illustrations" "2.90.10" + "@douyinfe/semi-theme-default" "2.90.10" "@tiptap/core" "^3.10.7" "@tiptap/extension-document" "^3.10.7" "@tiptap/extension-hard-break" "^3.10.7" @@ -2146,10 +2146,10 @@ bl@^4.0.3: inherits "^2.0.4" readable-stream "^3.4.0" -body-parser@2.2.1: - version "2.2.1" - resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-2.2.1.tgz#6df606b0eb0a6e3f783dde91dde182c24c82438c" - integrity sha512-nfDwkulwiZYQIGwxdy0RUmowMhKcFVcYXUU7m4QlKYim1rUtg83xm2yjZ40QjDuc291AJjjeSc9b++AWHSgSHw== +body-parser@2.2.2: + version "2.2.2" + resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-2.2.2.tgz#1a32cdb966beaf68de50a9dfbe5b58f83cb8890c" + integrity sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA== dependencies: bytes "^3.1.2" content-type "^1.0.5" @@ -2157,7 +2157,7 @@ body-parser@2.2.1: http-errors "^2.0.0" iconv-lite "^0.7.0" on-finished "^2.4.1" - qs "^6.14.0" + qs "^6.14.1" raw-body "^3.0.1" type-is "^2.0.1" @@ -5946,10 +5946,10 @@ puppeteer@^24.34.0: puppeteer-core "24.34.0" typed-query-selector "^2.12.0" -qs@^6.14.0: - version "6.14.0" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.14.0.tgz#c63fa40680d2c5c941412a0e899c89af60c0a930" - integrity sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w== +qs@^6.14.1: + version "6.14.1" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.14.1.tgz#a41d85b9d3902f31d27861790506294881871159" + integrity sha512-4EK3+xJl8Ts67nLYNwqw/dsFVnCf+qR7RgXSK9jEEm9unao3njwMDdmsdvoKBKHzxd7tCYz5e5M+SnMjdtXGQQ== dependencies: side-channel "^1.1.0" @@ -6033,17 +6033,17 @@ react-resizable@^3.0.5: prop-types "15.x" react-draggable "^4.0.3" -react-router-dom@7.11.0: - version "7.11.0" - resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-7.11.0.tgz#2165f63e52798bd0eb138480c098ad058cdf3413" - integrity sha512-e49Ir/kMGRzFOOrYQBdoitq3ULigw4lKbAyKusnvtDu2t4dBX4AGYPrzNvorXmVuOyeakai6FUPW5MmibvVG8g== +react-router-dom@7.12.0: + version "7.12.0" + resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-7.12.0.tgz#0f2a059c6b2c4ae04474fe4171c59fb48b9fb8cf" + integrity sha512-pfO9fiBcpEfX4Tx+iTYKDtPbrSLLCbwJ5EqP+SPYQu1VYCXdy79GSj0wttR0U4cikVdlImZuEZ/9ZNCgoaxwBA== dependencies: - react-router "7.11.0" + react-router "7.12.0" -react-router@7.11.0: - version "7.11.0" - resolved "https://registry.yarnpkg.com/react-router/-/react-router-7.11.0.tgz#d3b91567fdbe910caf9064ea69b7b4d9264f2945" - integrity sha512-uI4JkMmjbWCZc01WVP2cH7ZfSzH91JAZUDd7/nIprDgWxBV1TkkmLToFh7EbMTcMak8URFRa2YoBL/W8GWnCTQ== +react-router@7.12.0: + version "7.12.0" + resolved "https://registry.yarnpkg.com/react-router/-/react-router-7.12.0.tgz#459a86862abbedd02e76e686751fe71f9fd73a4f" + integrity sha512-kTPDYPFzDVGIIGNLS5VJykK0HfHLY5MF3b+xj0/tTyNYL1gF1qs7u67Z9jEhQk2sQ98SUaHxlG31g1JtF7IfVw== dependencies: cookie "^1.0.1" set-cookie-parser "^2.6.0" @@ -7234,10 +7234,10 @@ vfile@^6.0.0: "@types/unist" "^3.0.0" vfile-message "^4.0.0" -vite@7.3.0: - version "7.3.0" - resolved "https://registry.yarnpkg.com/vite/-/vite-7.3.0.tgz#066c7a835993a66e82004eac3e185d0d157fd658" - integrity sha512-dZwN5L1VlUBewiP6H9s2+B3e3Jg96D0vzN+Ry73sOefebhYr9f94wwkMNN/9ouoU8pV1BqA1d1zGk8928cx0rg== +vite@7.3.1: + version "7.3.1" + resolved "https://registry.yarnpkg.com/vite/-/vite-7.3.1.tgz#7f6cfe8fb9074138605e822a75d9d30b814d6507" + integrity sha512-w+N7Hifpc3gRjZ63vYBXA56dvvRlNWRczTdmCBBa+CotUzAPf5b7YMdMR/8CQoeYE5LX3W4wj6RYTgonm1b9DA== dependencies: esbuild "^0.27.0" fdir "^6.5.0"