diff --git a/README.md b/README.md index 77b0456..e1e983b 100755 --- a/README.md +++ b/README.md @@ -4,13 +4,13 @@ Searching an apartment in Germany can be a frustrating task. Not any longer though, as _Fredy_ will take over and will only notify you once new listings have been found that match your requirements. -_Fredy_ scrapes multiple services (Immonet, Immowelt etc.) and send new listings to you once they become available. The list of available services can easily be extended. For your convenience, _Fredy_ has a UI to help you configure your search jobs. +_Fredy_ scrapes multiple services (Immonet, Immowelt etc.) and send new listings to you once they become available. The list of available services can easily be extended. For your convenience, _Fredy_ has a UI to help you configure your search jobs. If _Fredy_ finds matching results, it will send them to you via Slack, Email, Telegram etc. (More adapters can be configured.) As _Fredy_ stores the listings it has found, new results will not be sent to you twice (and as a side-effect, _Fredy_ can show some statistics). Furthermore, _Fredy_ checks duplicates per scraping so that the same listings are not being sent twice or more when posted on various platforms (which happens more often than one might think). Fredy - Find Real Estates Damn EasY - Your personal real estate search bot | Product Hunt -# Sponsorship [![](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/orangecoding) +# Sponsorship [![](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/orangecoding) If you like my work, consider becoming a sponsor. I'm not expecting anybody to pay for _Fredy_ or any other Open Source Project I'm maintaining, however keep in mind, I'm doing all of this in my spare time :) Thanks. [![JetBrains logo.](https://resources.jetbrains.com/storage/products/company/brand/logos/jetbrains.svg)](https://jb.gg/OpenSourceSupport) @@ -20,9 +20,9 @@ _Fredy_ is supported by JetBrains under Open Source Support Program ## Demo If you want to try out _Fredy_, you can access the demo version [here](https://fredy.orange-coding.net) 🤘 -## Usage +## Usage -- Make sure to use Node.js 22 or above +- Make sure to use Node.js 20 or above - Run the following commands: ```ssh yarn (or npm install) @@ -87,16 +87,16 @@ yarn run test Immoscout has implemented advanced bot detection. In order to work around this, we are using a reversed engineered version of their mobile api. See See [Immoscout Reverse Engineering Documentation](https://github.com/orangecoding/fredy/blob/master/reverse-engineered-immoscout.md) # Analytics -Fredy is completely free (and will always remain free). However, it would be a huge help if you’d allow me to collect some analytical data. +Fredy is completely free (and will always remain free). However, it would be a huge help if you’d allow me to collect some analytical data. Before you freak out, let me explain... If you agree, Fredy will send a ping to my Mixpanel project each time it runs. The data includes: names of active adapters/providers, OS, architecture, Node version, and language. The information is entirely anonymous and helps me understand which adapters/providers are most frequently used.

**Thanks**🤘 -# Docker -Use the Dockerfile in this repository to build an image. +# Docker +Use the Dockerfile in this repository to build an image. -Example: `docker build -t fredy/fredy /path/to/your/Dockerfile` +Example: `docker build -t fredy/fredy /path/to/your/Dockerfile` Or use docker-compose: @@ -106,12 +106,16 @@ Or use the container that will be built automatically. `docker pull ghcr.io/orangecoding/fredy:master` -## Create & run a container +## Create & run a container Put your config.json into a path of your choice, such as `/path/to/your/conf/`. Example: `docker create --name fredy -v /path/to/your/conf/:/conf -p 9998:9998 fredy/fredy` +## Logs + +You can browse the logs with `docker logs fredy -f`. + ### 👐 Contributing Thanks to all the people who already contributed! @@ -121,6 +125,7 @@ Thanks to all the people who already contributed! See [Contributing](https://github.com/orangecoding/fredy/blob/master/CONTRIBUTING.md) -## Logs -You can browse the logs with `docker logs fredy -f`. +## Star History + +[![Star History Chart](https://api.star-history.com/svg?repos=orangecoding/fredy&type=Date)](https://www.star-history.com/#orangecoding/fredy&Date) diff --git a/lib/provider/immowelt.js b/lib/provider/immowelt.js index c8272ba..e8477e8 100755 --- a/lib/provider/immowelt.js +++ b/lib/provider/immowelt.js @@ -23,7 +23,7 @@ const config = { id: 'a@href', price: 'div[data-testid="cardmfe-price-testid"] | removeNewline | trim', size: 'div[data-testid="cardmfe-keyfacts-testid"] | removeNewline | trim', - title: '.css-jv3zx6', + title: 'div[data-testid="cardmfe-description-box-text-test-id"] > div:nth-of-type(2)', link: 'a@href', address: 'div[data-testid="cardmfe-description-box-address"] | removeNewline | trim', }, diff --git a/lib/services/immoscout/immoscout-web-translater.js b/lib/services/immoscout/immoscout-web-translater.js index a7b61a7..18a21e0 100644 --- a/lib/services/immoscout/immoscout-web-translater.js +++ b/lib/services/immoscout/immoscout-web-translater.js @@ -79,6 +79,7 @@ const PARAM_NAME_MAP = { geocoordinates: 'geocoordinates', shape: 'shape', sorting: 'sorting', + newbuilding: 'newbuilding', }; const EQUIPMENT_MAP = { @@ -89,6 +90,7 @@ const EQUIPMENT_MAP = { garden: 'garden', guesttoilet: 'guestToilet', balcony: 'balcony', + handicappedaccessible: 'handicappedAccessible', }; const REAL_ESTATE_TYPE = { @@ -98,6 +100,29 @@ const REAL_ESTATE_TYPE = { 'haus-kaufen': 'housebuy', }; +const WEB_PATH_TO_APARTMENT_EQUIPMENT_MAP = { + // Category "Balkon/Terrasse" + 'wohnung-mit-balkon-mieten': { equipment: ['balcony'] }, + 'wohnung-mit-garten-mieten': { equipment: ['garden'] }, + // Category "Wohnungstyp" + 'souterrainwohnung-mieten': { apartmenttypes: ['halfbasement'] }, + 'erdgeschosswohnung-mieten': { apartmenttypes: ['groundfloor'] }, + 'hochparterrewohnung-mieten': { apartmenttypes: ['raisedgroundfloor'] }, + 'etagenwohnung-mieten': { apartmenttypes: ['apartment'] }, + 'loft-mieten': { apartmenttypes: ['loft'] }, + 'maisonette-mieten': { apartmenttypes: ['maisonette'] }, + 'terrassenwohnung-mieten': { apartmenttypes: ['terracedflat'] }, + 'penthouse-mieten': { apartmenttypes: ['penthouse'] }, + 'dachgeschosswohnung-mieten': { apartmenttypes: ['roofstorey'] }, + // Category "Ausstattung" + 'wohnung-mit-garage-mieten': { equipment: ['parking'] }, + 'wohnung-mit-einbaukueche-mieten': { equipment: ['builtinkitchen'] }, + 'wohnung-mit-keller-mieten': { equipment: ['cellar'] }, + // Category "Merkmale" + 'neubauwohnung-mieten': { newbuilding: true }, + 'barrierefreie-wohnung-mieten': { equipment: ['handicappedaccessible'] }, +}; + export function convertWebToMobile(webUrl) { let url; try { @@ -112,9 +137,17 @@ export function convertWebToMobile(webUrl) { } const realTypeKey = segments.at(-1); - const realType = REAL_ESTATE_TYPE[realTypeKey]; + let realType = REAL_ESTATE_TYPE[realTypeKey]; + let additionalParamsFromWebPath; + if (!realType) { - throw new Error(`Real estate type not found: ${realTypeKey}`); + // Test for seo optimized apartment path (only used on the ImmoScout web app) + if (WEB_PATH_TO_APARTMENT_EQUIPMENT_MAP[realTypeKey]) { + additionalParamsFromWebPath = WEB_PATH_TO_APARTMENT_EQUIPMENT_MAP[realTypeKey]; + realType = REAL_ESTATE_TYPE['wohnung-mieten']; + } else { + throw new Error(`Real estate type not found: ${realTypeKey}`); + } } if (segments.includes('shape')) { @@ -132,6 +165,7 @@ export function convertWebToMobile(webUrl) { searchType: isRadius ? 'radius' : 'region', realestatetype: realType, ...(isRadius ? {} : { geocodes }), + ...additionalParamsFromWebPath, }; if (webParams.geocoordinates) { @@ -141,7 +175,11 @@ export function convertWebToMobile(webUrl) { for (const [key, val] of Object.entries(webParams)) { if (key === 'equipment') { const items = [].concat(val).flatMap((v) => `${v}`.split(',')); - mobileParams[PARAM_NAME_MAP[key]] = items.map((item) => EQUIPMENT_MAP[item.toLowerCase()]).filter(Boolean); + const currentEquipmentParams = mobileParams[PARAM_NAME_MAP[key]]; + mobileParams[PARAM_NAME_MAP[key]] = [ + ...(currentEquipmentParams ?? []), + ...items.map((item) => EQUIPMENT_MAP[item.toLowerCase()]).filter(Boolean), + ]; } else { mobileParams[PARAM_NAME_MAP[key]] = val; } diff --git a/package.json b/package.json index 0c56181..af806de 100755 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "fredy", - "version": "12.0.0", + "version": "11.2.1", "description": "[F]ind [R]eal [E]states [d]amn eas[y].", "scripts": { "start": "node prod.js", diff --git a/test/provider/testProvider.json b/test/provider/testProvider.json index 5c9ff42..a0e0d00 100644 --- a/test/provider/testProvider.json +++ b/test/provider/testProvider.json @@ -37,7 +37,7 @@ "enabled": true }, "wgGesucht": { - "url": "https://www.wg-gesucht.de/wg-zimmer-in-Duesseldorf.30.0.1.0.html?offer_filter=1&noDeact=1&city_id=30&category=0&rent_type=0&rMax=5000", + "url": "https://www.wg-gesucht.de/wg-zimmer-in-Duesseldorf.30.0.1.0.html", "enabled": true } } diff --git a/test/services/immoscout/immoscout-web-translater.test.js b/test/services/immoscout/immoscout-web-translater.test.js index ef95272..ace88ab 100644 --- a/test/services/immoscout/immoscout-web-translater.test.js +++ b/test/services/immoscout/immoscout-web-translater.test.js @@ -16,6 +16,15 @@ describe('#immoscout-mobile URL conversion', () => { expect(actualMobileUrl).to.equal(expectedMobileUrl); }); + // Test URL conversion of web-only SEO path + it('should convert a SEO web path to the correct query params', () => { + const webUrl = 'https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-mit-balkon-mieten?equipment=garden'; + + const converted = convertWebToMobile(webUrl); + const queryParams = new URL(converted).searchParams; + expect(queryParams.get('equipment').split(',')).to.include.members(['garden', 'balcony']); + }); + // Test URL conversion with unsupported query parameters it('should remove unsupported query parameters', () => { const webUrl = 'https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-mieten?minimuminternetspeed=100000';