fixing immoscout

index.js

@@ -10,6 +10,7 @@ import { ensureDemoUserExists, ensureAdminUserExists } from './lib/services/stor
 import { cleanupDemoAtMidnight } from './lib/services/demoCleanup.js';
 import { initTrackerCron } from './lib/services/tracking/Tracker-Cron.js';
 import logger from './lib/services/logger.js';
+import { bus } from './lib/services/events/event-bus.js';
 
 // Ensure sqlite directory exists before loading anything else (based on config.sqlitepath)
 const rawDir = config.sqlitepath || '/db';
@@ -45,29 +46,34 @@ ensureAdminUserExists();
 ensureDemoUserExists();
 await initTrackerCron();
 
-setInterval(
-  (function exec() {
-    const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
-    if (!config.demoMode) {
-      if (isDuringWorkingHoursOrNotSet) {
-        config.lastRun = Date.now();
-        jobStorage
-          .getJobs()
-          .filter((job) => job.enabled)
-          .forEach((job) => {
-            job.provider
-              .filter((p) => fetchedProvider.find((fp) => fp.metaInformation.id === p.id) != null)
-              .forEach(async (prov) => {
-                const pro = fetchedProvider.find((fp) => fp.metaInformation.id === prov.id);
-                pro.init(prov, job.blacklist);
-                await new FredyRuntime(pro.config, job.notificationAdapter, prov.id, job.id, similarityCache).execute();
-              });
-          });
-      } else {
-        logger.debug('Working hours set. Skipping as outside of working hours.');
-      }
+bus.on('jobs:runAll', () => {
+  logger.debug('Running Fredy Job manually');
+  execute();
+});
+
+const execute = () => {
+  const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
+  if (!config.demoMode) {
+    if (isDuringWorkingHoursOrNotSet) {
+      config.lastRun = Date.now();
+      jobStorage
+        .getJobs()
+        .filter((job) => job.enabled)
+        .forEach((job) => {
+          job.provider
+            .filter((p) => fetchedProvider.find((fp) => fp.metaInformation.id === p.id) != null)
+            .forEach(async (prov) => {
+              const pro = fetchedProvider.find((fp) => fp.metaInformation.id === prov.id);
+              pro.init(prov, job.blacklist);
+              await new FredyRuntime(pro.config, job.notificationAdapter, prov.id, job.id, similarityCache).execute();
+            });
+        });
+    } else {
+      logger.debug('Working hours set. Skipping as outside of working hours.');
     }
-    return exec;
-  })(),
-  INTERVAL,
-);
+  }
+};
+
+setInterval(execute, INTERVAL);
+//start once at startup
+execute();

lib/api/routes/jobRouter.js

@@ -4,8 +4,11 @@ import * as userStorage from '../../services/storage/userStorage.js';
 import { config } from '../../utils.js';
 import { isAdmin } from '../security.js';
 import logger from '../../services/logger.js';
+import { bus } from '../../services/events/event-bus.js';
+
 const service = restana();
 const jobRouter = service.newRouter();
+
 function doesJobBelongsToUser(job, req) {
   const userId = req.session.currentUser;
   if (userId == null) {
@@ -17,6 +20,7 @@ function doesJobBelongsToUser(job, req) {
   }
   return user.isAdmin || job.userId === user.id;
 }
+
 jobRouter.get('/', async (req, res) => {
   const isUserAdmin = isAdmin(req);
   //show only the jobs which belongs to the user (or all of the user is an admin)
@@ -30,6 +34,12 @@ jobRouter.get('/processingTimes', async (req, res) => {
   };
   res.send();
 });
+
+jobRouter.post('/startAll', async (req, res) => {
+  bus.emit('jobs:runAll');
+  res.send();
+});
+
 jobRouter.post('/', async (req, res) => {
   const { provider, notificationAdapter, name, blacklist = [], jobId, enabled } = req.body;
   try {

lib/provider/immoscout.js

@@ -3,7 +3,7 @@
  *
  * The mobile API provides the following endpoints:
  * - GET /search/total?{search parameters}: Returns the total number of listings for the given query
- *   Example: `curl -H "User-Agent: ImmoScout24_1410_30_._" https://api.mobile.immobilienscout24.de/search/total?searchType=region&realestatetype=apartmentrent&pricetype=calculatedtotalrent&geocodes=%2Fde%2Fberlin%2Fberlin `
+ *   Example: `curl -H "User-Agent: ImmoScout_27.3_26.0_._" https://api.mobile.immobilienscout24.de/search/total?searchType=region&realestatetype=apartmentrent&pricetype=calculatedtotalrent&geocodes=%2Fde%2Fberlin%2Fberlin `
  *
  * - POST /search/list?{search parameters}: Actually retrieves the listings. Body is json encoded and contains
  *   data specifying additional results (advertisements) to return. The format is as follows:
@@ -15,12 +15,12 @@
  *   ```
  *   It is not necessary to provide data for the specified keys.
  *
- *   Example: `curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calculatedtotalrent&realestatetype=apartmentrent&searchType=region&geocodes=%2Fde%2Fberlin%2Fberlin&pagenumber=1' -H "Connection: keep-alive" -H "User-Agent: ImmoScout24_1410_30_._" -H "Accept: application/json" -H "Content-Type: application/json" -d '{"supportedResultListType": [], "userData": {}}'`
+ *   Example: `curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calculatedtotalrent&realestatetype=apartmentrent&searchType=region&geocodes=%2Fde%2Fberlin%2Fberlin&pagenumber=1' -H "Connection: keep-alive" -H "User-Agent: ImmoScout_27.3_26.0_._" -H "Accept: application/json" -H "Content-Type: application/json" -d '{"supportedResultListType": [], "userData": {}}'`
 
  * - GET /expose/{id} - Returns the details of a listing. The response contains additional details not included in the
  *   listing response.
  *
- *   Example: `curl -H "User-Agent: ImmoScout24_1410_30_._" "https://api.mobile.immobilienscout24.de/expose/158382494"`
+ *   Example: `curl -H "User-Agent: ImmoScout_27.3_26.0_._" "https://api.mobile.immobilienscout24.de/expose/158382494"`
  *
  *
  * It is necessary to set the correct User Agent (see `getListings`) in the request header.
@@ -44,7 +44,7 @@ async function getListings(url) {
   const response = await fetch(url, {
     method: 'POST',
     headers: {
-      'User-Agent': 'ImmoScout24_1410_30_._',
+      'User-Agent': 'ImmoScout_27.3_26.0_._',
       'Content-Type': 'application/json',
     },
     body: JSON.stringify({

lib/services/events/event-bus.js

@@ -0,0 +1,2 @@
+import { EventEmitter } from 'node:events';
+export const bus = new EventEmitter();

lib/services/extractor/parser/parser.js

@@ -21,7 +21,7 @@ export function parse(crawlContainer, crawlFields, text, url) {
   const result = [];
 
   if ($(crawlContainer).length === 0) {
-    logger.warn('No elements in crawl container found for url ', url);
+    logger.debug('No elements in crawl container found for url ', url);
     return null;
   }
 

lib/services/storage/listingsStorage.js

@@ -85,11 +85,11 @@ export const storeListings = (jobId, providerId, listings) => {
 
   SqliteConnection.withTransaction((db) => {
     const stmt = db.prepare(
-      `INSERT INTO listings (id, hash, provider, job_id, price, size, title, image_url, description, address, city,
+      `INSERT INTO listings (id, hash, provider, job_id, price, size, title, image_url, description, address,
                              link, created_at)
-       VALUES (@id, @hash, @provider, @job_id, @price, @size, @title, @image_url, @description, @address, @city, @link,
+       VALUES (@id, @hash, @provider, @job_id, @price, @size, @title, @image_url, @description, @address, @link,
                @created_at)
-       ON CONFLICT(hash) DO NOTHING`,
+       ON CONFLICT(job_id, hash) DO NOTHING`,
     );
 
     for (const item of listings) {

lib/services/storage/migrations/sql/2.new-indexes-for-listings.js

@@ -0,0 +1,10 @@
+// Migration: there needs to be a unique index on job_id and hash as only
+// this makes the listing indeed unique
+
+export function up(db) {
+  db.exec(`
+    DROP INDEX IF EXISTS idx_listings_hash;
+    CREATE UNIQUE INDEX IF NOT EXISTS idx_listings_job_hash
+      ON listings (job_id, hash);
+  `);
+}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "12.1.0",
+  "version": "12.1.2",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "prepare": "husky",
@@ -56,6 +56,7 @@
     "Firefox ESR"
   ],
   "dependencies": {
+    "@douyinfe/semi-icons": "^2.86.0",
     "@douyinfe/semi-ui": "2.86.0",
     "@sendgrid/mail": "8.1.5",
     "@visactor/react-vchart": "^2.0.4",

reverse-engineered-immoscout.md

@@ -41,7 +41,7 @@ Challenges:
 _Returns the total number of listings for the given query._
 
 ```
-curl -H "User-Agent: ImmoScout24_1410_30_._" \
+curl -H "User-Agent: ImmoScout_27.3_26.0_._" \
      -H "Accept: application/json" \
      "https://api.mobile.immobilienscout24.de/search/total?searchType=region&realestatetype=apartmentrent&pricetype=calculatedtotalrent&geocodes=%2Fde%2Fberlin%2Fberlin"
 ```
@@ -63,7 +63,7 @@ _The body is json encoded and contains data specifying additional results (adver
 ```
 curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calculatedtotalrent&realestatetype=apartmentrent&searchType=region&geocodes=%2Fde%2Fberlin%2Fberlin&pagenumber=1' \
   -H "Connection: keep-alive" \
-  -H "User-Agent: ImmoScout24_1410_30_._" \
+  -H "User-Agent: ImmoScout_27.3_26.0_._" \
   -H "Accept: application/json" \
   -H "Content-Type: application/json" \
   -d '{"supportedResultListType":[],"userData":{}}'
@@ -78,7 +78,7 @@ curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calc
 The response contains additional details not included in the listing response.
 
 ```
-curl -H "User-Agent: ImmoScout24_1410_30_._" \
+curl -H "User-Agent: ImmoScout_27.3_26.0_._" \
      -H "Accept: application/json" \
      "https://api.mobile.immobilienscout24.de/expose/158382494"
 ```

test/services/immoscout/immoscout-web-translator.test.js

@@ -53,7 +53,7 @@ describe('#immoscout-mobile URL conversion', () => {
       const response = await fetch(url, {
         method: 'POST',
         headers: {
-          'User-Agent': 'ImmoScout24_1410_30_._',
+          'User-Agent': 'ImmoScout_27.3_26.0_._',
           'Content-Type': 'application/json',
         },
         body: JSON.stringify({

ui/src/views/jobs/ProcessingTimes.jsx

@@ -1,6 +1,8 @@
 import React from 'react';
 import { format } from '../../services/time/timeService';
-import { Descriptions } from '@douyinfe/semi-ui';
+import { Button, Descriptions, Toast } from '@douyinfe/semi-ui';
+import { IconPlayCircle } from '@douyinfe/semi-icons';
+import { xhrPost } from '../../services/xhr.js';
 
 export default function ProcessingTimes({ processingTimes = {} }) {
   if (Object.keys(processingTimes).length === 0) {
@@ -24,6 +26,19 @@ export default function ProcessingTimes({ processingTimes = {} }) {
             <Descriptions.Item itemKey="Next run">
               {format(processingTimes.lastRun + processingTimes.interval * 60000)}
             </Descriptions.Item>
+            <Descriptions.Item itemKey="Find Listings now">
+              <Button
+                size="small"
+                icon={<IconPlayCircle />}
+                aria-label="Start now"
+                onClick={async () => {
+                  await xhrPost('/api/jobs/startAll', null);
+                  Toast.success('Successfully triggered Fredy search.');
+                }}
+              >
+                Search now
+              </Button>
+            </Descriptions.Item>
           </>
         )}
       </Descriptions>

yarn.lock

@@ -973,7 +973,7 @@
     remark-gfm "^4.0.0"
     scroll-into-view-if-needed "^2.2.24"
 
-"@douyinfe/semi-icons@2.86.0":
+"@douyinfe/semi-icons@2.86.0", "@douyinfe/semi-icons@^2.86.0":
   version "2.86.0"
   resolved "https://registry.yarnpkg.com/@douyinfe/semi-icons/-/semi-icons-2.86.0.tgz#ee4355c81616ea4325627a3bb607ed9f9b9afac3"
   integrity sha512-KEDlYYP1wdOqN28Ck0YcdCx7mSks8SRY4w4KKbXPaROzYNEyT2BRcJxwysMHfxL2IDfsroHrRPJsX9pnrmQqTg==