next release version

conf/config.json

@@ -1 +1 @@
-{"interval":"60","port":9998,"workingHours":{"from":"","to":""},"demoMode":false,"analyticsEnabled":null,"sqlitepath":"/db"}
+{"interval":"60","port":9998,"workingHours":{"from":"","to":""},"demoMode":false,"analyticsEnabled":true,"sqlitepath":"/db"}

index.js

@@ -10,6 +10,7 @@ import { ensureDemoUserExists, ensureAdminUserExists } from './lib/services/stor
 import { cleanupDemoAtMidnight } from './lib/services/demoCleanup.js';
 import { initTrackerCron } from './lib/services/tracking/Tracker-Cron.js';
 import logger from './lib/services/logger.js';
+import { bus } from './lib/services/events/event-bus.js';
 
 // Ensure sqlite directory exists before loading anything else (based on config.sqlitepath)
 const rawDir = config.sqlitepath || '/db';
@@ -45,29 +46,34 @@ ensureAdminUserExists();
 ensureDemoUserExists();
 await initTrackerCron();
 
-setInterval(
+bus.on('jobs:runAll', () => {
-  (function exec() {
+  logger.debug('Running Fredy Job manually');
-    const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
+  execute();
-    if (!config.demoMode) {
+});
-      if (isDuringWorkingHoursOrNotSet) {
+
-        config.lastRun = Date.now();
+const execute = () => {
-        jobStorage
+  const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
-          .getJobs()
+  if (!config.demoMode) {
-          .filter((job) => job.enabled)
+    if (isDuringWorkingHoursOrNotSet) {
-          .forEach((job) => {
+      config.lastRun = Date.now();
-            job.provider
+      jobStorage
-              .filter((p) => fetchedProvider.find((fp) => fp.metaInformation.id === p.id) != null)
+        .getJobs()
-              .forEach(async (prov) => {
+        .filter((job) => job.enabled)
-                const pro = fetchedProvider.find((fp) => fp.metaInformation.id === prov.id);
+        .forEach((job) => {
-                pro.init(prov, job.blacklist);
+          job.provider
-                await new FredyRuntime(pro.config, job.notificationAdapter, prov.id, job.id, similarityCache).execute();
+            .filter((p) => fetchedProvider.find((fp) => fp.metaInformation.id === p.id) != null)
-              });
+            .forEach(async (prov) => {
-          });
+              const pro = fetchedProvider.find((fp) => fp.metaInformation.id === prov.id);
-      } else {
+              pro.init(prov, job.blacklist);
-        logger.debug('Working hours set. Skipping as outside of working hours.');
+              await new FredyRuntime(pro.config, job.notificationAdapter, prov.id, job.id, similarityCache).execute();
-      }
+            });
+        });
+    } else {
+      logger.debug('Working hours set. Skipping as outside of working hours.');
     }
-    return exec;
+  }
-  })(),
+};
-  INTERVAL,
+
-);
+setInterval(execute, INTERVAL);
+//start once at startup
+execute();

lib/api/api.js

@@ -7,6 +7,7 @@ import { loginRouter } from './routes/loginRoute.js';
 import { config } from '../utils.js';
 import { userRouter } from './routes/userRoute.js';
 import { jobRouter } from './routes/jobRouter.js';
+import { versionRouter } from './routes/versionRouter.js';
 import bodyParser from 'body-parser';
 import restana from 'restana';
 import files from 'serve-static';
@@ -23,6 +24,7 @@ service.use(cookieSession());
 service.use(staticService);
 service.use('/api/admin', authInterceptor());
 service.use('/api/jobs', authInterceptor());
+service.use('/api/version', authInterceptor());
 // /admin can only be accessed when user is having admin permissions
 service.use('/api/admin', adminInterceptor());
 service.use('/api/jobs/notificationAdapter', notificationAdapterRouter);
@@ -30,6 +32,7 @@ service.use('/api/admin/generalSettings', generalSettingsRouter);
 service.use('/api/jobs/provider', providerRouter);
 service.use('/api/jobs/insights', analyticsRouter);
 service.use('/api/admin/users', userRouter);
+service.use('/api/version', versionRouter);
 service.use('/api/jobs', jobRouter);
 service.use('/api/login', loginRouter);
 //this route is unsecured intentionally as it is being queried from the login page

lib/api/routes/jobRouter.js

@@ -4,8 +4,11 @@ import * as userStorage from '../../services/storage/userStorage.js';
 import { config } from '../../utils.js';
 import { isAdmin } from '../security.js';
 import logger from '../../services/logger.js';
+import { bus } from '../../services/events/event-bus.js';
+
 const service = restana();
 const jobRouter = service.newRouter();
+
 function doesJobBelongsToUser(job, req) {
   const userId = req.session.currentUser;
   if (userId == null) {
@@ -17,6 +20,7 @@ function doesJobBelongsToUser(job, req) {
   }
   return user.isAdmin || job.userId === user.id;
 }
+
 jobRouter.get('/', async (req, res) => {
   const isUserAdmin = isAdmin(req);
   //show only the jobs which belongs to the user (or all of the user is an admin)
@@ -30,6 +34,12 @@ jobRouter.get('/processingTimes', async (req, res) => {
   };
   res.send();
 });
+
+jobRouter.post('/startAll', async (req, res) => {
+  bus.emit('jobs:runAll');
+  res.send();
+});
+
 jobRouter.post('/', async (req, res) => {
   const { provider, notificationAdapter, name, blacklist = [], jobId, enabled } = req.body;
   try {

lib/api/routes/versionRouter.js

@@ -0,0 +1,30 @@
+import restana from 'restana';
+import fetch from 'node-fetch';
+import { getPackageVersion } from '../../utils.js';
+
+const service = restana();
+const versionRouter = service.newRouter();
+
+versionRouter.get('/', async (req, res) => {
+  const versionPayload = await getCurrentVersionFromGithub();
+  res.body = versionPayload == null ? { newVersion: false } : versionPayload;
+  res.send();
+});
+
+async function getCurrentVersionFromGithub() {
+  const raw = await fetch('https://api.github.com/repos/orangecoding/fredy/releases/latest');
+  const data = await raw.json();
+  const localFredyVersion = await getPackageVersion();
+  if (localFredyVersion === data.tag_name) {
+    return null;
+  }
+  return {
+    newVersion: true,
+    version: data.tag_name,
+    url: data.html_url,
+    body: data.body,
+    localFredyVersion,
+  };
+}
+
+export { versionRouter };

lib/provider/immoscout.js

@@ -3,7 +3,7 @@
  *
  * The mobile API provides the following endpoints:
  * - GET /search/total?{search parameters}: Returns the total number of listings for the given query
- *   Example: `curl -H "User-Agent: ImmoScout24_1410_30_._" https://api.mobile.immobilienscout24.de/search/total?searchType=region&realestatetype=apartmentrent&pricetype=calculatedtotalrent&geocodes=%2Fde%2Fberlin%2Fberlin `
+ *   Example: `curl -H "User-Agent: ImmoScout_27.3_26.0_._" https://api.mobile.immobilienscout24.de/search/total?searchType=region&realestatetype=apartmentrent&pricetype=calculatedtotalrent&geocodes=%2Fde%2Fberlin%2Fberlin `
  *
  * - POST /search/list?{search parameters}: Actually retrieves the listings. Body is json encoded and contains
  *   data specifying additional results (advertisements) to return. The format is as follows:
@@ -15,12 +15,12 @@
  *   ```
  *   It is not necessary to provide data for the specified keys.
  *
- *   Example: `curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calculatedtotalrent&realestatetype=apartmentrent&searchType=region&geocodes=%2Fde%2Fberlin%2Fberlin&pagenumber=1' -H "Connection: keep-alive" -H "User-Agent: ImmoScout24_1410_30_._" -H "Accept: application/json" -H "Content-Type: application/json" -d '{"supportedResultListType": [], "userData": {}}'`
+ *   Example: `curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calculatedtotalrent&realestatetype=apartmentrent&searchType=region&geocodes=%2Fde%2Fberlin%2Fberlin&pagenumber=1' -H "Connection: keep-alive" -H "User-Agent: ImmoScout_27.3_26.0_._" -H "Accept: application/json" -H "Content-Type: application/json" -d '{"supportedResultListType": [], "userData": {}}'`
 
  * - GET /expose/{id} - Returns the details of a listing. The response contains additional details not included in the
  *   listing response.
  *
- *   Example: `curl -H "User-Agent: ImmoScout24_1410_30_._" "https://api.mobile.immobilienscout24.de/expose/158382494"`
+ *   Example: `curl -H "User-Agent: ImmoScout_27.3_26.0_._" "https://api.mobile.immobilienscout24.de/expose/158382494"`
  *
  *
  * It is necessary to set the correct User Agent (see `getListings`) in the request header.
@@ -44,7 +44,7 @@ async function getListings(url) {
   const response = await fetch(url, {
     method: 'POST',
     headers: {
-      'User-Agent': 'ImmoScout24_1410_30_._',
+      'User-Agent': 'ImmoScout_27.3_26.0_._',
       'Content-Type': 'application/json',
     },
     body: JSON.stringify({

lib/services/events/event-bus.js

@@ -0,0 +1,2 @@
+import { EventEmitter } from 'node:events';
+export const bus = new EventEmitter();

lib/services/extractor/parser/parser.js

@@ -21,7 +21,7 @@ export function parse(crawlContainer, crawlFields, text, url) {
   const result = [];
 
   if ($(crawlContainer).length === 0) {
-    logger.warn('No elements in crawl container found for url ', url);
+    logger.debug('No elements in crawl container found for url ', url);
     return null;
   }
 

lib/services/storage/listingsStorage.js

@@ -85,11 +85,11 @@ export const storeListings = (jobId, providerId, listings) => {
 
   SqliteConnection.withTransaction((db) => {
     const stmt = db.prepare(
-      `INSERT INTO listings (id, hash, provider, job_id, price, size, title, image_url, description, address, city,
+      `INSERT INTO listings (id, hash, provider, job_id, price, size, title, image_url, description, address,
                              link, created_at)
-       VALUES (@id, @hash, @provider, @job_id, @price, @size, @title, @image_url, @description, @address, @city, @link,
+       VALUES (@id, @hash, @provider, @job_id, @price, @size, @title, @image_url, @description, @address, @link,
                @created_at)
-       ON CONFLICT(hash) DO NOTHING`,
+       ON CONFLICT(job_id, hash) DO NOTHING`,
     );
 
     for (const item of listings) {

lib/services/storage/migrations/sql/2.new-indexes-for-listings.js

@@ -0,0 +1,10 @@
+// Migration: there needs to be a unique index on job_id and hash as only
+// this makes the listing indeed unique
+
+export function up(db) {
+  db.exec(`
+    DROP INDEX IF EXISTS idx_listings_hash;
+    CREATE UNIQUE INDEX IF NOT EXISTS idx_listings_job_hash
+      ON listings (job_id, hash);
+  `);
+}

lib/services/tracking/Tracker.js

@@ -1,9 +1,7 @@
 import { getJobs } from '../storage/jobStorage.js';
 import { getUniqueId } from './uniqueId.js';
-import { config, inDevMode } from '../../utils.js';
+import { config, getPackageVersion, inDevMode } from '../../utils.js';
 import os from 'os';
-import { readFileSync } from 'fs';
-import { packageUp } from 'package-up';
 import fetch from 'node-fetch';
 import logger from '../logger.js';
 
@@ -77,15 +75,3 @@ function enrichTrackingObject(trackingObject) {
     version,
   };
 }
-
-async function getPackageVersion() {
-  try {
-    const packagePath = await packageUp();
-    const packageJson = readFileSync(packagePath, 'utf8');
-    const json = JSON.parse(packageJson);
-    return json.version;
-  } catch (error) {
-    logger.error('Error reading version from package.json', error);
-  }
-  return 'N/A';
-}

lib/utils.js

@@ -3,8 +3,9 @@ import { fileURLToPath } from 'node:url';
 import { readFile } from 'fs/promises';
 import { createHash } from 'crypto';
 import { DEFAULT_CONFIG } from './defaultConfig.js';
-import fs from 'fs';
+import fs, { readFileSync } from 'fs';
 import logger from './services/logger.js';
+import { packageUp } from 'package-up';
 
 const RE_GT = />/g;
 const RE_WEBP = /\/format\/webp/gi;
@@ -196,6 +197,22 @@ const normalizeImageUrl = (url) => {
   return u;
 };
 
+/**
+ * returns Fredy's version
+ * @returns {Promise<*|string>}
+ */
+async function getPackageVersion() {
+  try {
+    const packagePath = await packageUp();
+    const packageJson = readFileSync(packagePath, 'utf8');
+    const json = JSON.parse(packageJson);
+    return json.version;
+  } catch (error) {
+    logger.error('Error reading version from package.json', error);
+  }
+  return 'N/A';
+}
+
 await refreshConfig();
 
 export { isOneOf };
@@ -206,6 +223,7 @@ export { duringWorkingHoursOrNotSet };
 export { getDirName };
 export { config };
 export { buildHash };
+export { getPackageVersion };
 export default {
   isOneOf,
   nullOrEmpty,

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "12.1.0",
+  "version": "12.1.3",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "prepare": "husky",
@@ -56,6 +56,7 @@
     "Firefox ESR"
   ],
   "dependencies": {
+    "@douyinfe/semi-icons": "^2.86.0",
     "@douyinfe/semi-ui": "2.86.0",
     "@sendgrid/mail": "8.1.5",
     "@visactor/react-vchart": "^2.0.4",

reverse-engineered-immoscout.md

@@ -41,7 +41,7 @@ Challenges:
 _Returns the total number of listings for the given query._
 
 ```
-curl -H "User-Agent: ImmoScout24_1410_30_._" \
+curl -H "User-Agent: ImmoScout_27.3_26.0_._" \
      -H "Accept: application/json" \
      "https://api.mobile.immobilienscout24.de/search/total?searchType=region&realestatetype=apartmentrent&pricetype=calculatedtotalrent&geocodes=%2Fde%2Fberlin%2Fberlin"
 ```
@@ -63,7 +63,7 @@ _The body is json encoded and contains data specifying additional results (adver
 ```
 curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calculatedtotalrent&realestatetype=apartmentrent&searchType=region&geocodes=%2Fde%2Fberlin%2Fberlin&pagenumber=1' \
   -H "Connection: keep-alive" \
-  -H "User-Agent: ImmoScout24_1410_30_._" \
+  -H "User-Agent: ImmoScout_27.3_26.0_._" \
   -H "Accept: application/json" \
   -H "Content-Type: application/json" \
   -d '{"supportedResultListType":[],"userData":{}}'
@@ -78,7 +78,7 @@ curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calc
 The response contains additional details not included in the listing response.
 
 ```
-curl -H "User-Agent: ImmoScout24_1410_30_._" \
+curl -H "User-Agent: ImmoScout_27.3_26.0_._" \
      -H "Accept: application/json" \
      "https://api.mobile.immobilienscout24.de/expose/158382494"
 ```

test/services/immoscout/immoscout-web-translator.test.js

@@ -53,7 +53,7 @@ describe('#immoscout-mobile URL conversion', () => {
       const response = await fetch(url, {
         method: 'POST',
         headers: {
-          'User-Agent': 'ImmoScout24_1410_30_._',
+          'User-Agent': 'ImmoScout_27.3_26.0_._',
           'Content-Type': 'application/json',
         },
         body: JSON.stringify({

ui/src/App.jsx

@@ -18,11 +18,13 @@ import Jobs from './views/jobs/Jobs';
 import './App.less';
 import TrackingModal from './components/tracking/TrackingModal.jsx';
 import { Banner } from '@douyinfe/semi-ui';
+import VersionBanner from './components/version/VersionBanner.jsx';
 
 export default function FredyApp() {
   const actions = useActions();
   const [loading, setLoading] = React.useState(true);
   const currentUser = useSelector((state) => state.user.currentUser);
+  const versionUpdate = useSelector((state) => state.versionUpdate.versionUpdate);
   const settings = useSelector((state) => state.generalSettings.settings);
 
   useEffect(() => {
@@ -34,6 +36,7 @@ export default function FredyApp() {
         await actions.jobs.getProcessingTimes();
         await actions.notificationAdapter.getAdapter();
         await actions.generalSettings.getGeneralSettings();
+        await actions.versionUpdate.getVersionUpdate();
       }
       setLoading(false);
     }
@@ -53,7 +56,6 @@ export default function FredyApp() {
       <Route path="*" element={<Navigate to="/login" replace />} />
     </Routes>
   );
-
   return loading ? null : needsLogin() ? (
     login()
   ) : (
@@ -62,7 +64,7 @@ export default function FredyApp() {
         <Logout />
         <Logo width={190} white />
         <Menu isAdmin={isAdmin()} />
-
+        {versionUpdate?.newVersion && <VersionBanner />}
         {settings.demoMode && (
           <>
             <Banner

ui/src/components/version/VersionBanner.jsx

@@ -0,0 +1,43 @@
+import React from 'react';
+import { Banner, Descriptions } from '@douyinfe/semi-ui';
+import { useSelector } from '../../services/state/store.js';
+
+import './VersionBanner.less';
+
+export default function VersionBanner() {
+  const versionUpdate = useSelector((state) => state.versionUpdate.versionUpdate);
+  return (
+    <Banner
+      className="versionBanner"
+      type="success"
+      icon={null}
+      description={
+        <div>
+          <p>A new version of Fredy is available. Update now to take advantage of the latest features and bug fixes.</p>
+          <Descriptions row size="small">
+            <Descriptions.Item itemKey="Your Version">{versionUpdate.localFredyVersion}</Descriptions.Item>
+            <Descriptions.Item itemKey="Latest Version">{versionUpdate.version}</Descriptions.Item>
+            <Descriptions.Item itemKey="Github Release">
+              <a href={versionUpdate.url} target="_blank" rel="noreferrer">
+                {versionUpdate.url}
+              </a>{' '}
+            </Descriptions.Item>
+          </Descriptions>
+          <p>
+            <b>
+              <small>Release Notes</small>
+            </b>
+          </p>
+          <pre>{stripFullChangelog(versionUpdate.body)}</pre>
+        </div>
+      }
+    />
+  );
+
+  function stripFullChangelog(text) {
+    if (text == null) {
+      return '';
+    }
+    return text.replace(/(?:\r?\n)\*\*Full Changelog\*\*[\s\S]*$/u, '');
+  }
+}

ui/src/components/version/VersionBanner.less

@@ -0,0 +1,3 @@
+.versionBanner {
+  margin-bottom: 1rem;
+}

ui/src/services/state/store.js

@@ -118,6 +118,18 @@ export const useFredyState = create(
             }
           },
         },
+        versionUpdate: {
+          async getVersionUpdate() {
+            try {
+              const response = await xhrGet('/api/version');
+              set((state) => ({
+                versionUpdate: { ...state.versionUpdate, versionUpdate: response.json },
+              }));
+            } catch (Exception) {
+              console.error('Error while trying to get resource for api/version. Error:', Exception);
+            }
+          },
+        },
       };
 
       // Initial state
@@ -125,6 +137,7 @@ export const useFredyState = create(
         notificationAdapter: [],
         generalSettings: { settings: {} },
         demoMode: { demoMode: false },
+        versionUpdate: {},
         provider: [],
         jobs: { jobs: [], insights: {}, processingTimes: {} },
         user: { users: [], currentUser: null },
@@ -135,6 +148,7 @@ export const useFredyState = create(
         notificationAdapter: { ...effects.notificationAdapter },
         generalSettings: { ...effects.generalSettings },
         demoMode: { ...effects.demoMode },
+        versionUpdate: { ...effects.versionUpdate },
         provider: { ...effects.provider },
         jobs: { ...effects.jobs },
         user: { ...effects.user },

ui/src/views/jobs/ProcessingTimes.jsx

@@ -1,6 +1,8 @@
 import React from 'react';
 import { format } from '../../services/time/timeService';
-import { Descriptions } from '@douyinfe/semi-ui';
+import { Button, Descriptions, Toast } from '@douyinfe/semi-ui';
+import { IconPlayCircle } from '@douyinfe/semi-icons';
+import { xhrPost } from '../../services/xhr.js';
 
 export default function ProcessingTimes({ processingTimes = {} }) {
   if (Object.keys(processingTimes).length === 0) {
@@ -24,6 +26,19 @@ export default function ProcessingTimes({ processingTimes = {} }) {
             <Descriptions.Item itemKey="Next run">
               {format(processingTimes.lastRun + processingTimes.interval * 60000)}
             </Descriptions.Item>
+            <Descriptions.Item itemKey="Find Listings now">
+              <Button
+                size="small"
+                icon={<IconPlayCircle />}
+                aria-label="Start now"
+                onClick={async () => {
+                  await xhrPost('/api/jobs/startAll', null);
+                  Toast.success('Successfully triggered Fredy search.');
+                }}
+              >
+                Search now
+              </Button>
+            </Descriptions.Item>
           </>
         )}
       </Descriptions>

yarn.lock

@@ -973,7 +973,7 @@
     remark-gfm "^4.0.0"
     scroll-into-view-if-needed "^2.2.24"
 
-"@douyinfe/semi-icons@2.86.0":
+"@douyinfe/semi-icons@2.86.0", "@douyinfe/semi-icons@^2.86.0":
   version "2.86.0"
   resolved "https://registry.yarnpkg.com/@douyinfe/semi-icons/-/semi-icons-2.86.0.tgz#ee4355c81616ea4325627a3bb607ed9f9b9afac3"
   integrity sha512-KEDlYYP1wdOqN28Ck0YcdCx7mSks8SRY4w4KKbXPaROzYNEyT2BRcJxwysMHfxL2IDfsroHrRPJsX9pnrmQqTg==