Merge branch 'master' of github.com:orangecoding/fredy

README.md

@@ -1,3 +1,16 @@
+<p align="center">
+  <a href="https://fredy.orange-coding.net/">
+    <img alt="Expo logo" width="400" src="https://github.com/orangecoding/fredy/blob/master/doc/logo.png">
+  </a>
+</p>
+
+
+ ![Tests](https://github.com/orangecoding/fredy/actions/workflows/test.yml/badge.svg)
+[![Docker](https://github.com/orangecoding/fredy/actions/workflows/docker.yml/badge.svg)](https://github.com/orangecoding/fredy/actions/workflows/docker.yml)
+![Source](https://github.com/orangecoding/fredy/actions/workflows/check_source.yml/badge.svg)
+![Docker Pulls](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fghcr-badge.elias.eu.org%2Fapi%2Forangecoding%2Ffredy%2Ffredy&query=%24.downloadCount&label=Docker%20Pulls)
+
+
 # Fredy 🏡 – Your Self-Hosted Real Estate Finder for Germany
 
 Finding an apartment or house in Germany can be stressful and
@@ -11,12 +24,7 @@ With a modern architecture, Fredy provides a **clean Web UI**, removes
 duplicates across platforms, and stores results so you never see the
 same listing twice.
 
-<img src="https://github.com/orangecoding/fredy/blob/master/doc/logo.png" width="400">
 
-![Tests](https://github.com/orangecoding/fredy/actions/workflows/test.yml/badge.svg)
-[![Docker](https://github.com/orangecoding/fredy/actions/workflows/docker.yml/badge.svg)](https://github.com/orangecoding/fredy/actions/workflows/docker.yml)
-![Source](https://github.com/orangecoding/fredy/actions/workflows/check_source.yml/badge.svg)
-![Docker Pulls](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fghcr-badge.elias.eu.org%2Fapi%2Forangecoding%2Ffredy%2Ffredy&query=%24.downloadCount&label=Docker%20Pulls)
 
 ------------------------------------------------------------------------
 
@@ -45,6 +53,11 @@ Fredy is proudly backed by the **JetBrains Open Source Support Program**.
 
 ------------------------------------------------------------------------
 
+## 👨‍🏫 Demo
+You can try out Fredy here: [Fredy Demo](https://fredy-demo.orange-coding.net/)
+
+------------------------------------------------------------------------
+
 ## 🚀 Quick Start
 
 ### With Docker
@@ -53,7 +66,11 @@ Fredy is proudly backed by the **JetBrains Open Source Support Program**.
 > In order to start Fredy, you must provide a config.json. As a start, use the one in this repo: https://github.com/orangecoding/fredy/blob/master/conf/config.json
 
 ``` bash
-docker run -d --name fredy -v fredy_conf:/conf -p 9998:9998 ghcr.io/orangecoding/fredy:master
+docker run -d --name fredy \
+  -v fredy_conf:/conf \
+  -v fredy_db:/db \
+  -p 9998:9998 \
+  ghcr.io/orangecoding/fredy:master
 ```
 
 Logs:
@@ -128,7 +145,7 @@ Immoscout has implemented advanced bot detection. In order to work around this,
 
 Fredy is completely free (and will always remain free). However, it would be a huge help if you’d allow me to collect some analytical data.
 Before you freak out, let me explain...  
-If you agree, Fredy will send a ping to my Mixpanel project each time it runs.  
+If you agree, Fredy will send a ping once every 6 hours to my internal tracking project (Will be open sourced soon).  
 The data includes: names of active adapters/providers, OS, architecture, Node version, and language. The information is entirely anonymous and helps me understand which adapters/providers are most frequently used.</p>
 
 **Thanks**🤘

index.js

@@ -6,9 +6,9 @@ import * as jobStorage from './lib/services/storage/jobStorage.js';
 import FredyRuntime from './lib/FredyRuntime.js';
 import { duringWorkingHoursOrNotSet } from './lib/utils.js';
 import './lib/api/api.js';
-import { track } from './lib/services/tracking/Tracker.js';
 import { handleDemoUser } from './lib/services/storage/userStorage.js';
 import { cleanupDemoAtMidnight } from './lib/services/demoCleanup.js';
+import { initTrackerCron } from './lib/services/tracking/Tracker-Cron.js';
 //if db folder does not exist, ensure to create it before loading anything else
 if (!fs.existsSync('./db')) {
   fs.mkdirSync('./db');
@@ -29,13 +29,13 @@ const fetchedProvider = await Promise.all(
 );
 
 handleDemoUser();
+await initTrackerCron();
 
 setInterval(
   (function exec() {
     const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
     if (!config.demoMode) {
       if (isDuringWorkingHoursOrNotSet) {
-        track();
         config.lastRun = Date.now();
         jobStorage
           .getJobs()

lib/api/routes/jobRouter.js

@@ -3,7 +3,6 @@ import * as jobStorage from '../../services/storage/jobStorage.js';
 import * as userStorage from '../../services/storage/userStorage.js';
 import { config } from '../../utils.js';
 import { isAdmin } from '../security.js';
-import { trackDemoJobCreated } from '../../services/tracking/Tracker.js';
 const service = restana();
 const jobRouter = service.newRouter();
 function doesJobBelongsToUser(job, req) {
@@ -46,11 +45,6 @@ jobRouter.post('/', async (req, res) => {
     res.send(new Error(error));
     console.error(error);
   }
-  trackDemoJobCreated({
-    name,
-    provider,
-    adapter: notificationAdapter,
-  });
   res.send();
 });
 jobRouter.delete('', async (req, res) => {

lib/api/routes/loginRoute.js

@@ -27,7 +27,7 @@ loginRouter.post('/', async (req, res) => {
   }
   if (user.password === hasher.hash(password)) {
     if (config.demoMode) {
-      trackDemoAccessed();
+      await trackDemoAccessed();
     }
 
     req.session.currentUser = user.id;

lib/notification/adapter/telegram.js

@@ -63,31 +63,41 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
   const jobName = job == null ? jobKey : job.name;
 
   const throttledCall = getThrottled(chatId, async function (endpoint, body) {
-    await fetch(`https://api.telegram.org/bot${token}/${endpoint}`, {
+    const res = await fetch(`https://api.telegram.org/bot${token}/${endpoint}`, {
       method: 'post',
       body: JSON.stringify(body),
       headers: { 'Content-Type': 'application/json' },
     });
+    return res;
   });
 
   const promises = newListings.map(async (o) => {
     const img = normalizeImageUrl(o.image);
+    const textPayload = {
+      chat_id: chatId,
+      text: buildText(jobName, serviceName, o),
+      parse_mode: 'HTML',
+      disable_web_page_preview: true,
+    };
 
-    if (img) {
+    if (!img) {
-      return throttledCall('sendPhoto', {
+      return throttledCall('sendMessage', textPayload);
+    }
+
+    try {
+      return await throttledCall('sendPhoto', {
         chat_id: chatId,
         photo: img,
         caption: buildCaption(jobName, serviceName, o),
         parse_mode: 'HTML',
       });
+    } catch (e) {
+      // If we see a timeout due to sending an image, try sending it without
+      if (e && (e.code === 'ETIMEDOUT' || e.errno === 'ETIMEDOUT')) {
+        return throttledCall('sendMessage', textPayload);
+      }
+      throw e;
     }
-
-    return throttledCall('sendMessage', {
-      chat_id: chatId,
-      text: buildText(jobName, serviceName, o),
-      parse_mode: 'HTML',
-      disable_web_page_preview: true,
-    });
   });
 
   return Promise.all(promises);

lib/services/extractor/parser/parser.js

@@ -66,7 +66,9 @@ export function parse(crawlContainer, crawlFields, text, url) {
     if (parsedObject.id != null) {
       result.push(parsedObject);
     } else {
-      console.warn('ID not found. Not relaying object.');
+      /* eslint-disable no-console */
+      console.debug('ID not found. Not relaying object.');
+      /* eslint-enable no-console */
     }
   });
 

lib/services/tracking/Tracker-Cron.js

@@ -0,0 +1,17 @@
+import cron from 'node-cron';
+import { config, inDevMode } from '../../utils.js';
+import { trackMainEvent } from './Tracker.js';
+
+async function runTask() {
+  //make sure to only send tracking events if the user gave us the green light and we are not in dev mode
+  if (config.analyticsEnabled && !inDevMode()) {
+    await trackMainEvent();
+  }
+}
+
+export async function initTrackerCron() {
+  //run directly on start
+  await runTask();
+  // then every 6 hours
+  cron.schedule('0 */6 * * *', runTask);
+}

lib/services/tracking/Tracker.js

@@ -1,65 +1,65 @@
-import Mixpanel from 'mixpanel';
 import { getJobs } from '../storage/jobStorage.js';
 import { getUniqueId } from './uniqueId.js';
 import { config, inDevMode } from '../../utils.js';
 import os from 'os';
 import { readFileSync } from 'fs';
 import { packageUp } from 'package-up';
+import fetch from 'node-fetch';
 
-const mixpanelTracker = Mixpanel.init('718670ef1c58c0208256c1e408a3d75e');
+const deviceId = getUniqueId() || 'N/A';
-const distinct_id = getUniqueId() || 'N/A';
 const version = await getPackageVersion();
+const FREDY_TRACKING_URL = 'https://fredy.orange-coding.net/tracking';
 
-export const track = function () {
+export const trackMainEvent = async () => {
-  //only send tracking information if the user allowed to do so.
+  try {
-  if (config.analyticsEnabled && !inDevMode()) {
+    if (config.analyticsEnabled && !inDevMode()) {
-    const activeProvider = new Set();
+      const activeProvider = new Set();
-    const activeAdapter = new Set();
+      const activeAdapter = new Set();
 
-    const jobs = getJobs();
+      const jobs = getJobs();
 
-    if (jobs != null && jobs.length > 0) {
+      if (jobs != null && jobs.length > 0) {
-      jobs.forEach((job) => {
+        jobs.forEach((job) => {
-        job.provider.forEach((provider) => {
+          job.provider.forEach((provider) => activeProvider.add(provider.id));
-          activeProvider.add(provider.id);
+          job.notificationAdapter.forEach((adapter) => activeAdapter.add(adapter.id));
         });
-        job.notificationAdapter.forEach((adapter) => {
-          activeAdapter.add(adapter.id);
-        });
-      });
 
-      mixpanelTracker.track(
+        const trackingObj = enrichTrackingObject({
-        'fredy_tracking',
-        enrichTrackingObject({
           adapter: Array.from(activeAdapter),
           provider: Array.from(activeProvider),
-        }),
+        });
-      );
+
+        await fetch(`${FREDY_TRACKING_URL}/main`, {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify(trackingObj),
+        });
+      }
     }
+  } catch (error) {
+    console.warn('Error sending tracking data', error);
   }
 };
 
 /**
  * Note, this will only be used when Fredy runs in demo mode
  */
-export function trackDemoJobCreated(jobData) {
+export async function trackDemoAccessed() {
   if (config.analyticsEnabled && !inDevMode() && config.demoMode) {
-    mixpanelTracker.track('demoJobCreated', enrichTrackingObject(jobData));
+    try {
-  }
+      await fetch(`${FREDY_TRACKING_URL}/demo/accessed`, {
-}
+        method: 'POST',
-
+        headers: { 'Content-Type': 'application/json' },
-/**
+      });
- * Note, this will only be used when Fredy runs in demo mode
+    } catch (error) {
- */
+      console.warn('Error sending tracking data', error);
-export function trackDemoAccessed() {
+    }
-  if (config.analyticsEnabled && !inDevMode() && config.demoMode) {
-    mixpanelTracker.track('demoAccessed', enrichTrackingObject({}));
   }
 }
 
 function enrichTrackingObject(trackingObject) {
-  const operating_system = os.platform();
+  const operatingSystem = os.platform();
-  const os_version = os.release();
+  const osVersion = os.release();
   const arch = process.arch;
   const language = process.env.LANG || 'en';
   const nodeVersion = process.version || 'N/A';
@@ -67,13 +67,13 @@ function enrichTrackingObject(trackingObject) {
   return {
     ...trackingObject,
     isDemo: config.demoMode,
-    operating_system,
+    operatingSystem,
-    os_version,
+    osVersion,
     arch,
     nodeVersion,
     language,
-    distinct_id,
+    deviceId,
-    fredy_version: version,
+    version,
   };
 }
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "11.6.0",
+  "version": "11.6.3",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "prepare": "husky",
@@ -70,8 +70,8 @@
     "lodash": "4.17.21",
     "lowdb": "7.0.1",
     "markdown": "^0.5.0",
-    "mixpanel": "^0.18.1",
     "nanoid": "5.1.5",
+    "node-cron": "^4.2.1",
     "node-fetch": "3.3.2",
     "node-mailjet": "6.0.9",
     "p-throttle": "^8.0.0",

ui/src/components/tracking/TrackingModal.jsx

@@ -43,7 +43,8 @@ export default function TrackingModal() {
         </p>
         <p>
           However, it would be a huge help if you’d allow me to collect some analytical data. Wait, before you click
-          "no", let me explain. If you agree, Fredy will send a ping to my Mixpanel project each time it runs.
+          "no", let me explain. If you agree, Fredy will send a ping once every 6 hours to my internal tracking project.
+          (Will be open-sourced soon)
         </p>
         <p>
           The data includes: names of active adapters/providers, OS, architecture, Node version, and language. The

yarn.lock

@@ -1979,13 +1979,6 @@ acorn@^8.0.0, acorn@^8.15.0:
   resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.15.0.tgz#a360898bc415edaac46c8241f6383975b930b816"
   integrity sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==
 
-agent-base@6:
-  version "6.0.2"
-  resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77"
-  integrity sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==
-  dependencies:
-    debug "4"
-
 agent-base@^7.1.0, agent-base@^7.1.2:
   version "7.1.4"
   resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-7.1.4.tgz#e3cd76d4c548ee895d3c3fd8dc1f6c5b9032e7a8"
@@ -4056,14 +4049,6 @@ http-proxy-agent@^7.0.0, http-proxy-agent@^7.0.1:
     agent-base "^7.1.0"
     debug "^4.3.4"
 
-https-proxy-agent@5.0.0:
-  version "5.0.0"
-  resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz#e2a90542abb68a762e0a0850f6c9edadfd8506b2"
-  integrity sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==
-  dependencies:
-    agent-base "6"
-    debug "4"
-
 https-proxy-agent@^7.0.6:
   version "7.0.6"
   resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz#da8dfeac7da130b05c2ba4b59c9b6cd66611a6b9"
@@ -5450,13 +5435,6 @@ mixin-object@^2.0.1:
     for-in "^0.1.3"
     is-extendable "^0.1.1"
 
-mixpanel@^0.18.1:
-  version "0.18.1"
-  resolved "https://registry.yarnpkg.com/mixpanel/-/mixpanel-0.18.1.tgz#beefdce6c260165f4e2059c8cdd34c5c557162f7"
-  integrity sha512-YD1xfn6WP6ZLQ6Pmgh0KgdXhueJEsrodThMTsHzHMH0VbWa9ck8s+ynDtM83OSgt+yQ61W/SQNrH8Y4wIwocGg==
-  dependencies:
-    https-proxy-agent "5.0.0"
-
 mkdirp-classic@^0.5.2, mkdirp-classic@^0.5.3:
   version "0.5.3"
   resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113"
@@ -5543,6 +5521,11 @@ node-abi@^3.3.0:
   dependencies:
     semver "^7.3.5"
 
+node-cron@^4.2.1:
+  version "4.2.1"
+  resolved "https://registry.yarnpkg.com/node-cron/-/node-cron-4.2.1.tgz#6979be4aee4702f06322d21220df8de252c8e265"
+  integrity sha512-lgimEHPE/QDgFlywTd8yTR61ptugX3Qer29efeyWw2rv259HtGBNn1vZVmp8lB9uo9wC0t/AT4iGqXxia+CJFg==
+
 node-domexception@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5"