Compare commits

...

25 Commits

Author SHA1 Message Date
Christian Kellner
206f768b41 next version 2025-07-25 13:21:12 +02:00
Alexander Roidl
2302f69ff3 Rename NPM startup scripts (#144)
* feat: rename npm start scripts
2025-07-25 13:13:04 +02:00
Alexander Roidl
9bb33e723a Workflow to check sourcecode's linting and formatting (#146)
* ci: workflow to check sourcecode

* fix: make workflow to check source fail for incorrect linting/formatting

* ci: change step name for workflow to check sourcecode
2025-07-23 08:58:43 +02:00
Alexander Roidl
cca1463a68 chore: run formatter (#145) 2025-07-23 08:47:26 +02:00
Alexander Roidl
314b1818d7 Formatting and linting pre-commit hook (#143) 2025-07-22 21:39:52 +02:00
Christian Kellner
25cc7fb650 next release version 2025-07-22 20:01:01 +02:00
Alexander Roidl
78df4b21a6 Remove leading commas from listings in Telegram messages (#142) 2025-07-22 19:58:16 +02:00
weakmap@gmail.com
d89b078237 lol 2025-07-19 22:41:30 +02:00
weakmap@gmail.com
395199a4a2 fixing duplicate provider removal / ugrade dependencies 2025-07-19 20:10:19 +02:00
weakmap@gmail.com
c2680fe49f next release version 2025-06-14 19:26:17 +02:00
weakmap@gmail.com
2b862b2d98 fixing blacklist 2025-06-14 19:25:52 +02:00
weakmap@gmail.com
9065448b6b upgrade dependencies 2025-06-14 19:12:55 +02:00
weakmap@gmail.com
b9f49cb5b2 upgrade dependencies 2025-06-14 19:06:27 +02:00
weakmap@gmail.com
53121742c2 improving error message 2025-06-14 19:03:23 +02:00
Christian Kellner
1a3eae0390 next version 2025-06-04 09:47:42 +02:00
Christian Kellner
a42905d63f fixing docker ignore issue 2025-06-04 09:46:07 +02:00
Christian Kellner
9917491728 Merge branch 'master' of github.com:orangecoding/fredy 2025-06-04 09:29:50 +02:00
Christian Kellner
f032e6a724 test: verify unrelated text yields no similarity (#130) 2025-06-04 09:15:53 +02:00
Christian Kellner
111c154ae3 Fix job ownership verification (#132) 2025-06-04 09:15:36 +02:00
Christian Kellner
2194ffe0f4 Fix typo in README (#133) 2025-06-04 09:15:15 +02:00
Christian Kellner
cfa25fc0e0 docs: fix adapter sentence (#131) 2025-06-04 09:14:57 +02:00
Christian Kellner
d50dd61f3e Merge branch 'master' of github.com:orangecoding/fredy 2025-06-04 09:12:00 +02:00
Christian Kellner
31e7f77bde uprade restana & vite 2025-05-27 12:01:26 +02:00
Christian Kellner
a418d64f1a uprade dependencies 2025-05-27 11:51:57 +02:00
Christian Kellner
d099872950 Update README.md 2025-05-26 13:23:36 +02:00
43 changed files with 1672 additions and 1338 deletions

View File

@@ -2,5 +2,6 @@ node_modules/
npm-debug.log npm-debug.log
test/ test/
db/ db/
conf/
.git/ .git/
.github/ .github/

26
.github/workflows/check_source.yml vendored Normal file
View File

@@ -0,0 +1,26 @@
name: Check the source code
on:
push:
branches: [master]
pull_request:
branches: [master]
jobs:
check_source_code:
name: Check the source code
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
cache: 'yarn'
- name: Install dependencies
run: yarn install
- name: Check formatting
run: yarn format:check
- name: Lint
run: yarn lint

View File

@@ -1,4 +1,4 @@
name: "Close stale issues and PRs" name: Close stale issues and PRs
on: on:
schedule: schedule:

View File

@@ -13,8 +13,7 @@ jobs:
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
- name: Setup Node.js - uses: actions/setup-node@v4
uses: actions/setup-node@v4
with: with:
node-version: 20 node-version: 20
cache: 'yarn' cache: 'yarn'

4
.prettierrc Normal file
View File

@@ -0,0 +1,4 @@
{
"singleQuote": true,
"printWidth": 120
}

View File

@@ -106,14 +106,14 @@ exports.config = {
``` ```
#### Running Tests #### Running Tests
If you've written a new provider you are an awesome person. If you now write tests for it, you are even more awesome. And who doesn't want to be more awesome right? If you've written a new provider you are an awesome person. If you now write tests for it, you are even more awesome. And who doesn't want to be more awesome, right?
#### Codestyle #### Codestyle
I'm using Eslint to maintain quote style and quality. Do not skip it... I'm using ESLint to maintain quote style and quality. Do not skip it...
##### To do before merging: ##### To-do before merging:
- executed tests? (`pnpm test`) - Have you executed the tests? (`yarn test`)
- sure the changes are useful for everybody? Or is it maybe a custom modification just for your case? - Are you sure the changes are useful for everybody? Or is it maybe a custom modification just for your case?
_Thanks!_ :heart: _Thanks!_ :heart:

View File

@@ -11,16 +11,16 @@ ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
# Copy lockfiles first to leverage cache for dependencies # Copy lockfiles first to leverage cache for dependencies
COPY package.json yarn.lock ./ COPY package.json yarn.lock .
# Set Yarn timeout, install dependencies and PM2 globally # Set Yarn timeout, install dependencies and PM2 globally
RUN yarn config set network-timeout 600000 \ RUN yarn config set network-timeout 600000 \
&& yarn install --frozen-lockfile \ && yarn --frozen-lockfile \
&& yarn global add pm2 && yarn global add pm2
# Copy application source and build production assets # Copy application source and build production assets
COPY . ./ COPY . .
RUN yarn run prod RUN yarn build:frontend
# Prepare runtime directories and symlinks for data and config # Prepare runtime directories and symlinks for data and config
RUN mkdir -p /db /conf \ RUN mkdir -p /db /conf \

View File

@@ -1,6 +1,6 @@
<img src="https://github.com/orangecoding/fredy/blob/master/doc/logo.png" width="400"> <img src="https://github.com/orangecoding/fredy/blob/master/doc/logo.png" width="400">
![Build Status](https://github.com/orangecoding/fredy/actions/workflows/test.yml/badge.svg) [![Create and publish Docker image](https://github.com/orangecoding/fredy/actions/workflows/docker.yml/badge.svg)](https://github.com/orangecoding/fredy/actions/workflows/docker.yml) ![Test](https://github.com/orangecoding/fredy/actions/workflows/test.yml/badge.svg) [![Create and publish Docker image](https://github.com/orangecoding/fredy/actions/workflows/docker.yml/badge.svg)](https://github.com/orangecoding/fredy/actions/workflows/docker.yml) ![Check the sourcecode](https://github.com/orangecoding/fredy/actions/workflows/check_source.yml/badge.svg)
Searching an apartment in Germany can be a frustrating task. Not any longer though, as _Fredy_ will take over and will only notify you once new listings have been found that match your requirements. Searching an apartment in Germany can be a frustrating task. Not any longer though, as _Fredy_ will take over and will only notify you once new listings have been found that match your requirements.
@@ -8,8 +8,6 @@ _Fredy_ scrapes multiple services (Immonet, Immowelt etc.) and send new listings
If _Fredy_ finds matching results, it will send them to you via Slack, Email, Telegram etc. (More adapters can be configured.) As _Fredy_ stores the listings it has found, new results will not be sent to you twice (and as a side-effect, _Fredy_ can show some statistics). Furthermore, _Fredy_ checks duplicates per scraping so that the same listings are not being sent twice or more when posted on various platforms (which happens more often than one might think). If _Fredy_ finds matching results, it will send them to you via Slack, Email, Telegram etc. (More adapters can be configured.) As _Fredy_ stores the listings it has found, new results will not be sent to you twice (and as a side-effect, _Fredy_ can show some statistics). Furthermore, _Fredy_ checks duplicates per scraping so that the same listings are not being sent twice or more when posted on various platforms (which happens more often than one might think).
<a href="https://www.producthunt.com/posts/fredy-find-real-estates-damn-easy?embed=true&utm_source=badge-featured&utm_medium=badge&utm_source=badge-fredy&#0045;find&#0045;real&#0045;estates&#0045;damn&#0045;easy" target="_blank"><img src="https://api.producthunt.com/widgets/embed-image/v1/featured.svg?post_id=965690&theme=light&t=1747292331626" alt="Fredy&#0032;&#0045;&#0032;Find&#0032;Real&#0032;Estates&#0032;Damn&#0032;EasY&#0032; - Your&#0032;personal&#0032;real&#0032;estate&#0032;search&#0032;bot | Product Hunt" style="width: 250px; height: 54px;" width="250" height="54" /></a>
# Sponsorship [![](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/orangecoding) # Sponsorship [![](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/orangecoding)
If you like my work, consider becoming a sponsor. I'm not expecting anybody to pay for _Fredy_ or any other Open Source Project I'm maintaining, however keep in mind, I'm doing all of this in my spare time :) Thanks. If you like my work, consider becoming a sponsor. I'm not expecting anybody to pay for _Fredy_ or any other Open Source Project I'm maintaining, however keep in mind, I'm doing all of this in my spare time :) Thanks.
@@ -25,9 +23,9 @@ If you want to try out _Fredy_, you can access the demo version [here](https://f
- Make sure to use Node.js 20 or above - Make sure to use Node.js 20 or above
- Run the following commands: - Run the following commands:
```ssh ```ssh
yarn (or npm install) yarn
yarn run prod yarn run start:backend
yarn run start yarn run start:frontend
``` ```
_Fredy_ will start with the default port, set to `9998`. You can access _Fredy_ by opening your browser at `http://localhost:9998`. The default login is `admin`, both for username and password. You should change the password as soon as possible when you plan to run Fredy on a server. _Fredy_ will start with the default port, set to `9998`. You can access _Fredy_ by opening your browser at `http://localhost:9998`. The default login is `admin`, both for username and password. You should change the password as soon as possible when you plan to run Fredy on a server.
@@ -48,7 +46,7 @@ A provider contains the URL that points to the search results for the respective
**It is important that you order the search results by date, so that _Fredy_ always picks the latest results first!** **It is important that you order the search results by date, so that _Fredy_ always picks the latest results first!**
#### Adapter #### Adapter
_Fredy_ supports multiple adapters, such as Slack, SendGrid, Telegram etc. A search job can have as many adapters as supported by _Fredy_. Each adapter needs different configuration values, which you have to provide when using them. A adapter dictactes how the frontend renders by telling the frontend what information it needs in order to send listings to the user. _Fredy_ supports multiple adapters, such as Slack, SendGrid, Telegram etc. A search job can have as many adapters as supported by _Fredy_. Each adapter needs different configuration values, which you have to provide when using them. An adapter dictates how the frontend renders by telling the frontend what information it needs in order to send listings to the user.
#### Jobs #### Jobs
A Job wraps adapters and providers. _Fredy_ runs the configured jobs in a specific interval (can be configured in `/conf/config.json`). A Job wraps adapters and providers. _Fredy_ runs the configured jobs in a specific interval (can be configured in `/conf/config.json`).
@@ -63,14 +61,13 @@ As an administrator, you can create, edit and remove users from _Fredy_. Be care
# Development # Development
### Running Fredy in development mode ### Running Fredy in development mode
To run _Fredy_ in development mode, you need to run the backend & frontend separately.
Start the backend with: Start the backend with:
```shell ```shell
yarn run start yarn run start:backend:dev
``` ```
For the frontend, run: For the frontend, run:
```shell ```shell
yarn run dev yarn run start:frontend:dev
``` ```
You should now be able to access _Fredy_ from your browser. Check your Terminal to see what port the frontend is running on. You should now be able to access _Fredy_ from your browser. Check your Terminal to see what port the frontend is running on.
@@ -84,7 +81,7 @@ yarn run test
![Architecture](/doc/architecture.jpg "Architecture") ![Architecture](/doc/architecture.jpg "Architecture")
### Immoscout ### Immoscout
Immoscout has implemented advanced bot detection. In order to work around this, we are using a reversed engineered version of their mobile api. See See [Immoscout Reverse Engineering Documentation](https://github.com/orangecoding/fredy/blob/master/reverse-engineered-immoscout.md) Immoscout has implemented advanced bot detection. In order to work around this, we are using a reversed engineered version of their mobile api. See [Immoscout Reverse Engineering Documentation](https://github.com/orangecoding/fredy/blob/master/reverse-engineered-immoscout.md)
# Analytics # Analytics
Fredy is completely free (and will always remain free). However, it would be a huge help if youd allow me to collect some analytical data. Fredy is completely free (and will always remain free). However, it would be a huge help if youd allow me to collect some analytical data.

View File

@@ -25,7 +25,7 @@ if(config.demoMode){
} }
/* eslint-enable no-console */ /* eslint-enable no-console */
const fetchedProvider = await Promise.all( const fetchedProvider = await Promise.all(
provider.filter((provider) => provider.endsWith('.js')).map(async (pro) => import(`${path}/${pro}`)) provider.filter((provider) => provider.endsWith('.js')).map(async (pro) => import(`${path}/${pro}`)),
); );
handleDemoUser(); handleDemoUser();
@@ -58,5 +58,5 @@ setInterval(
} }
return exec; return exec;
})(), })(),
INTERVAL INTERVAL,
); );

View File

@@ -15,7 +15,7 @@ function doesJobBelongsToUser(job, req) {
if (user == null) { if (user == null) {
return false; return false;
} }
return user.isAdmin || job.userId === job.userId; return user.isAdmin || job.userId === user.id;
} }
jobRouter.get('/', async (req, res) => { jobRouter.get('/', async (req, res) => {
const isUserAdmin = isAdmin(req); const isUserAdmin = isAdmin(req);

View File

@@ -22,16 +22,18 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
const { token, chatId } = notificationConfig.find((adapter) => adapter.id === config.id).fields; const { token, chatId } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
const job = getJob(jobKey); const job = getJob(jobKey);
const jobName = job == null ? jobKey : job.name; const jobName = job == null ? jobKey : job.name;
//we have to split messages into chunk, because otherwise messages are going to become too big and will fail // we have to split messages into chunks, because otherwise messages are going to become too big and will fail
const chunks = arrayChunks(newListings, MAX_ENTITIES_PER_CHUNK); const chunks = arrayChunks(newListings, MAX_ENTITIES_PER_CHUNK);
const promises = chunks.map((chunk) => { const promises = chunks.map((chunk) => {
let message = `<i>${jobName}</i> (${serviceName}) found <b>${newListings.length}</b> new listings:\n\n`; const messageParagraphs = [];
message += chunk.map(
messageParagraphs.push(`<i>${jobName}</i> (${serviceName}) found <b>${newListings.length}</b> new listings:`);
messageParagraphs.push(...chunk.map(
(o) => (o) =>
`<a href='${o.link}'><b>${shorten(o.title.replace(/\*/g, ''), 45).trim()}</b></a>\n` + `<a href='${o.link}'><b>${shorten(o.title.replace(/\*/g, ''), 45).trim()}</b></a>\n` +
[o.address, o.price, o.size].join(' | ') + [o.address, o.price, o.size].join(' | ')
'\n\n', ));
);
/** /**
* This is to not break the rate limit. It is to only send 1 message per second * This is to not break the rate limit. It is to only send 1 message per second
*/ */
@@ -41,7 +43,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
method: 'post', method: 'post',
body: JSON.stringify({ body: JSON.stringify({
chat_id: chatId, chat_id: chatId,
text: message, text: messageParagraphs.join('\n\n'),
parse_mode: 'HTML', parse_mode: 'HTML',
disable_web_page_preview: true, disable_web_page_preview: true,
}), }),

View File

@@ -6,7 +6,7 @@ const adapter = await Promise.all(
fs fs
.readdirSync('./lib/notification/adapter') .readdirSync('./lib/notification/adapter')
.filter((file) => file.endsWith('.js')) .filter((file) => file.endsWith('.js'))
.map(async (integPath) => await import(`${path}/${integPath}`)) .map(async (integPath) => await import(`${path}/${integPath}`)),
); );
if (adapter.length === 0) { if (adapter.length === 0) {

View File

@@ -36,7 +36,7 @@
*/ */
import utils, { buildHash } from '../utils.js'; import utils, { buildHash } from '../utils.js';
import { convertWebToMobile } from '../services/immoscout/immoscout-web-translater.js'; import { convertWebToMobile } from '../services/immoscout/immoscout-web-translator.js';
let appliedBlackList = []; let appliedBlackList = [];
async function getListings(url) { async function getListings(url) {

View File

@@ -7,7 +7,9 @@ function nullOrEmpty(val) {
} }
function normalize(o) { function normalize(o) {
const link = nullOrEmpty(o.link) ? 'NO LINK' : `https://www.neubaukompass.de${o.link.substring(o.link.indexOf('/neubau'))}`; const link = nullOrEmpty(o.link)
? 'NO LINK'
: `https://www.neubaukompass.de${o.link.substring(o.link.indexOf('/neubau'))}`;
const id = buildHash(o.link, o.price); const id = buildHash(o.link, o.price);
return Object.assign(o, { id, link }); return Object.assign(o, { id, link });
} }

View File

@@ -8,7 +8,7 @@ export function loadParser(text) {
export function parse(crawlContainer, crawlFields, text, url) { export function parse(crawlContainer, crawlFields, text, url) {
if (!text) { if (!text) {
console.warn('Cannot parse, text was empty for url ', url); console.warn('No content found for ', url);
return null; return null;
} }

View File

@@ -86,7 +86,7 @@ export const removeUser = (userId) => {
db.chain db.chain
.set( .set(
'user', 'user',
user.filter((u) => u.id !== userId) user.filter((u) => u.id !== userId),
) )
.value(); .value();
db.write(); db.write();
@@ -95,8 +95,12 @@ export const removeUser = (userId) => {
export const handleDemoUser = () => { export const handleDemoUser = () => {
if (!config.demoMode) { if (!config.demoMode) {
const user = db.chain.get('user').value(); const user = db.chain.get('user').value();
db.chain.get('user').value(); db.chain
db.chain.set('user', user.filter((u) => u.username !== 'demo')).value(); .set(
'user',
user.filter((u) => u.username !== 'demo'),
)
.value();
db.write(); db.write();
} else { } else {
const demoUser = db.chain const demoUser = db.chain
@@ -104,7 +108,8 @@ export const handleDemoUser = () => {
.filter((u) => u.username === 'demo') .filter((u) => u.username === 'demo')
.value(); .value();
if (demoUser == null || demoUser.length === 0) { if (demoUser == null || demoUser.length === 0) {
db.chain.get('user') db.chain
.get('user')
.value() .value()
.push({ .push({
id: nanoid(), id: nanoid(),
@@ -116,4 +121,3 @@ export const handleDemoUser = () => {
} }
} }
}; };

View File

@@ -9,12 +9,9 @@ function inDevMode(){
} }
function isOneOf(word, arr) { function isOneOf(word, arr) {
if (arr == null || arr.length === 0) { if (!arr || arr.length === 0 || word == null) return false;
return false; const lowerWord = word.toLowerCase();
} return arr.some(item => lowerWord.indexOf(item.toLowerCase()) !== -1);
const expression = String.raw`\b(${arr.join('|')})\b`;
const blacklist = new RegExp(expression, 'ig');
return blacklist.test(word);
} }
function nullOrEmpty(val) { function nullOrEmpty(val) {

View File

@@ -1,21 +1,25 @@
{ {
"name": "fredy", "name": "fredy",
"version": "11.2.2", "version": "11.3.0",
"description": "[F]ind [R]eal [E]states [d]amn eas[y].", "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
"scripts": { "scripts": {
"start": "node prod.js", "prepare": "husky",
"dev": "yarn && rm -rf ./ui/public/* && vite", "start:backend": "x-var NODE_ENV=production node index.js",
"ui": "rm -rf ./ui/public/* && vite", "start:backend:dev": "nodemon --watch index.js --watch lib",
"prod": "yarn && vite build --emptyOutDir", "start:frontend": "vite -m production",
"format": "prettier --write lib/**/*.js ui/src/**/*.jsx test/**/*.js *.js --single-quote --print-width 120", "start:frontend:dev": "vite",
"build:frontend": "vite build",
"format": "prettier --write lib/**/*.js ui/src/**/*.jsx test/**/*.js *.js",
"format:check": "prettier --check lib/**/*.js ui/src/**/*.jsx test/**/*.js *.js",
"test": "mocha --loader=esmock --timeout 3000000 test/**/*.test.js", "test": "mocha --loader=esmock --timeout 3000000 test/**/*.test.js",
"lint": "eslint ./index.js ./lib/**/*.js ./test/**/*.js ./ui/src/**/*.jsx" "lint": "eslint index.js lib/**/*.js test/**/*.js ui/src/**/*.jsx",
"lint:fix": "yarn lint --fix"
}, },
"type": "module", "type": "module",
"lint-staged": { "lint-staged": {
"*.js": [ "*.{js,jsx}": [
"eslint ./index.js ./lib/**/*.js ./test/**/*.js", "yarn lint",
"prettier --single-quote --print-width 120 --write" "yarn format"
] ]
}, },
"main": "index.js", "main": "index.js",
@@ -50,17 +54,17 @@
"Firefox ESR" "Firefox ESR"
], ],
"dependencies": { "dependencies": {
"@douyinfe/semi-ui": "2.79.0", "@douyinfe/semi-ui": "2.83.0",
"@rematch/core": "2.2.0", "@rematch/core": "2.2.0",
"@rematch/loading": "2.1.2", "@rematch/loading": "2.1.2",
"@sendgrid/mail": "8.1.5", "@sendgrid/mail": "8.1.5",
"@vitejs/plugin-react": "4.4.1", "@vitejs/plugin-react": "4.7.0",
"better-sqlite3": "^11.10.0", "better-sqlite3": "^11.10.0",
"body-parser": "2.2.0", "body-parser": "2.2.0",
"cheerio": "^1.0.0", "cheerio": "^1.1.0",
"cookie-session": "2.1.0", "cookie-session": "2.1.1",
"handlebars": "4.7.8", "handlebars": "4.7.8",
"highcharts": "12.2.0", "highcharts": "12.3.0",
"highcharts-react-official": "3.2.2", "highcharts-react-official": "3.2.2",
"lodash": "4.17.21", "lodash": "4.17.21",
"lowdb": "6.0.1", "lowdb": "6.0.1",
@@ -70,10 +74,10 @@
"node-fetch": "3.3.2", "node-fetch": "3.3.2",
"node-mailjet": "6.0.8", "node-mailjet": "6.0.8",
"package-up": "^5.0.0", "package-up": "^5.0.0",
"puppeteer": "^24.8.2", "puppeteer": "^24.14.0",
"puppeteer-extra": "^3.3.6", "puppeteer-extra": "^3.3.6",
"puppeteer-extra-plugin-stealth": "^2.11.2", "puppeteer-extra-plugin-stealth": "^2.11.2",
"query-string": "9.1.2", "query-string": "9.2.2",
"react": "18.3.1", "react": "18.3.1",
"react-dom": "18.3.1", "react-dom": "18.3.1",
"react-redux": "9.2.0", "react-redux": "9.2.0",
@@ -81,28 +85,30 @@
"react-router-dom": "5.3.0", "react-router-dom": "5.3.0",
"redux": "5.0.1", "redux": "5.0.1",
"redux-thunk": "3.1.0", "redux-thunk": "3.1.0",
"restana": "4.9.9", "restana": "5.0.0",
"serve-static": "1.16.2", "serve-static": "2.2.0",
"slack": "11.0.2", "slack": "11.0.2",
"string-similarity": "^4.0.4", "string-similarity": "^4.0.4",
"vite": "5.4.11" "vite": "7.0.5",
"x-var": "^2.1.0"
}, },
"devDependencies": { "devDependencies": {
"@babel/core": "7.27.1", "@babel/core": "7.27.3",
"@babel/eslint-parser": "7.27.1", "@babel/eslint-parser": "7.27.5",
"@babel/preset-env": "7.27.2", "@babel/preset-env": "7.27.2",
"@babel/preset-react": "7.27.1", "@babel/preset-react": "7.27.1",
"chai": "5.2.0", "chai": "5.2.1",
"eslint": "8.56.0", "eslint": "8.56.0",
"eslint-config-prettier": "8.8.0", "eslint-config-prettier": "8.8.0",
"eslint-plugin-react": "7.37.4", "eslint-plugin-react": "7.37.5",
"esmock": "2.7.0", "esmock": "2.7.1",
"history": "5.3.0", "history": "5.3.0",
"husky": "9.1.7", "husky": "9.1.7",
"less": "4.3.0", "less": "4.4.0",
"lint-staged": "15.5.2", "lint-staged": "15.5.2",
"mocha": "10.8.2", "mocha": "10.8.2",
"prettier": "3.5.3", "nodemon": "^3.1.10",
"prettier": "3.6.2",
"redux-logger": "3.0.6" "redux-logger": "3.0.6"
} }
} }

View File

@@ -1,2 +0,0 @@
process.env.NODE_ENV = 'production';
import('./index.js');

View File

@@ -77,4 +77,4 @@ curl -H "User-Agent: ImmoScout24_1410_30_._" \
## Parameters ## Parameters
The parameters between web and mobile are very different which is why we have to translate them. Please see `immoscout-web-translator.js`. The parameters between web and mobile are very different which is why we have to translate them. Please see [/lib/services/immoscout/immoscout-web-translator.js](https://github.com/orangecoding/fredy/blob/master/lib/services/immoscout/immoscout-web-translator.js).

View File

@@ -34,6 +34,7 @@ describe('similarityCheck', () => {
check.setCacheEntry( check.setCacheEntry(
'where |X| and |Y| are the cardinalities of the two sets (i.e. the number of elements in each set). The Sørensen index equals twice the number of elements common to both sets divided by the sum of the number of elements in each set.', 'where |X| and |Y| are the cardinalities of the two sets (i.e. the number of elements in each set). The Sørensen index equals twice the number of elements common to both sets divided by the sum of the number of elements in each set.',
); );
expect(check.hasSimilarEntries('unrelated text')).to.be.false;
}); });
}); });
}); });

View File

@@ -66,15 +66,17 @@ export default function FredyApp() {
{settings.demoMode && ( {settings.demoMode && (
<> <>
<Banner fullMode={true} <Banner
fullMode={true}
type="info" type="info"
bordered bordered
closeIcon={null} closeIcon={null}
description="You're currently viewing the demo version of Fredy. Jobs won't scrape websites, and any changes you make will be reverted at midnight." description="You're currently viewing the demo version of Fredy. Jobs won't scrape websites, and any changes you make will be reverted at midnight."
/> />
<br /> <br />
</>)} </>
{(settings.analyticsEnabled === null && !settings.demoMode) && <TrackingModal/>} )}
{settings.analyticsEnabled === null && !settings.demoMode && <TrackingModal />}
<Switch> <Switch>
<Route name="Insufficient Permission" path={'/403'} component={InsufficientPermission} /> <Route name="Insufficient Permission" path={'/403'} component={InsufficientPermission} />
<Route name="Create new Job" path={'/jobs/new'} component={JobMutation} /> <Route name="Create new Job" path={'/jobs/new'} component={JobMutation} />

View File

@@ -41,3 +41,7 @@ a:active {
background-color: transparent; background-color: transparent;
text-decoration: underline; text-decoration: underline;
} }
.semi-icon:not(.semi-tabs-bar .semi-tabs-tab .semi-icon) {
vertical-align: middle;
}

View File

@@ -23,5 +23,5 @@ root.render(
<App /> <App />
</LocaleProvider> </LocaleProvider>
</HashRouter> </HashRouter>
</Provider> </Provider>,
); );

View File

@@ -30,7 +30,7 @@ export default function ProviderTable({ providerData = [], onRemove } = {}) {
render: (_, record) => { render: (_, record) => {
return ( return (
<div style={{ float: 'right' }}> <div style={{ float: 'right' }}>
<Button type="danger" icon={<IconDelete />} onClick={() => onRemove(record.id)} /> <Button type="danger" icon={<IconDelete />} onClick={() => onRemove(record.url)} />
</div> </div>
); );
}, },

View File

@@ -4,16 +4,21 @@ import Logo from '../logo/Logo.jsx';
import { xhrPost } from '../../services/xhr.js'; import { xhrPost } from '../../services/xhr.js';
import './TrackingModal.less'; import './TrackingModal.less';
import inDevelopment from '../../services/developmentMode.js';
const saveResponse = async (analyticsEnabled) => { const saveResponse = async (analyticsEnabled) => {
await xhrPost('/api/admin/generalSettings', { await xhrPost('/api/admin/generalSettings', {
analyticsEnabled analyticsEnabled,
}); });
}; };
export default function TrackingModal() { export default function TrackingModal() {
if (inDevelopment()) {
return null;
}
return <Modal return (
<Modal
visible={true} visible={true}
onOk={async () => { onOk={async () => {
await saveResponse(true); await saveResponse(true);
@@ -32,17 +37,20 @@ export default function TrackingModal() {
<div className="trackingModal__description"> <div className="trackingModal__description">
<p>Hey 👋</p> <p>Hey 👋</p>
<p>Fed up with popups? Yeah, me too. But this ones important, and I promise it will only appear once ;)</p> <p>Fed up with popups? Yeah, me too. But this ones important, and I promise it will only appear once ;)</p>
<p>Fredy is completely free (and will always remain free). If youd like, you can support me by donating <p>
through my GitHub, but theres absolutely no obligation to do so.</p> Fredy is completely free (and will always remain free). If youd like, you can support me by donating through
<p>However, it would be a huge my GitHub, but theres absolutely no obligation to do so.
help if youd allow me to collect some analytical data. Wait, before you click "no", let me explain. If </p>
you <p>
agree, Fredy will send a ping to my Mixpanel project each time it runs.</p> However, it would be a huge help if youd allow me to collect some analytical data. Wait, before you click
<p>The data includes: names of "no", let me explain. If you agree, Fredy will send a ping to my Mixpanel project each time it runs.
active adapters/providers, OS, architecture, Node version, and language. The information is entirely </p>
anonymous and helps me understand which adapters/providers are most frequently used.</p> <p>
The data includes: names of active adapters/providers, OS, architecture, Node version, and language. The
information is entirely anonymous and helps me understand which adapters/providers are most frequently used.
</p>
<p>Thanks🤘</p> <p>Thanks🤘</p>
</div> </div>
</Modal>; </Modal>
);
} }

View File

@@ -0,0 +1,4 @@
export default function isDevelopmentMode(){
const inDevMode= import.meta.env.MODE;
return inDevMode != null && inDevMode === 'development';
}

View File

@@ -8,7 +8,14 @@ import Headline from '../../components/headline/Headline';
import { xhrPost } from '../../services/xhr'; import { xhrPost } from '../../services/xhr';
import { SegmentPart } from '../../components/segment/SegmentPart'; import { SegmentPart } from '../../components/segment/SegmentPart';
import { Banner, Toast } from '@douyinfe/semi-ui'; import { Banner, Toast } from '@douyinfe/semi-ui';
import {IconSave, IconCalendar, IconRefresh, IconSignal, IconLineChartStroked, IconSearch} from '@douyinfe/semi-icons'; import {
IconSave,
IconCalendar,
IconRefresh,
IconSignal,
IconLineChartStroked,
IconSearch,
} from '@douyinfe/semi-icons';
import './GeneralSettings.less'; import './GeneralSettings.less';
function formatFromTimestamp(ts) { function formatFromTimestamp(ts) {
@@ -97,7 +104,7 @@ const GeneralSettings = function GeneralSettings() {
to: workingHourTo, to: workingHourTo,
}, },
demoMode, demoMode,
analyticsEnabled analyticsEnabled,
}); });
} catch (exception) { } catch (exception) {
console.error(exception); console.error(exception);
@@ -175,24 +182,18 @@ const GeneralSettings = function GeneralSettings() {
</SegmentPart> </SegmentPart>
<Divider margin="1rem" /> <Divider margin="1rem" />
<SegmentPart <SegmentPart name="Analytics" helpText="Insights into the usage of Fredy." Icon={IconLineChartStroked}>
name="Analytics"
helpText="Insights into the usage of Fredy."
Icon={IconLineChartStroked}
>
<Banner <Banner
fullMode={false} fullMode={false}
type="info" type="info"
closeIcon={null} closeIcon={null}
title={ title={<div style={{ fontWeight: 600, fontSize: '14px', lineHeight: '20px' }}>Explanation</div>}
<div style={{fontWeight: 600, fontSize: '14px', lineHeight: '20px'}}>
Explanation
</div>
}
style={{ marginBottom: '1rem' }} style={{ marginBottom: '1rem' }}
description={ description={
<div> <div>
Analytics are disabled by default. If you choose to enable them, we will begin tracking the following:<br/> Analytics are disabled by default. If you choose to enable them, we will begin tracking the
following:
<br />
<ul> <ul>
<li>Name of active provider (e.g. Immoscout)</li> <li>Name of active provider (e.g. Immoscout)</li>
<li>Name of active adapter (e.g. Console)</li> <li>Name of active adapter (e.g. Console)</li>
@@ -201,35 +202,26 @@ const GeneralSettings = function GeneralSettings() {
<li>node version</li> <li>node version</li>
<li>arch</li> <li>arch</li>
</ul> </ul>
The data is sent anonymously and helps me understand which providers or adapters are being used the most. In the end it helps me to improve fredy. The data is sent anonymously and helps me understand which providers or adapters are being used the
most. In the end it helps me to improve fredy.
</div> </div>
} }
/> />
<Checkbox <Checkbox checked={analyticsEnabled} onChange={(e) => setAnalyticsEnabled(e.target.checked)}>
checked={analyticsEnabled} {' '}
onChange={(e) => setAnalyticsEnabled(e.target.checked)} Enabled
> Enabled
</Checkbox> </Checkbox>
</SegmentPart> </SegmentPart>
<Divider margin="1rem" /> <Divider margin="1rem" />
<SegmentPart <SegmentPart name="Demo Mode" helpText="If enabled, Fredy runs in demo mode." Icon={IconSearch}>
name="Demo Mode"
helpText="If enabled, Fredy runs in demo mode."
Icon={IconSearch}
>
<Banner <Banner
fullMode={false} fullMode={false}
type="info" type="info"
closeIcon={null} closeIcon={null}
title={ title={<div style={{ fontWeight: 600, fontSize: '14px', lineHeight: '20px' }}>Explanation</div>}
<div style={{fontWeight: 600, fontSize: '14px', lineHeight: '20px'}}>
Explanation
</div>
}
style={{ marginBottom: '1rem' }} style={{ marginBottom: '1rem' }}
description={ description={
<div> <div>
@@ -239,12 +231,10 @@ const GeneralSettings = function GeneralSettings() {
} }
/> />
<Checkbox <Checkbox checked={demoMode} onChange={(e) => setDemoMode(e.target.checked)}>
checked={demoMode} {' '}
onChange={(e) => setDemoMode(e.target.checked)} Enabled
> Enabled
</Checkbox> </Checkbox>
</SegmentPart> </SegmentPart>
<Divider margin="1rem" /> <Divider margin="1rem" />

View File

@@ -1,6 +1,6 @@
import React from 'react'; import React from 'react';
import { format } from '../../services/time/timeService'; import { format } from '../../services/time/timeService';
import {Banner, Descriptions} from '@douyinfe/semi-ui'; import { Descriptions } from '@douyinfe/semi-ui';
export default function ProcessingTimes({ processingTimes = {} }) { export default function ProcessingTimes({ processingTimes = {} }) {
if (Object.keys(processingTimes).length === 0) { if (Object.keys(processingTimes).length === 0) {

View File

@@ -94,7 +94,7 @@ export default function JobMutator() {
<form> <form>
<SegmentPart name="Name"> <SegmentPart name="Name">
<Input <Input
autofocus autoFocus
type="text" type="text"
maxLength={40} maxLength={40}
placeholder="Name" placeholder="Name"
@@ -124,8 +124,8 @@ export default function JobMutator() {
<ProviderTable <ProviderTable
providerData={providerData} providerData={providerData}
onRemove={(providerId) => { onRemove={(providerUrl) => {
setProviderData(providerData.filter((provider) => provider.id !== providerId)); setProviderData(providerData.filter((provider) => provider.url !== providerUrl));
}} }}
/> />
</SegmentPart> </SegmentPart>

View File

@@ -83,7 +83,7 @@ export default function NotificationAdapterMutator({
id: selectedAdapter.id, id: selectedAdapter.id,
name: selectedAdapter.name, name: selectedAdapter.name,
fields: selectedAdapter.fields || {}, fields: selectedAdapter.fields || {},
}) }),
); );
setSelectedAdapter(null); setSelectedAdapter(null);
@@ -114,7 +114,7 @@ export default function NotificationAdapterMutator({
setSuccessMessage('It seems like it worked! Please check your service.'); setSuccessMessage('It seems like it worked! Please check your service.');
}) })
.catch((error) => .catch((error) =>
setValidationMessage(`This did not work :-( I've received the following error: ${error.json.message}`) setValidationMessage(`This did not work :-( I've received the following error: ${error.json.message}`),
); );
}; };
@@ -229,7 +229,7 @@ export default function NotificationAdapterMutator({
.filter((option) => .filter((option) =>
editNotificationAdapter != null editNotificationAdapter != null
? true ? true
: selected.find((selectedOption) => selectedOption.id === option.key) == null : selected.find((selectedOption) => selectedOption.id === option.key) == null,
) )
.sort(sortAdapter)} .sort(sortAdapter)}
onChange={(value) => { onChange={(value) => {

View File

@@ -45,7 +45,7 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
url: providerUrl, url: providerUrl,
id: selectedProvider.id, id: selectedProvider.id,
name: selectedProvider.name, name: selectedProvider.name,
}) }),
); );
setProviderUrl(null); setProviderUrl(null);
setSelectedProvider(null); setSelectedProvider(null);
@@ -101,7 +101,7 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
description={ description={
<div> <div>
<p> <p>
Currently, our Immoscout implementation does not drawing shapes on a map. Use a radius instead. Currently, our Immoscout implementation does not support drawing shapes on a map. Use a radius instead.
</p> </p>
</div> </div>
} }

View File

@@ -87,12 +87,15 @@ export default function Login() {
Login Login
</Button> </Button>
<br /> <br />
{demoMode && <Banner fullMode={true} {demoMode && (
<Banner
fullMode={true}
type="info" type="info"
bordered bordered
closeIcon={null} closeIcon={null}
description="This is the demo version of Fredy. Use 'demo' as both the username and password to log in." description="This is the demo version of Fredy. Use 'demo' as both the username and password to log in."
/>} />
)}
</div> </div>
</form> </form>
</div> </div>

View File

@@ -6,6 +6,7 @@ export default defineConfig({
build: { build: {
chunkSizeWarningLimit: 9999999, chunkSizeWarningLimit: 9999999,
outDir: './ui/public', outDir: './ui/public',
emptyOutDir: true,
}, },
plugins: [react()], plugins: [react()],
server: { server: {

1407
yarn.lock

File diff suppressed because it is too large Load Diff