Compare commits

...

9 Commits

Author SHA1 Message Date
Christian Kellner
206f768b41 next version 2025-07-25 13:21:12 +02:00
Alexander Roidl
2302f69ff3 Rename NPM startup scripts (#144)
* feat: rename npm start scripts
2025-07-25 13:13:04 +02:00
Alexander Roidl
9bb33e723a Workflow to check sourcecode's linting and formatting (#146)
* ci: workflow to check sourcecode

* fix: make workflow to check source fail for incorrect linting/formatting

* ci: change step name for workflow to check sourcecode
2025-07-23 08:58:43 +02:00
Alexander Roidl
cca1463a68 chore: run formatter (#145) 2025-07-23 08:47:26 +02:00
Alexander Roidl
314b1818d7 Formatting and linting pre-commit hook (#143) 2025-07-22 21:39:52 +02:00
Christian Kellner
25cc7fb650 next release version 2025-07-22 20:01:01 +02:00
Alexander Roidl
78df4b21a6 Remove leading commas from listings in Telegram messages (#142) 2025-07-22 19:58:16 +02:00
weakmap@gmail.com
d89b078237 lol 2025-07-19 22:41:30 +02:00
weakmap@gmail.com
395199a4a2 fixing duplicate provider removal / ugrade dependencies 2025-07-19 20:10:19 +02:00
37 changed files with 1273 additions and 1047 deletions

26
.github/workflows/check_source.yml vendored Normal file
View File

@@ -0,0 +1,26 @@
name: Check the source code
on:
push:
branches: [master]
pull_request:
branches: [master]
jobs:
check_source_code:
name: Check the source code
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
cache: 'yarn'
- name: Install dependencies
run: yarn install
- name: Check formatting
run: yarn format:check
- name: Lint
run: yarn lint

View File

@@ -1,4 +1,4 @@
name: "Close stale issues and PRs"
name: Close stale issues and PRs
on:
schedule:

View File

@@ -13,8 +13,7 @@ jobs:
steps:
- uses: actions/checkout@v4
- name: Setup Node.js
uses: actions/setup-node@v4
- uses: actions/setup-node@v4
with:
node-version: 20
cache: 'yarn'

4
.prettierrc Normal file
View File

@@ -0,0 +1,4 @@
{
"singleQuote": true,
"printWidth": 120
}

View File

@@ -106,14 +106,14 @@ exports.config = {
```
#### Running Tests
If you've written a new provider you are an awesome person. If you now write tests for it, you are even more awesome. And who doesn't want to be more awesome right?
If you've written a new provider you are an awesome person. If you now write tests for it, you are even more awesome. And who doesn't want to be more awesome, right?
#### Codestyle
I'm using Eslint to maintain quote style and quality. Do not skip it...
I'm using ESLint to maintain quote style and quality. Do not skip it...
##### To do before merging:
##### To-do before merging:
- executed tests? (`pnpm test`)
- sure the changes are useful for everybody? Or is it maybe a custom modification just for your case?
- Have you executed the tests? (`yarn test`)
- Are you sure the changes are useful for everybody? Or is it maybe a custom modification just for your case?
_Thanks!_ :heart:

View File

@@ -11,16 +11,16 @@ ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
# Copy lockfiles first to leverage cache for dependencies
COPY package.json yarn.lock ./
COPY package.json yarn.lock .
# Set Yarn timeout, install dependencies and PM2 globally
RUN yarn config set network-timeout 600000 \
&& yarn install --frozen-lockfile \
&& yarn --frozen-lockfile \
&& yarn global add pm2
# Copy application source and build production assets
COPY . ./
RUN yarn run prod
COPY . .
RUN yarn build:frontend
# Prepare runtime directories and symlinks for data and config
RUN mkdir -p /db /conf \

View File

@@ -1,6 +1,6 @@
<img src="https://github.com/orangecoding/fredy/blob/master/doc/logo.png" width="400">
![Build Status](https://github.com/orangecoding/fredy/actions/workflows/test.yml/badge.svg) [![Create and publish Docker image](https://github.com/orangecoding/fredy/actions/workflows/docker.yml/badge.svg)](https://github.com/orangecoding/fredy/actions/workflows/docker.yml)
![Test](https://github.com/orangecoding/fredy/actions/workflows/test.yml/badge.svg) [![Create and publish Docker image](https://github.com/orangecoding/fredy/actions/workflows/docker.yml/badge.svg)](https://github.com/orangecoding/fredy/actions/workflows/docker.yml) ![Check the sourcecode](https://github.com/orangecoding/fredy/actions/workflows/check_source.yml/badge.svg)
Searching an apartment in Germany can be a frustrating task. Not any longer though, as _Fredy_ will take over and will only notify you once new listings have been found that match your requirements.
@@ -23,9 +23,9 @@ If you want to try out _Fredy_, you can access the demo version [here](https://f
- Make sure to use Node.js 20 or above
- Run the following commands:
```ssh
yarn (or npm install)
yarn run prod
yarn run start
yarn
yarn run start:backend
yarn run start:frontend
```
_Fredy_ will start with the default port, set to `9998`. You can access _Fredy_ by opening your browser at `http://localhost:9998`. The default login is `admin`, both for username and password. You should change the password as soon as possible when you plan to run Fredy on a server.
@@ -61,14 +61,13 @@ As an administrator, you can create, edit and remove users from _Fredy_. Be care
# Development
### Running Fredy in development mode
To run _Fredy_ in development mode, you need to run the backend & frontend separately.
Start the backend with:
```shell
yarn run start
yarn run start:backend:dev
```
For the frontend, run:
```shell
yarn run dev
yarn run start:frontend:dev
```
You should now be able to access _Fredy_ from your browser. Check your Terminal to see what port the frontend is running on.

View File

@@ -25,7 +25,7 @@ if(config.demoMode){
}
/* eslint-enable no-console */
const fetchedProvider = await Promise.all(
provider.filter((provider) => provider.endsWith('.js')).map(async (pro) => import(`${path}/${pro}`))
provider.filter((provider) => provider.endsWith('.js')).map(async (pro) => import(`${path}/${pro}`)),
);
handleDemoUser();
@@ -58,5 +58,5 @@ setInterval(
}
return exec;
})(),
INTERVAL
INTERVAL,
);

View File

@@ -22,16 +22,18 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
const { token, chatId } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
const job = getJob(jobKey);
const jobName = job == null ? jobKey : job.name;
//we have to split messages into chunk, because otherwise messages are going to become too big and will fail
// we have to split messages into chunks, because otherwise messages are going to become too big and will fail
const chunks = arrayChunks(newListings, MAX_ENTITIES_PER_CHUNK);
const promises = chunks.map((chunk) => {
let message = `<i>${jobName}</i> (${serviceName}) found <b>${newListings.length}</b> new listings:\n\n`;
message += chunk.map(
const messageParagraphs = [];
messageParagraphs.push(`<i>${jobName}</i> (${serviceName}) found <b>${newListings.length}</b> new listings:`);
messageParagraphs.push(...chunk.map(
(o) =>
`<a href='${o.link}'><b>${shorten(o.title.replace(/\*/g, ''), 45).trim()}</b></a>\n` +
[o.address, o.price, o.size].join(' | ') +
'\n\n',
);
[o.address, o.price, o.size].join(' | ')
));
/**
* This is to not break the rate limit. It is to only send 1 message per second
*/
@@ -41,7 +43,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
method: 'post',
body: JSON.stringify({
chat_id: chatId,
text: message,
text: messageParagraphs.join('\n\n'),
parse_mode: 'HTML',
disable_web_page_preview: true,
}),

View File

@@ -6,7 +6,7 @@ const adapter = await Promise.all(
fs
.readdirSync('./lib/notification/adapter')
.filter((file) => file.endsWith('.js'))
.map(async (integPath) => await import(`${path}/${integPath}`))
.map(async (integPath) => await import(`${path}/${integPath}`)),
);
if (adapter.length === 0) {

View File

@@ -36,7 +36,7 @@
*/
import utils, { buildHash } from '../utils.js';
import { convertWebToMobile } from '../services/immoscout/immoscout-web-translater.js';
import { convertWebToMobile } from '../services/immoscout/immoscout-web-translator.js';
let appliedBlackList = [];
async function getListings(url) {

View File

@@ -7,7 +7,9 @@ function nullOrEmpty(val) {
}
function normalize(o) {
const link = nullOrEmpty(o.link) ? 'NO LINK' : `https://www.neubaukompass.de${o.link.substring(o.link.indexOf('/neubau'))}`;
const link = nullOrEmpty(o.link)
? 'NO LINK'
: `https://www.neubaukompass.de${o.link.substring(o.link.indexOf('/neubau'))}`;
const id = buildHash(o.link, o.price);
return Object.assign(o, { id, link });
}

View File

@@ -1,21 +1,25 @@
{
"name": "fredy",
"version": "11.2.4",
"version": "11.3.0",
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
"scripts": {
"start": "node prod.js",
"dev": "yarn && rm -rf ./ui/public/* && vite",
"ui": "rm -rf ./ui/public/* && vite",
"prod": "yarn && vite build --emptyOutDir",
"format": "prettier --write lib/**/*.js ui/src/**/*.jsx test/**/*.js *.js --single-quote --print-width 120",
"prepare": "husky",
"start:backend": "x-var NODE_ENV=production node index.js",
"start:backend:dev": "nodemon --watch index.js --watch lib",
"start:frontend": "vite -m production",
"start:frontend:dev": "vite",
"build:frontend": "vite build",
"format": "prettier --write lib/**/*.js ui/src/**/*.jsx test/**/*.js *.js",
"format:check": "prettier --check lib/**/*.js ui/src/**/*.jsx test/**/*.js *.js",
"test": "mocha --loader=esmock --timeout 3000000 test/**/*.test.js",
"lint": "eslint ./index.js ./lib/**/*.js ./test/**/*.js ./ui/src/**/*.jsx"
"lint": "eslint index.js lib/**/*.js test/**/*.js ui/src/**/*.jsx",
"lint:fix": "yarn lint --fix"
},
"type": "module",
"lint-staged": {
"*.js": [
"eslint ./index.js ./lib/**/*.js ./test/**/*.js",
"prettier --single-quote --print-width 120 --write"
"*.{js,jsx}": [
"yarn lint",
"yarn format"
]
},
"main": "index.js",
@@ -50,17 +54,17 @@
"Firefox ESR"
],
"dependencies": {
"@douyinfe/semi-ui": "2.80.0",
"@douyinfe/semi-ui": "2.83.0",
"@rematch/core": "2.2.0",
"@rematch/loading": "2.1.2",
"@sendgrid/mail": "8.1.5",
"@vitejs/plugin-react": "4.5.2",
"@vitejs/plugin-react": "4.7.0",
"better-sqlite3": "^11.10.0",
"body-parser": "2.2.0",
"cheerio": "^1.1.0",
"cookie-session": "2.1.0",
"cookie-session": "2.1.1",
"handlebars": "4.7.8",
"highcharts": "12.2.0",
"highcharts": "12.3.0",
"highcharts-react-official": "3.2.2",
"lodash": "4.17.21",
"lowdb": "6.0.1",
@@ -70,10 +74,10 @@
"node-fetch": "3.3.2",
"node-mailjet": "6.0.8",
"package-up": "^5.0.0",
"puppeteer": "^24.10.1",
"puppeteer": "^24.14.0",
"puppeteer-extra": "^3.3.6",
"puppeteer-extra-plugin-stealth": "^2.11.2",
"query-string": "9.2.0",
"query-string": "9.2.2",
"react": "18.3.1",
"react-dom": "18.3.1",
"react-redux": "9.2.0",
@@ -85,24 +89,26 @@
"serve-static": "2.2.0",
"slack": "11.0.2",
"string-similarity": "^4.0.4",
"vite": "6.3.5"
"vite": "7.0.5",
"x-var": "^2.1.0"
},
"devDependencies": {
"@babel/core": "7.27.3",
"@babel/eslint-parser": "7.27.5",
"@babel/preset-env": "7.27.2",
"@babel/preset-react": "7.27.1",
"chai": "5.2.0",
"chai": "5.2.1",
"eslint": "8.56.0",
"eslint-config-prettier": "8.8.0",
"eslint-plugin-react": "7.37.5",
"esmock": "2.7.0",
"esmock": "2.7.1",
"history": "5.3.0",
"husky": "9.1.7",
"less": "4.3.0",
"less": "4.4.0",
"lint-staged": "15.5.2",
"mocha": "10.8.2",
"prettier": "3.5.3",
"nodemon": "^3.1.10",
"prettier": "3.6.2",
"redux-logger": "3.0.6"
}
}

View File

@@ -1,2 +0,0 @@
process.env.NODE_ENV = 'production';
import('./index.js');

View File

@@ -77,4 +77,4 @@ curl -H "User-Agent: ImmoScout24_1410_30_._" \
## Parameters
The parameters between web and mobile are very different which is why we have to translate them. Please see `immoscout-web-translator.js`.
The parameters between web and mobile are very different which is why we have to translate them. Please see [/lib/services/immoscout/immoscout-web-translator.js](https://github.com/orangecoding/fredy/blob/master/lib/services/immoscout/immoscout-web-translator.js).

View File

@@ -66,15 +66,17 @@ export default function FredyApp() {
{settings.demoMode && (
<>
<Banner fullMode={true}
<Banner
fullMode={true}
type="info"
bordered
closeIcon={null}
description="You're currently viewing the demo version of Fredy. Jobs won't scrape websites, and any changes you make will be reverted at midnight."
/>
<br />
</>)}
{(settings.analyticsEnabled === null && !settings.demoMode) && <TrackingModal/>}
</>
)}
{settings.analyticsEnabled === null && !settings.demoMode && <TrackingModal />}
<Switch>
<Route name="Insufficient Permission" path={'/403'} component={InsufficientPermission} />
<Route name="Create new Job" path={'/jobs/new'} component={JobMutation} />

View File

@@ -41,3 +41,7 @@ a:active {
background-color: transparent;
text-decoration: underline;
}
.semi-icon:not(.semi-tabs-bar .semi-tabs-tab .semi-icon) {
vertical-align: middle;
}

View File

@@ -23,5 +23,5 @@ root.render(
<App />
</LocaleProvider>
</HashRouter>
</Provider>
</Provider>,
);

View File

@@ -30,7 +30,7 @@ export default function ProviderTable({ providerData = [], onRemove } = {}) {
render: (_, record) => {
return (
<div style={{ float: 'right' }}>
<Button type="danger" icon={<IconDelete />} onClick={() => onRemove(record.id)} />
<Button type="danger" icon={<IconDelete />} onClick={() => onRemove(record.url)} />
</div>
);
},

View File

@@ -4,16 +4,21 @@ import Logo from '../logo/Logo.jsx';
import { xhrPost } from '../../services/xhr.js';
import './TrackingModal.less';
import inDevelopment from '../../services/developmentMode.js';
const saveResponse = async (analyticsEnabled) => {
await xhrPost('/api/admin/generalSettings', {
analyticsEnabled
analyticsEnabled,
});
};
export default function TrackingModal() {
if (inDevelopment()) {
return null;
}
return <Modal
return (
<Modal
visible={true}
onOk={async () => {
await saveResponse(true);
@@ -32,17 +37,20 @@ export default function TrackingModal() {
<div className="trackingModal__description">
<p>Hey 👋</p>
<p>Fed up with popups? Yeah, me too. But this ones important, and I promise it will only appear once ;)</p>
<p>Fredy is completely free (and will always remain free). If youd like, you can support me by donating
through my GitHub, but theres absolutely no obligation to do so.</p>
<p>However, it would be a huge
help if youd allow me to collect some analytical data. Wait, before you click "no", let me explain. If
you
agree, Fredy will send a ping to my Mixpanel project each time it runs.</p>
<p>The data includes: names of
active adapters/providers, OS, architecture, Node version, and language. The information is entirely
anonymous and helps me understand which adapters/providers are most frequently used.</p>
<p>
Fredy is completely free (and will always remain free). If youd like, you can support me by donating through
my GitHub, but theres absolutely no obligation to do so.
</p>
<p>
However, it would be a huge help if youd allow me to collect some analytical data. Wait, before you click
"no", let me explain. If you agree, Fredy will send a ping to my Mixpanel project each time it runs.
</p>
<p>
The data includes: names of active adapters/providers, OS, architecture, Node version, and language. The
information is entirely anonymous and helps me understand which adapters/providers are most frequently used.
</p>
<p>Thanks🤘</p>
</div>
</Modal>;
</Modal>
);
}

View File

@@ -0,0 +1,4 @@
export default function isDevelopmentMode(){
const inDevMode= import.meta.env.MODE;
return inDevMode != null && inDevMode === 'development';
}

View File

@@ -8,7 +8,14 @@ import Headline from '../../components/headline/Headline';
import { xhrPost } from '../../services/xhr';
import { SegmentPart } from '../../components/segment/SegmentPart';
import { Banner, Toast } from '@douyinfe/semi-ui';
import {IconSave, IconCalendar, IconRefresh, IconSignal, IconLineChartStroked, IconSearch} from '@douyinfe/semi-icons';
import {
IconSave,
IconCalendar,
IconRefresh,
IconSignal,
IconLineChartStroked,
IconSearch,
} from '@douyinfe/semi-icons';
import './GeneralSettings.less';
function formatFromTimestamp(ts) {
@@ -97,7 +104,7 @@ const GeneralSettings = function GeneralSettings() {
to: workingHourTo,
},
demoMode,
analyticsEnabled
analyticsEnabled,
});
} catch (exception) {
console.error(exception);
@@ -175,24 +182,18 @@ const GeneralSettings = function GeneralSettings() {
</SegmentPart>
<Divider margin="1rem" />
<SegmentPart
name="Analytics"
helpText="Insights into the usage of Fredy."
Icon={IconLineChartStroked}
>
<SegmentPart name="Analytics" helpText="Insights into the usage of Fredy." Icon={IconLineChartStroked}>
<Banner
fullMode={false}
type="info"
closeIcon={null}
title={
<div style={{fontWeight: 600, fontSize: '14px', lineHeight: '20px'}}>
Explanation
</div>
}
title={<div style={{ fontWeight: 600, fontSize: '14px', lineHeight: '20px' }}>Explanation</div>}
style={{ marginBottom: '1rem' }}
description={
<div>
Analytics are disabled by default. If you choose to enable them, we will begin tracking the following:<br/>
Analytics are disabled by default. If you choose to enable them, we will begin tracking the
following:
<br />
<ul>
<li>Name of active provider (e.g. Immoscout)</li>
<li>Name of active adapter (e.g. Console)</li>
@@ -201,35 +202,26 @@ const GeneralSettings = function GeneralSettings() {
<li>node version</li>
<li>arch</li>
</ul>
The data is sent anonymously and helps me understand which providers or adapters are being used the most. In the end it helps me to improve fredy.
The data is sent anonymously and helps me understand which providers or adapters are being used the
most. In the end it helps me to improve fredy.
</div>
}
/>
<Checkbox
checked={analyticsEnabled}
onChange={(e) => setAnalyticsEnabled(e.target.checked)}
> Enabled
<Checkbox checked={analyticsEnabled} onChange={(e) => setAnalyticsEnabled(e.target.checked)}>
{' '}
Enabled
</Checkbox>
</SegmentPart>
<Divider margin="1rem" />
<SegmentPart
name="Demo Mode"
helpText="If enabled, Fredy runs in demo mode."
Icon={IconSearch}
>
<SegmentPart name="Demo Mode" helpText="If enabled, Fredy runs in demo mode." Icon={IconSearch}>
<Banner
fullMode={false}
type="info"
closeIcon={null}
title={
<div style={{fontWeight: 600, fontSize: '14px', lineHeight: '20px'}}>
Explanation
</div>
}
title={<div style={{ fontWeight: 600, fontSize: '14px', lineHeight: '20px' }}>Explanation</div>}
style={{ marginBottom: '1rem' }}
description={
<div>
@@ -239,12 +231,10 @@ const GeneralSettings = function GeneralSettings() {
}
/>
<Checkbox
checked={demoMode}
onChange={(e) => setDemoMode(e.target.checked)}
> Enabled
<Checkbox checked={demoMode} onChange={(e) => setDemoMode(e.target.checked)}>
{' '}
Enabled
</Checkbox>
</SegmentPart>
<Divider margin="1rem" />

View File

@@ -1,6 +1,6 @@
import React from 'react';
import { format } from '../../services/time/timeService';
import {Banner, Descriptions} from '@douyinfe/semi-ui';
import { Descriptions } from '@douyinfe/semi-ui';
export default function ProcessingTimes({ processingTimes = {} }) {
if (Object.keys(processingTimes).length === 0) {

View File

@@ -124,8 +124,8 @@ export default function JobMutator() {
<ProviderTable
providerData={providerData}
onRemove={(providerId) => {
setProviderData(providerData.filter((provider) => provider.id !== providerId));
onRemove={(providerUrl) => {
setProviderData(providerData.filter((provider) => provider.url !== providerUrl));
}}
/>
</SegmentPart>

View File

@@ -83,7 +83,7 @@ export default function NotificationAdapterMutator({
id: selectedAdapter.id,
name: selectedAdapter.name,
fields: selectedAdapter.fields || {},
})
}),
);
setSelectedAdapter(null);
@@ -114,7 +114,7 @@ export default function NotificationAdapterMutator({
setSuccessMessage('It seems like it worked! Please check your service.');
})
.catch((error) =>
setValidationMessage(`This did not work :-( I've received the following error: ${error.json.message}`)
setValidationMessage(`This did not work :-( I've received the following error: ${error.json.message}`),
);
};
@@ -229,7 +229,7 @@ export default function NotificationAdapterMutator({
.filter((option) =>
editNotificationAdapter != null
? true
: selected.find((selectedOption) => selectedOption.id === option.key) == null
: selected.find((selectedOption) => selectedOption.id === option.key) == null,
)
.sort(sortAdapter)}
onChange={(value) => {

View File

@@ -45,7 +45,7 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
url: providerUrl,
id: selectedProvider.id,
name: selectedProvider.name,
})
}),
);
setProviderUrl(null);
setSelectedProvider(null);
@@ -101,7 +101,7 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
description={
<div>
<p>
Currently, our Immoscout implementation does not drawing shapes on a map. Use a radius instead.
Currently, our Immoscout implementation does not support drawing shapes on a map. Use a radius instead.
</p>
</div>
}

View File

@@ -87,12 +87,15 @@ export default function Login() {
Login
</Button>
<br />
{demoMode && <Banner fullMode={true}
{demoMode && (
<Banner
fullMode={true}
type="info"
bordered
closeIcon={null}
description="This is the demo version of Fredy. Use 'demo' as both the username and password to log in."
/>}
/>
)}
</div>
</form>
</div>

View File

@@ -6,6 +6,7 @@ export default defineConfig({
build: {
chunkSizeWarningLimit: 9999999,
outDir: './ui/public',
emptyOutDir: true,
},
plugins: [react()],
server: {

838
yarn.lock

File diff suppressed because it is too large Load Diff