Compare commits

...

7 Commits

Author SHA1 Message Date
orangecoding
28eddc5d7f next release version 2025-09-20 19:49:32 +02:00
Iaroslav Postovalov
0ca9c5ae02 Add health check for Docker container (#179)
- Introduced `HealthCheck` in `docker-compose.yml` to monitor container status.
- Added a test step to validate container's health using Docker Compose in the GitHub workflow.
- Updated `Dockerfile` to include `curl` for health check commands.
2025-09-20 19:39:48 +02:00
orangecoding
a7d0037edd next release version 2025-09-20 19:37:47 +02:00
orangecoding
f339a2e2cf adding version banner to check if a new version of fredy is available 2025-09-20 19:37:27 +02:00
orangecoding
da8fd13973 fixing immoscout 2025-09-19 21:11:28 +02:00
orangecoding
7deffc64af next release version 2025-09-18 20:48:49 +02:00
orangecoding
d1dad7fd3b adding new unique index, adding button to start now 2025-09-18 20:48:25 +02:00
24 changed files with 251 additions and 62 deletions

View File

@@ -57,3 +57,41 @@ jobs:
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
# Test container health with docker compose
- name: Test container with docker compose
run: |
echo "Starting container with docker compose..."
docker compose up --build -d
echo "Waiting for container to be ready (60 seconds for start_period)..."
sleep 60
echo "Monitoring container health for 30 seconds..."
SECONDS_ELAPSED=0
HEALTH_CHECK_INTERVAL=5
TOTAL_DURATION=30
while [ $SECONDS_ELAPSED -lt $TOTAL_DURATION ]; do
HEALTH_STATUS=$(docker inspect --format='{{.State.Health.Status}}' fredy 2>/dev/null || echo "not_found")
CONTAINER_STATUS=$(docker inspect --format='{{.State.Status}}' fredy 2>/dev/null || echo "not_found")
echo "[$SECONDS_ELAPSED/$TOTAL_DURATION sec] Container: $CONTAINER_STATUS, Health: $HEALTH_STATUS"
# Check if container is not running or unhealthy
if [ "$CONTAINER_STATUS" != "running" ]; then
echo "Container stopped running! Status: $CONTAINER_STATUS"
docker compose logs fredy
exit 1
fi
if [ "$HEALTH_STATUS" = "unhealthy" ]; then
echo "Container is unhealthy!"
docker compose logs fredy
docker inspect --format='{{json .State.Health}}' fredy | jq
exit 1
fi
sleep $HEALTH_CHECK_INTERVAL
SECONDS_ELAPSED=$((SECONDS_ELAPSED + HEALTH_CHECK_INTERVAL))
done
docker compose down

View File

@@ -2,9 +2,10 @@ FROM node:22-slim
WORKDIR /fredy
# Install Chromium without extra recommended packages and clean apt cache
# Install Chromium and curl without extra recommended packages and clean apt cache
# curl is needed for the health check
RUN apt-get update \
&& apt-get install -y --no-install-recommends chromium \
&& apt-get install -y --no-install-recommends chromium curl \
&& rm -rf /var/lib/apt/lists/*
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \

View File

@@ -1 +1 @@
{"interval":"60","port":9998,"workingHours":{"from":"","to":""},"demoMode":false,"analyticsEnabled":null,"sqlitepath":"/db"}
{"interval":"60","port":9998,"workingHours":{"from":"","to":""},"demoMode":false,"analyticsEnabled":true,"sqlitepath":"/db"}

View File

@@ -11,5 +11,12 @@ services:
- ./conf:/conf
- ./db:/db
ports:
- 9998:9998
- "9998:9998"
restart: unless-stopped
healthcheck:
# The container will immediately stop when health check fails after retries
test: ["CMD-SHELL", "curl --fail --silent --show-error --max-time 5 http://localhost:9998/ || exit 1"]
interval: 120s
timeout: 10s
retries: 1
start_period: 10s

View File

@@ -10,6 +10,7 @@ import { ensureDemoUserExists, ensureAdminUserExists } from './lib/services/stor
import { cleanupDemoAtMidnight } from './lib/services/demoCleanup.js';
import { initTrackerCron } from './lib/services/tracking/Tracker-Cron.js';
import logger from './lib/services/logger.js';
import { bus } from './lib/services/events/event-bus.js';
// Ensure sqlite directory exists before loading anything else (based on config.sqlitepath)
const rawDir = config.sqlitepath || '/db';
@@ -45,29 +46,34 @@ ensureAdminUserExists();
ensureDemoUserExists();
await initTrackerCron();
setInterval(
(function exec() {
const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
if (!config.demoMode) {
if (isDuringWorkingHoursOrNotSet) {
config.lastRun = Date.now();
jobStorage
.getJobs()
.filter((job) => job.enabled)
.forEach((job) => {
job.provider
.filter((p) => fetchedProvider.find((fp) => fp.metaInformation.id === p.id) != null)
.forEach(async (prov) => {
const pro = fetchedProvider.find((fp) => fp.metaInformation.id === prov.id);
pro.init(prov, job.blacklist);
await new FredyRuntime(pro.config, job.notificationAdapter, prov.id, job.id, similarityCache).execute();
});
});
} else {
logger.debug('Working hours set. Skipping as outside of working hours.');
}
bus.on('jobs:runAll', () => {
logger.debug('Running Fredy Job manually');
execute();
});
const execute = () => {
const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
if (!config.demoMode) {
if (isDuringWorkingHoursOrNotSet) {
config.lastRun = Date.now();
jobStorage
.getJobs()
.filter((job) => job.enabled)
.forEach((job) => {
job.provider
.filter((p) => fetchedProvider.find((fp) => fp.metaInformation.id === p.id) != null)
.forEach(async (prov) => {
const pro = fetchedProvider.find((fp) => fp.metaInformation.id === prov.id);
pro.init(prov, job.blacklist);
await new FredyRuntime(pro.config, job.notificationAdapter, prov.id, job.id, similarityCache).execute();
});
});
} else {
logger.debug('Working hours set. Skipping as outside of working hours.');
}
return exec;
})(),
INTERVAL,
);
}
};
setInterval(execute, INTERVAL);
//start once at startup
execute();

View File

@@ -7,6 +7,7 @@ import { loginRouter } from './routes/loginRoute.js';
import { config } from '../utils.js';
import { userRouter } from './routes/userRoute.js';
import { jobRouter } from './routes/jobRouter.js';
import { versionRouter } from './routes/versionRouter.js';
import bodyParser from 'body-parser';
import restana from 'restana';
import files from 'serve-static';
@@ -23,6 +24,7 @@ service.use(cookieSession());
service.use(staticService);
service.use('/api/admin', authInterceptor());
service.use('/api/jobs', authInterceptor());
service.use('/api/version', authInterceptor());
// /admin can only be accessed when user is having admin permissions
service.use('/api/admin', adminInterceptor());
service.use('/api/jobs/notificationAdapter', notificationAdapterRouter);
@@ -30,6 +32,7 @@ service.use('/api/admin/generalSettings', generalSettingsRouter);
service.use('/api/jobs/provider', providerRouter);
service.use('/api/jobs/insights', analyticsRouter);
service.use('/api/admin/users', userRouter);
service.use('/api/version', versionRouter);
service.use('/api/jobs', jobRouter);
service.use('/api/login', loginRouter);
//this route is unsecured intentionally as it is being queried from the login page

View File

@@ -4,8 +4,11 @@ import * as userStorage from '../../services/storage/userStorage.js';
import { config } from '../../utils.js';
import { isAdmin } from '../security.js';
import logger from '../../services/logger.js';
import { bus } from '../../services/events/event-bus.js';
const service = restana();
const jobRouter = service.newRouter();
function doesJobBelongsToUser(job, req) {
const userId = req.session.currentUser;
if (userId == null) {
@@ -17,6 +20,7 @@ function doesJobBelongsToUser(job, req) {
}
return user.isAdmin || job.userId === user.id;
}
jobRouter.get('/', async (req, res) => {
const isUserAdmin = isAdmin(req);
//show only the jobs which belongs to the user (or all of the user is an admin)
@@ -30,6 +34,12 @@ jobRouter.get('/processingTimes', async (req, res) => {
};
res.send();
});
jobRouter.post('/startAll', async (req, res) => {
bus.emit('jobs:runAll');
res.send();
});
jobRouter.post('/', async (req, res) => {
const { provider, notificationAdapter, name, blacklist = [], jobId, enabled } = req.body;
try {

View File

@@ -0,0 +1,30 @@
import restana from 'restana';
import fetch from 'node-fetch';
import { getPackageVersion } from '../../utils.js';
const service = restana();
const versionRouter = service.newRouter();
versionRouter.get('/', async (req, res) => {
const versionPayload = await getCurrentVersionFromGithub();
res.body = versionPayload == null ? { newVersion: false } : versionPayload;
res.send();
});
async function getCurrentVersionFromGithub() {
const raw = await fetch('https://api.github.com/repos/orangecoding/fredy/releases/latest');
const data = await raw.json();
const localFredyVersion = await getPackageVersion();
if (localFredyVersion === data.tag_name) {
return null;
}
return {
newVersion: true,
version: data.tag_name,
url: data.html_url,
body: data.body,
localFredyVersion,
};
}
export { versionRouter };

View File

@@ -3,7 +3,7 @@
*
* The mobile API provides the following endpoints:
* - GET /search/total?{search parameters}: Returns the total number of listings for the given query
* Example: `curl -H "User-Agent: ImmoScout24_1410_30_._" https://api.mobile.immobilienscout24.de/search/total?searchType=region&realestatetype=apartmentrent&pricetype=calculatedtotalrent&geocodes=%2Fde%2Fberlin%2Fberlin `
* Example: `curl -H "User-Agent: ImmoScout_27.3_26.0_._" https://api.mobile.immobilienscout24.de/search/total?searchType=region&realestatetype=apartmentrent&pricetype=calculatedtotalrent&geocodes=%2Fde%2Fberlin%2Fberlin `
*
* - POST /search/list?{search parameters}: Actually retrieves the listings. Body is json encoded and contains
* data specifying additional results (advertisements) to return. The format is as follows:
@@ -15,12 +15,12 @@
* ```
* It is not necessary to provide data for the specified keys.
*
* Example: `curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calculatedtotalrent&realestatetype=apartmentrent&searchType=region&geocodes=%2Fde%2Fberlin%2Fberlin&pagenumber=1' -H "Connection: keep-alive" -H "User-Agent: ImmoScout24_1410_30_._" -H "Accept: application/json" -H "Content-Type: application/json" -d '{"supportedResultListType": [], "userData": {}}'`
* Example: `curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calculatedtotalrent&realestatetype=apartmentrent&searchType=region&geocodes=%2Fde%2Fberlin%2Fberlin&pagenumber=1' -H "Connection: keep-alive" -H "User-Agent: ImmoScout_27.3_26.0_._" -H "Accept: application/json" -H "Content-Type: application/json" -d '{"supportedResultListType": [], "userData": {}}'`
* - GET /expose/{id} - Returns the details of a listing. The response contains additional details not included in the
* listing response.
*
* Example: `curl -H "User-Agent: ImmoScout24_1410_30_._" "https://api.mobile.immobilienscout24.de/expose/158382494"`
* Example: `curl -H "User-Agent: ImmoScout_27.3_26.0_._" "https://api.mobile.immobilienscout24.de/expose/158382494"`
*
*
* It is necessary to set the correct User Agent (see `getListings`) in the request header.
@@ -44,7 +44,7 @@ async function getListings(url) {
const response = await fetch(url, {
method: 'POST',
headers: {
'User-Agent': 'ImmoScout24_1410_30_._',
'User-Agent': 'ImmoScout_27.3_26.0_._',
'Content-Type': 'application/json',
},
body: JSON.stringify({

View File

@@ -0,0 +1,2 @@
import { EventEmitter } from 'node:events';
export const bus = new EventEmitter();

View File

@@ -21,7 +21,7 @@ export function parse(crawlContainer, crawlFields, text, url) {
const result = [];
if ($(crawlContainer).length === 0) {
logger.warn('No elements in crawl container found for url ', url);
logger.debug('No elements in crawl container found for url ', url);
return null;
}

View File

@@ -85,11 +85,11 @@ export const storeListings = (jobId, providerId, listings) => {
SqliteConnection.withTransaction((db) => {
const stmt = db.prepare(
`INSERT INTO listings (id, hash, provider, job_id, price, size, title, image_url, description, address, city,
`INSERT INTO listings (id, hash, provider, job_id, price, size, title, image_url, description, address,
link, created_at)
VALUES (@id, @hash, @provider, @job_id, @price, @size, @title, @image_url, @description, @address, @city, @link,
VALUES (@id, @hash, @provider, @job_id, @price, @size, @title, @image_url, @description, @address, @link,
@created_at)
ON CONFLICT(hash) DO NOTHING`,
ON CONFLICT(job_id, hash) DO NOTHING`,
);
for (const item of listings) {

View File

@@ -0,0 +1,10 @@
// Migration: there needs to be a unique index on job_id and hash as only
// this makes the listing indeed unique
export function up(db) {
db.exec(`
DROP INDEX IF EXISTS idx_listings_hash;
CREATE UNIQUE INDEX IF NOT EXISTS idx_listings_job_hash
ON listings (job_id, hash);
`);
}

View File

@@ -1,9 +1,7 @@
import { getJobs } from '../storage/jobStorage.js';
import { getUniqueId } from './uniqueId.js';
import { config, inDevMode } from '../../utils.js';
import { config, getPackageVersion, inDevMode } from '../../utils.js';
import os from 'os';
import { readFileSync } from 'fs';
import { packageUp } from 'package-up';
import fetch from 'node-fetch';
import logger from '../logger.js';
@@ -77,15 +75,3 @@ function enrichTrackingObject(trackingObject) {
version,
};
}
async function getPackageVersion() {
try {
const packagePath = await packageUp();
const packageJson = readFileSync(packagePath, 'utf8');
const json = JSON.parse(packageJson);
return json.version;
} catch (error) {
logger.error('Error reading version from package.json', error);
}
return 'N/A';
}

View File

@@ -3,8 +3,9 @@ import { fileURLToPath } from 'node:url';
import { readFile } from 'fs/promises';
import { createHash } from 'crypto';
import { DEFAULT_CONFIG } from './defaultConfig.js';
import fs from 'fs';
import fs, { readFileSync } from 'fs';
import logger from './services/logger.js';
import { packageUp } from 'package-up';
const RE_GT = />/g;
const RE_WEBP = /\/format\/webp/gi;
@@ -196,6 +197,22 @@ const normalizeImageUrl = (url) => {
return u;
};
/**
* returns Fredy's version
* @returns {Promise<*|string>}
*/
async function getPackageVersion() {
try {
const packagePath = await packageUp();
const packageJson = readFileSync(packagePath, 'utf8');
const json = JSON.parse(packageJson);
return json.version;
} catch (error) {
logger.error('Error reading version from package.json', error);
}
return 'N/A';
}
await refreshConfig();
export { isOneOf };
@@ -206,6 +223,7 @@ export { duringWorkingHoursOrNotSet };
export { getDirName };
export { config };
export { buildHash };
export { getPackageVersion };
export default {
isOneOf,
nullOrEmpty,

View File

@@ -1,6 +1,6 @@
{
"name": "fredy",
"version": "12.1.0",
"version": "12.1.4",
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
"scripts": {
"prepare": "husky",
@@ -56,6 +56,7 @@
"Firefox ESR"
],
"dependencies": {
"@douyinfe/semi-icons": "^2.86.0",
"@douyinfe/semi-ui": "2.86.0",
"@sendgrid/mail": "8.1.5",
"@visactor/react-vchart": "^2.0.4",

View File

@@ -41,7 +41,7 @@ Challenges:
_Returns the total number of listings for the given query._
```
curl -H "User-Agent: ImmoScout24_1410_30_._" \
curl -H "User-Agent: ImmoScout_27.3_26.0_._" \
-H "Accept: application/json" \
"https://api.mobile.immobilienscout24.de/search/total?searchType=region&realestatetype=apartmentrent&pricetype=calculatedtotalrent&geocodes=%2Fde%2Fberlin%2Fberlin"
```
@@ -63,7 +63,7 @@ _The body is json encoded and contains data specifying additional results (adver
```
curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calculatedtotalrent&realestatetype=apartmentrent&searchType=region&geocodes=%2Fde%2Fberlin%2Fberlin&pagenumber=1' \
-H "Connection: keep-alive" \
-H "User-Agent: ImmoScout24_1410_30_._" \
-H "User-Agent: ImmoScout_27.3_26.0_._" \
-H "Accept: application/json" \
-H "Content-Type: application/json" \
-d '{"supportedResultListType":[],"userData":{}}'
@@ -78,7 +78,7 @@ curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calc
The response contains additional details not included in the listing response.
```
curl -H "User-Agent: ImmoScout24_1410_30_._" \
curl -H "User-Agent: ImmoScout_27.3_26.0_._" \
-H "Accept: application/json" \
"https://api.mobile.immobilienscout24.de/expose/158382494"
```

View File

@@ -53,7 +53,7 @@ describe('#immoscout-mobile URL conversion', () => {
const response = await fetch(url, {
method: 'POST',
headers: {
'User-Agent': 'ImmoScout24_1410_30_._',
'User-Agent': 'ImmoScout_27.3_26.0_._',
'Content-Type': 'application/json',
},
body: JSON.stringify({

View File

@@ -18,11 +18,13 @@ import Jobs from './views/jobs/Jobs';
import './App.less';
import TrackingModal from './components/tracking/TrackingModal.jsx';
import { Banner } from '@douyinfe/semi-ui';
import VersionBanner from './components/version/VersionBanner.jsx';
export default function FredyApp() {
const actions = useActions();
const [loading, setLoading] = React.useState(true);
const currentUser = useSelector((state) => state.user.currentUser);
const versionUpdate = useSelector((state) => state.versionUpdate.versionUpdate);
const settings = useSelector((state) => state.generalSettings.settings);
useEffect(() => {
@@ -34,6 +36,7 @@ export default function FredyApp() {
await actions.jobs.getProcessingTimes();
await actions.notificationAdapter.getAdapter();
await actions.generalSettings.getGeneralSettings();
await actions.versionUpdate.getVersionUpdate();
}
setLoading(false);
}
@@ -53,7 +56,6 @@ export default function FredyApp() {
<Route path="*" element={<Navigate to="/login" replace />} />
</Routes>
);
return loading ? null : needsLogin() ? (
login()
) : (
@@ -62,7 +64,7 @@ export default function FredyApp() {
<Logout />
<Logo width={190} white />
<Menu isAdmin={isAdmin()} />
{versionUpdate?.newVersion && <VersionBanner />}
{settings.demoMode && (
<>
<Banner

View File

@@ -0,0 +1,43 @@
import React from 'react';
import { Banner, Descriptions } from '@douyinfe/semi-ui';
import { useSelector } from '../../services/state/store.js';
import './VersionBanner.less';
export default function VersionBanner() {
const versionUpdate = useSelector((state) => state.versionUpdate.versionUpdate);
return (
<Banner
className="versionBanner"
type="success"
icon={null}
description={
<div>
<p>A new version of Fredy is available. Update now to take advantage of the latest features and bug fixes.</p>
<Descriptions row size="small">
<Descriptions.Item itemKey="Your Version">{versionUpdate.localFredyVersion}</Descriptions.Item>
<Descriptions.Item itemKey="Latest Version">{versionUpdate.version}</Descriptions.Item>
<Descriptions.Item itemKey="Github Release">
<a href={versionUpdate.url} target="_blank" rel="noreferrer">
{versionUpdate.url}
</a>{' '}
</Descriptions.Item>
</Descriptions>
<p>
<b>
<small>Release Notes</small>
</b>
</p>
<pre>{stripFullChangelog(versionUpdate.body)}</pre>
</div>
}
/>
);
function stripFullChangelog(text) {
if (text == null) {
return '';
}
return text.replace(/(?:\r?\n)\*\*Full Changelog\*\*[\s\S]*$/u, '');
}
}

View File

@@ -0,0 +1,3 @@
.versionBanner {
margin-bottom: 1rem;
}

View File

@@ -118,6 +118,18 @@ export const useFredyState = create(
}
},
},
versionUpdate: {
async getVersionUpdate() {
try {
const response = await xhrGet('/api/version');
set((state) => ({
versionUpdate: { ...state.versionUpdate, versionUpdate: response.json },
}));
} catch (Exception) {
console.error('Error while trying to get resource for api/version. Error:', Exception);
}
},
},
};
// Initial state
@@ -125,6 +137,7 @@ export const useFredyState = create(
notificationAdapter: [],
generalSettings: { settings: {} },
demoMode: { demoMode: false },
versionUpdate: {},
provider: [],
jobs: { jobs: [], insights: {}, processingTimes: {} },
user: { users: [], currentUser: null },
@@ -135,6 +148,7 @@ export const useFredyState = create(
notificationAdapter: { ...effects.notificationAdapter },
generalSettings: { ...effects.generalSettings },
demoMode: { ...effects.demoMode },
versionUpdate: { ...effects.versionUpdate },
provider: { ...effects.provider },
jobs: { ...effects.jobs },
user: { ...effects.user },

View File

@@ -1,6 +1,8 @@
import React from 'react';
import { format } from '../../services/time/timeService';
import { Descriptions } from '@douyinfe/semi-ui';
import { Button, Descriptions, Toast } from '@douyinfe/semi-ui';
import { IconPlayCircle } from '@douyinfe/semi-icons';
import { xhrPost } from '../../services/xhr.js';
export default function ProcessingTimes({ processingTimes = {} }) {
if (Object.keys(processingTimes).length === 0) {
@@ -24,6 +26,19 @@ export default function ProcessingTimes({ processingTimes = {} }) {
<Descriptions.Item itemKey="Next run">
{format(processingTimes.lastRun + processingTimes.interval * 60000)}
</Descriptions.Item>
<Descriptions.Item itemKey="Find Listings now">
<Button
size="small"
icon={<IconPlayCircle />}
aria-label="Start now"
onClick={async () => {
await xhrPost('/api/jobs/startAll', null);
Toast.success('Successfully triggered Fredy search.');
}}
>
Search now
</Button>
</Descriptions.Item>
</>
)}
</Descriptions>

View File

@@ -973,7 +973,7 @@
remark-gfm "^4.0.0"
scroll-into-view-if-needed "^2.2.24"
"@douyinfe/semi-icons@2.86.0":
"@douyinfe/semi-icons@2.86.0", "@douyinfe/semi-icons@^2.86.0":
version "2.86.0"
resolved "https://registry.yarnpkg.com/@douyinfe/semi-icons/-/semi-icons-2.86.0.tgz#ee4355c81616ea4325627a3bb607ed9f9b9afac3"
integrity sha512-KEDlYYP1wdOqN28Ck0YcdCx7mSks8SRY4w4KKbXPaROzYNEyT2BRcJxwysMHfxL2IDfsroHrRPJsX9pnrmQqTg==