mirror of
https://github.com/orangecoding/fredy.git
synced 2026-06-16 12:31:07 +00:00
Compare commits
53 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
362166651d | ||
|
|
a020117a78 | ||
|
|
9207280ab4 | ||
|
|
94384df36d | ||
|
|
730cc52187 | ||
|
|
e82db5b6db | ||
|
|
2f8c021819 | ||
|
|
72c2c02e49 | ||
|
|
48c0360111 | ||
|
|
63c947896e | ||
|
|
2a814b6bb6 | ||
|
|
3249881771 | ||
|
|
3b727ea708 | ||
|
|
a2a765f43d | ||
|
|
c17a815263 | ||
|
|
7a2dacaa61 | ||
|
|
359e00e69f | ||
|
|
bc9c56a224 | ||
|
|
6bef907416 | ||
|
|
6c7d655277 | ||
|
|
c132e64437 | ||
|
|
1dcb852ea1 | ||
|
|
019b9ac87b | ||
|
|
0d23d43e79 | ||
|
|
324afee483 | ||
|
|
e95ebb9624 | ||
|
|
c29387c85d | ||
|
|
322ae199b0 | ||
|
|
b3300169fa | ||
|
|
9296bcdc86 | ||
|
|
44edf47393 | ||
|
|
bbebc2a1a2 | ||
|
|
d2978c14db | ||
|
|
5ceac25aa6 | ||
|
|
34b68e1f52 | ||
|
|
696ae451d3 | ||
|
|
317ef79336 | ||
|
|
6428e7ad78 | ||
|
|
2bcec04d55 | ||
|
|
ee2112a24d | ||
|
|
5a54448288 | ||
|
|
f1b8709ab7 | ||
|
|
b56e13aa16 | ||
|
|
a834abc31c | ||
|
|
573868eccb | ||
|
|
1a210d7c1c | ||
|
|
996b841cfb | ||
|
|
b2e294e38c | ||
|
|
8afeaa05d9 | ||
|
|
ec47137b89 | ||
|
|
33161de087 | ||
|
|
acab23207e | ||
|
|
2896d531e4 |
1
.github/FUNDING.yml
vendored
1
.github/FUNDING.yml
vendored
@@ -1,3 +1,4 @@
|
|||||||
# These are supported funding model platforms
|
# These are supported funding model platforms
|
||||||
|
|
||||||
github: [orangecoding]
|
github: [orangecoding]
|
||||||
|
ko_fi: orangecoding
|
||||||
|
|||||||
47
.github/ISSUE_TEMPLATE/bug.yml
vendored
47
.github/ISSUE_TEMPLATE/bug.yml
vendored
@@ -5,6 +5,40 @@ labels: [bug]
|
|||||||
assignees: []
|
assignees: []
|
||||||
|
|
||||||
body:
|
body:
|
||||||
|
- type: markdown
|
||||||
|
attributes:
|
||||||
|
value: |
|
||||||
|
## Please attach a debug bundle (available since Fredy 22.5.0+)
|
||||||
|
|
||||||
|
Since **Fredy 22.5.0** you can export a debug bundle that contains a system
|
||||||
|
snapshot (`sys.txt`, Fredy version, Node.js version, OS, Docker detection,
|
||||||
|
CPU, memory, sanitized settings) and the full log buffer (`logs.txt`) that
|
||||||
|
Fredy recorded while you reproduced the issue. Attaching it dramatically
|
||||||
|
speeds up triage.
|
||||||
|
Oh and before you ask: I decided against simply putting all logs into the debug
|
||||||
|
due to privacy reasons :)
|
||||||
|
|
||||||
|
**The bundle is only useful when the error is actually inside `logs.txt`.**
|
||||||
|
That means you have to record first, reproduce after:
|
||||||
|
|
||||||
|
1. Log in to Fredy as **admin** and open **Settings → Debug**.
|
||||||
|
2. Click **"Enable debug logging" / "Debug-Logging aktivieren"**. A red banner
|
||||||
|
appears across the whole app while recording is on.
|
||||||
|
3. **Now reproduce the bug.** Trigger the broken job, click the failing
|
||||||
|
button, wait for the failing scrape — whatever it was.
|
||||||
|
4. Come back to **Settings → Debug** and confirm the progress bar moved
|
||||||
|
(i.e. log entries were actually written). If it stayed at 0%, nothing was
|
||||||
|
captured and the bundle won't help us.
|
||||||
|
5. Click **"Download debug information" / "Debug Informationen herunterladen"**
|
||||||
|
and drop the resulting `FredyDebug-*.zip` into the "Screenshots / Logs"
|
||||||
|
field below.
|
||||||
|
6. Optional but recommended: click **"Disable debug logging"** to stop the
|
||||||
|
recording, and **"Delete stored debug logs"** once you have the zip so the
|
||||||
|
database does not keep them around.
|
||||||
|
|
||||||
|
On Fredy versions older than 22.5.0, paste the relevant log lines from your
|
||||||
|
console / Docker / systemd journal manually instead.
|
||||||
|
|
||||||
- type: textarea
|
- type: textarea
|
||||||
id: description
|
id: description
|
||||||
attributes:
|
attributes:
|
||||||
@@ -49,8 +83,11 @@ body:
|
|||||||
id: screenshots
|
id: screenshots
|
||||||
attributes:
|
attributes:
|
||||||
label: Screenshots / Logs
|
label: Screenshots / Logs
|
||||||
description: Add screenshots or paste log output to help explain the problem.
|
description: |
|
||||||
placeholder: "Drag and drop screenshots here, or paste logs."
|
Drop the FredyDebug-*.zip here (see the instructions at the top, available
|
||||||
|
since Fredy 22.5.0) and/or any additional screenshots. If you cannot produce
|
||||||
|
the bundle, paste relevant log lines instead.
|
||||||
|
placeholder: "Drag and drop the FredyDebug-*.zip and any screenshots here."
|
||||||
validations:
|
validations:
|
||||||
required: false
|
required: false
|
||||||
|
|
||||||
@@ -58,8 +95,10 @@ body:
|
|||||||
id: environment
|
id: environment
|
||||||
attributes:
|
attributes:
|
||||||
label: Environment
|
label: Environment
|
||||||
description: Provide details about your environment.
|
description: |
|
||||||
placeholder: "OS: macOS 15, Browser: Chrome 124, App version: 1.2.3"
|
Provide details about your environment. You can copy most of this from
|
||||||
|
sys.txt inside the debug bundle.
|
||||||
|
placeholder: "OS: macOS 15, Browser: Chrome 124, App version: 22.5.0, Docker: yes"
|
||||||
validations:
|
validations:
|
||||||
required: true
|
required: true
|
||||||
|
|
||||||
|
|||||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -7,3 +7,4 @@ npm-debug.log
|
|||||||
.idea
|
.idea
|
||||||
.vscode
|
.vscode
|
||||||
tools/release/config.json
|
tools/release/config.json
|
||||||
|
.agents
|
||||||
@@ -46,7 +46,7 @@ index.js (startup)
|
|||||||
├── runMigrations()
|
├── runMigrations()
|
||||||
├── getProviders() # lazily imports lib/provider/*.js
|
├── getProviders() # lazily imports lib/provider/*.js
|
||||||
├── similarityCache.init() # preloads hash cache from DB
|
├── similarityCache.init() # preloads hash cache from DB
|
||||||
├── api.js # starts restana HTTP server
|
├── api.js # starts fastify HTTP server
|
||||||
└── initJobExecutionService() # registers event-bus listeners + starts scheduler
|
└── initJobExecutionService() # registers event-bus listeners + starts scheduler
|
||||||
|
|
||||||
scheduler (every N minutes) or manual trigger via POST /api/jobs/:id/run
|
scheduler (every N minutes) or manual trigger via POST /api/jobs/:id/run
|
||||||
|
|||||||
118
README.md
118
README.md
@@ -23,7 +23,7 @@
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
# Fredy 🏡 – Your Self-Hosted Real Estate Finder for Germany
|
# Fredy 🏡 - Your Self-Hosted Real Estate Finder for Germany
|
||||||
|
|
||||||
Finding an apartment or house in Germany can be stressful and
|
Finding an apartment or house in Germany can be stressful and
|
||||||
time-consuming.\
|
time-consuming.\
|
||||||
@@ -55,8 +55,11 @@ same listing twice.
|
|||||||
|
|
||||||
## 🤝 Sponsorship [](https://github.com/sponsors/orangecoding)
|
## 🤝 Sponsorship [](https://github.com/sponsors/orangecoding)
|
||||||
|
|
||||||
I maintain Fredy and other open-source projects in my free time.\
|
I maintain Fredy and other open-source projects in my free time, if you find it useful, consider supporting the project ❤️
|
||||||
If you find it useful, consider supporting the project 💙
|
|
||||||
|
#### Support me on
|
||||||
|
[Ko-Fi](https://ko-fi.com/orangecoding) | [Github](https://github.com/sponsors/orangecoding)
|
||||||
|
----
|
||||||
|
|
||||||
Fredy is proudly backed by the **JetBrains Open Source Support Program**.
|
Fredy is proudly backed by the **JetBrains Open Source Support Program**.
|
||||||
|
|
||||||
@@ -167,6 +170,40 @@ For more information on how to set it up and use it, please refer to the [MCP Re
|
|||||||
|
|
||||||
Immoscout has implemented advanced bot detection. In order to work around this, we are using a reversed engineered version of their mobile api. See [Immoscout Reverse Engineering Documentation](https://github.com/orangecoding/fredy/blob/master/reverse-engineered-immoscout.md)
|
Immoscout has implemented advanced bot detection. In order to work around this, we are using a reversed engineered version of their mobile api. See [Immoscout Reverse Engineering Documentation](https://github.com/orangecoding/fredy/blob/master/reverse-engineered-immoscout.md)
|
||||||
|
|
||||||
|
## 🛡️ Bot Detection & Proxies
|
||||||
|
|
||||||
|
Most browser-based providers (immowelt, immonet, kleinanzeigen, ...) are scraped through a hardened headless browser ([CloakBrowser](https://www.npmjs.com/package/cloakbrowser)). It makes the **browser fingerprint** indistinguishable from a real Chrome, which is enough when you run Fredy on a normal home connection.
|
||||||
|
|
||||||
|
On a **server / VPS the requests usually originate from a datacenter IP**, and providers behind anti-bot systems (e.g. AWS CloudFront/WAF) block those based on **IP reputation alone**, no matter how perfect the fingerprint is. The typical symptom: it works locally but you get `We have been detected as a bot :-/` on the server.
|
||||||
|
|
||||||
|
### The fix: a residential proxy
|
||||||
|
|
||||||
|
A **residential proxy** routes Fredy's browser through the internet connection of a real household, so the provider sees a "normal user" IP instead of a datacenter. For German portals, use a **German (DE) residential** (or mobile/4G) proxy. Plain VPNs and **datacenter proxies do not help** here, they share the same bad reputation as your server.
|
||||||
|
|
||||||
|
**Configure it** under **Settings → Execution → Proxy URL**. Supported formats:
|
||||||
|
|
||||||
|
```
|
||||||
|
http://user:pass@host:port
|
||||||
|
socks5://user:pass@host:port
|
||||||
|
```
|
||||||
|
|
||||||
|
Leave the field empty to disable. The proxy applies to all headless-browser providers and takes effect on the next job run (no restart needed). Immoscout uses a separate mobile API and is not affected.
|
||||||
|
|
||||||
|
### Where to get a residential proxy
|
||||||
|
|
||||||
|
Residential proxies are a paid service (usually billed per GB, Fredy's traffic is small). Well-known providers offering German residential IPs include:
|
||||||
|
|
||||||
|
| Provider | Notes |
|
||||||
|
|---|---|
|
||||||
|
| [IPRoyal](https://iproyal.com) | Pay-as-you-go, no monthly minimum, good for low volume |
|
||||||
|
| [Webshare](https://www.webshare.io) | Cheap entry tier, has a small free plan to test with |
|
||||||
|
| [Decodo (formerly Smartproxy)](https://decodo.com) | Easy setup, country/city targeting |
|
||||||
|
| [SOAX](https://soax.com) | Residential + mobile, fine-grained geo-targeting |
|
||||||
|
| [Bright Data](https://brightdata.com) | Largest pool, most features, higher complexity/price |
|
||||||
|
| [Oxylabs](https://oxylabs.io) | Enterprise-grade, larger plans |
|
||||||
|
|
||||||
|
This is not an endorsement, pick whatever fits your budget. For low-volume use like Fredy, a pay-as-you-go plan (e.g. IPRoyal) or a cheap entry tier (e.g. Webshare) is usually plenty. Make sure to select **Germany** as the proxy location and keep the search interval reasonable (the higher the interval, the less you look like a bot).
|
||||||
|
|
||||||
## Analytics
|
## Analytics
|
||||||
|
|
||||||
Fredy is completely free (and will always remain free). However, it would be a huge help if you’d allow me to collect some analytical data.
|
Fredy is completely free (and will always remain free). However, it would be a huge help if you’d allow me to collect some analytical data.
|
||||||
@@ -176,6 +213,50 @@ The data includes: names of active adapters/providers, OS, architecture, Node ve
|
|||||||
|
|
||||||
**Thanks**🤘
|
**Thanks**🤘
|
||||||
|
|
||||||
|
## 🐞 Debug Information
|
||||||
|
|
||||||
|
Since Fredy **22.5.0** there is a built-in way to capture everything Fredy logs into the
|
||||||
|
database for a limited time and download it as a single zip file. This is the recommended
|
||||||
|
way to attach diagnostics to a bug report. I decided against simply putting all logs into
|
||||||
|
a debug bundle due to privacy reasons!
|
||||||
|
|
||||||
|
**How it works**
|
||||||
|
|
||||||
|
- Debug logging is **opt-in** and admin-only. As long as it is off, Fredy behaves exactly
|
||||||
|
as before (console output only, nothing in the DB).
|
||||||
|
- When you turn it on, every log line (`debug`, `info`, `warn`, `error`) is additionally
|
||||||
|
written into the `debug_logs` SQLite table. The console keeps logging at its usual level.
|
||||||
|
- The recorded data is hard-capped at **5 MiB** via a rolling buffer: once the cap is hit,
|
||||||
|
the oldest entries are dropped automatically so the newest ones always survive.
|
||||||
|
- The on/off flag is persisted, so debug logging stays on across restarts (and you'll see
|
||||||
|
the warning banner everywhere until you turn it off again).
|
||||||
|
|
||||||
|
**Capturing a debug bundle**
|
||||||
|
|
||||||
|
1. Open Fredy as an **admin** and go to **Settings → Debug**.
|
||||||
|
2. Click **"Enable debug logging" / "Debug-Logging aktivieren"**. A red banner appears on
|
||||||
|
every page while recording is on.
|
||||||
|
3. **Reproduce the bug**.
|
||||||
|
4. Come back to **Settings → Debug** and check the progress bar, if it stayed at 0 %,
|
||||||
|
nothing was captured.
|
||||||
|
5. Click **"Download debug information" / "Debug Informationen herunterladen"**. You get a
|
||||||
|
zip named `YYYY-MM-DD-FredyDebug-<version>.zip` containing two files:
|
||||||
|
- `logs.txt` - every log line captured while recording was on, prefixed with timestamp
|
||||||
|
and level.
|
||||||
|
- `sys.txt` - runtime snapshot (Fredy version, Node.js version, OS, Docker detection,
|
||||||
|
CPU, memory, sanitized settings). Proxy credentials and session secrets are
|
||||||
|
**stripped** before export.
|
||||||
|
6. Attach the zip to the bug report.
|
||||||
|
7. Optional but recommended: click **"Disable debug logging"** to stop recording, and
|
||||||
|
**"Delete stored debug logs"** once you've sent the zip so the DB does not keep them
|
||||||
|
around.
|
||||||
|
|
||||||
|
**What is _not_ included**
|
||||||
|
|
||||||
|
- passwords/privacy relevant things
|
||||||
|
- Anything that Fredy itself does not pass through its `logger`. If a third-party library
|
||||||
|
writes directly to `process.stderr`, that output stays on the console only.
|
||||||
|
|
||||||
## 🛠️ Development
|
## 🛠️ Development
|
||||||
|
|
||||||
### Development Mode
|
### Development Mode
|
||||||
@@ -206,6 +287,37 @@ If you have to refresh the fixtures (every once in a while needed because the pr
|
|||||||
yarn run download-fixtures
|
yarn run download-fixtures
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Adding a new language
|
||||||
|
|
||||||
|
Fredy's UI is fully multilingual. Translation files live in `ui/src/locales/`. To add a new language, create a single JSON file there, no code changes required.
|
||||||
|
|
||||||
|
**Example: `ui/src/locales/fr.json`**
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"_meta": {
|
||||||
|
"flag": "🇫🇷",
|
||||||
|
"name": "Français",
|
||||||
|
"locale": "fr-FR",
|
||||||
|
"semiLocale": "fr"
|
||||||
|
},
|
||||||
|
"nav.dashboard": "Tableau de bord",
|
||||||
|
"common.save": "Enregistrer",
|
||||||
|
...
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The `_meta` fields:
|
||||||
|
|
||||||
|
| Field | Description |
|
||||||
|
|---|---|
|
||||||
|
| `flag` | Unicode flag emoji shown in the language selector |
|
||||||
|
| `name` | Display name shown in the language selector |
|
||||||
|
| `locale` | BCP 47 locale string used for date and number formatting (e.g. `fr-FR`) |
|
||||||
|
| `semiLocale` | Semi UI locale key for component-level strings (date pickers, pagination, etc.) |
|
||||||
|
|
||||||
|
> **Important:** `semiLocale` must exactly match a locale filename from the Semi UI locale sources (without the `.js` extension). See the [available Semi UI locales on GitHub](https://github.com/DouyinFE/semi-design/tree/main/packages/semi-ui/locale/source) for the full list of supported keys.
|
||||||
|
|
||||||
|
After adding the file, rebuild the frontend (`yarn build:frontend` or restart the dev server) and the new language will appear automatically in **Settings → User Settings → Language**.
|
||||||
|
|
||||||
------------------------------------------------------------------------
|
------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|||||||
@@ -11,6 +11,8 @@
|
|||||||
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
|
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
|
||||||
|
|
||||||
<title>Fredy || Real Estate Finder</title>
|
<title>Fredy || Real Estate Finder</title>
|
||||||
|
<link rel="icon" type="image/png" href="/ui/src/assets/heart.png" />
|
||||||
|
<link rel="apple-touch-icon" href="/ui/src/assets/heart.png" />
|
||||||
<link rel="preconnect" href="https://fonts.googleapis.com" />
|
<link rel="preconnect" href="https://fonts.googleapis.com" />
|
||||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
||||||
<link href="https://fonts.googleapis.com/css2?family=Outfit:wght@300;400;500;600;700&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet" />
|
<link href="https://fonts.googleapis.com/css2?family=Outfit:wght@300;400;500;600;700&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet" />
|
||||||
|
|||||||
7
index.js
7
index.js
@@ -10,6 +10,7 @@ import { runMigrations } from './lib/services/storage/migrations/migrate.js';
|
|||||||
import { ensureDemoUserExists, ensureAdminUserExists } from './lib/services/storage/userStorage.js';
|
import { ensureDemoUserExists, ensureAdminUserExists } from './lib/services/storage/userStorage.js';
|
||||||
import { initTrackerCron } from './lib/services/crons/tracker-cron.js';
|
import { initTrackerCron } from './lib/services/crons/tracker-cron.js';
|
||||||
import logger from './lib/services/logger.js';
|
import logger from './lib/services/logger.js';
|
||||||
|
import { reloadEnabledFromSettings } from './lib/services/debug/debugLogStorage.js';
|
||||||
import { initActiveCheckerCron } from './lib/services/crons/listing-alive-cron.js';
|
import { initActiveCheckerCron } from './lib/services/crons/listing-alive-cron.js';
|
||||||
import { initGeocodingCron } from './lib/services/crons/geocoding-cron.js';
|
import { initGeocodingCron } from './lib/services/crons/geocoding-cron.js';
|
||||||
import { getSettings } from './lib/services/storage/settingsStorage.js';
|
import { getSettings } from './lib/services/storage/settingsStorage.js';
|
||||||
@@ -42,6 +43,12 @@ await runMigrations();
|
|||||||
|
|
||||||
const settings = await getSettings();
|
const settings = await getSettings();
|
||||||
|
|
||||||
|
// Restore the persisted on/off flag for opt-in DB log capture so it survives a
|
||||||
|
// Fredy restart. reloadEnabledFromSettings() also (un)wires the logger sink based
|
||||||
|
// on the restored flag, so the logger hot path stays cost-free when nobody enabled
|
||||||
|
// the feature.
|
||||||
|
await reloadEnabledFromSettings();
|
||||||
|
|
||||||
// Ensure the sqlite directory exists before loading anything else (based on config.sqlitepath)
|
// Ensure the sqlite directory exists before loading anything else (based on config.sqlitepath)
|
||||||
const { dir: sqliteDir } = await computeDbPath();
|
const { dir: sqliteDir } = await computeDbPath();
|
||||||
if (!fs.existsSync(sqliteDir)) {
|
if (!fs.existsSync(sqliteDir)) {
|
||||||
|
|||||||
@@ -5,9 +5,10 @@
|
|||||||
|
|
||||||
import { NoNewListingsWarning } from './errors.js';
|
import { NoNewListingsWarning } from './errors.js';
|
||||||
import {
|
import {
|
||||||
storeListings,
|
|
||||||
getKnownListingHashesForJobAndProvider,
|
|
||||||
deleteListingsById,
|
deleteListingsById,
|
||||||
|
getKnownListingHashesForJobAndProvider,
|
||||||
|
storeListings,
|
||||||
|
updateListingDistance,
|
||||||
} from './services/storage/listingsStorage.js';
|
} from './services/storage/listingsStorage.js';
|
||||||
import { getJob } from './services/storage/jobStorage.js';
|
import { getJob } from './services/storage/jobStorage.js';
|
||||||
import * as notify from './notification/notify.js';
|
import * as notify from './notification/notify.js';
|
||||||
@@ -16,8 +17,7 @@ import urlModifier from './services/queryStringMutator.js';
|
|||||||
import logger from './services/logger.js';
|
import logger from './services/logger.js';
|
||||||
import { geocodeAddress } from './services/geocoding/geoCodingService.js';
|
import { geocodeAddress } from './services/geocoding/geoCodingService.js';
|
||||||
import { distanceMeters } from './services/listings/distanceCalculator.js';
|
import { distanceMeters } from './services/listings/distanceCalculator.js';
|
||||||
import { getUserSettings, getSettings } from './services/storage/settingsStorage.js';
|
import { getSettings, getUserSettings } from './services/storage/settingsStorage.js';
|
||||||
import { updateListingDistance } from './services/storage/listingsStorage.js';
|
|
||||||
import booleanPointInPolygon from '@turf/boolean-point-in-polygon';
|
import booleanPointInPolygon from '@turf/boolean-point-in-polygon';
|
||||||
import { formatListing } from './utils/formatListing.js';
|
import { formatListing } from './utils/formatListing.js';
|
||||||
|
|
||||||
@@ -38,11 +38,15 @@ import { formatListing } from './utils/formatListing.js';
|
|||||||
* 3) Normalize listings to the provider schema
|
* 3) Normalize listings to the provider schema
|
||||||
* 4) Filter out incomplete/blacklisted listings
|
* 4) Filter out incomplete/blacklisted listings
|
||||||
* 5) Identify new listings (vs. previously stored hashes)
|
* 5) Identify new listings (vs. previously stored hashes)
|
||||||
* 6) Persist new listings
|
* 6) Optionally enrich new listings via provider.fetchDetails
|
||||||
* 7) Filter out entries similar to already seen ones
|
* 7) Optionally re-apply the provider blacklist using the (now enriched)
|
||||||
* 8) Filter out entries that do not match the job's specFilter
|
* description — only when the user opted in via
|
||||||
* 9) Filter out entries that do not match the job's spatialFilter
|
* `blacklist_filter_on_provider_details`
|
||||||
* 10) Dispatch notifications
|
* 8) Persist new listings
|
||||||
|
* 9) Filter out entries similar to already seen ones
|
||||||
|
* 10) Filter out entries that do not match the job's specFilter
|
||||||
|
* 11) Filter out entries that do not match the job's spatialFilter
|
||||||
|
* 12) Dispatch notifications
|
||||||
*/
|
*/
|
||||||
class FredyPipelineExecutioner {
|
class FredyPipelineExecutioner {
|
||||||
/**
|
/**
|
||||||
@@ -86,6 +90,7 @@ class FredyPipelineExecutioner {
|
|||||||
.then(this._filter.bind(this))
|
.then(this._filter.bind(this))
|
||||||
.then(this._findNew.bind(this))
|
.then(this._findNew.bind(this))
|
||||||
.then(this._fetchDetails.bind(this))
|
.then(this._fetchDetails.bind(this))
|
||||||
|
.then(this._filterAfterDetails.bind(this))
|
||||||
.then(this._geocode.bind(this))
|
.then(this._geocode.bind(this))
|
||||||
.then(this._save.bind(this))
|
.then(this._save.bind(this))
|
||||||
.then(this._calculateDistance.bind(this))
|
.then(this._calculateDistance.bind(this))
|
||||||
@@ -97,10 +102,10 @@ class FredyPipelineExecutioner {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Optionally enrich new listings with data from their detail pages.
|
* Optionally, enrich new listings with data from their detail pages.
|
||||||
* Only called when the provider config defines a `fetchDetails` function.
|
* Only called when the provider config defines a `fetchDetails` function.
|
||||||
* Runs all fetches in parallel. Each individual fetch must handle its own errors
|
* Fetches are performed sequentially to avoid overloading the provider or
|
||||||
* and always resolve (never reject) to avoid aborting other listings.
|
* the shared browser instance.
|
||||||
*
|
*
|
||||||
* @param {Listing[]} newListings New listings to enrich.
|
* @param {Listing[]} newListings New listings to enrich.
|
||||||
* @returns {Promise<Listing[]>} Resolves with enriched listings.
|
* @returns {Promise<Listing[]>} Resolves with enriched listings.
|
||||||
@@ -132,7 +137,7 @@ class FredyPipelineExecutioner {
|
|||||||
for (const listing of newListings) {
|
for (const listing of newListings) {
|
||||||
if (listing.address) {
|
if (listing.address) {
|
||||||
const coords = await geocodeAddress(listing.address);
|
const coords = await geocodeAddress(listing.address);
|
||||||
if (coords) {
|
if (coords && coords.lat !== -1 && coords.lng !== -1) {
|
||||||
listing.latitude = coords.lat;
|
listing.latitude = coords.lat;
|
||||||
listing.longitude = coords.lng;
|
listing.longitude = coords.lng;
|
||||||
}
|
}
|
||||||
@@ -199,9 +204,9 @@ class FredyPipelineExecutioner {
|
|||||||
const toDeleteListingByIds = [];
|
const toDeleteListingByIds = [];
|
||||||
const keptListings = newListings.filter((listing) => {
|
const keptListings = newListings.filter((listing) => {
|
||||||
const filterOut =
|
const filterOut =
|
||||||
(minRooms && listing.rooms && listing.rooms < minRooms) ||
|
(minRooms && listing.rooms != null && listing.rooms < minRooms) ||
|
||||||
(minSize && listing.size && listing.size < minSize) ||
|
(minSize && listing.size != null && listing.size < minSize) ||
|
||||||
(maxPrice && listing.price && listing.price > maxPrice);
|
(maxPrice && listing.price != null && listing.price > maxPrice);
|
||||||
|
|
||||||
if (filterOut) {
|
if (filterOut) {
|
||||||
toDeleteListingByIds.push(listing.id);
|
toDeleteListingByIds.push(listing.id);
|
||||||
@@ -223,24 +228,15 @@ class FredyPipelineExecutioner {
|
|||||||
* @param {string} url The provider URL to fetch from.
|
* @param {string} url The provider URL to fetch from.
|
||||||
* @returns {Promise<ParsedListing[]>} Resolves with an array of listings (empty when none found).
|
* @returns {Promise<ParsedListing[]>} Resolves with an array of listings (empty when none found).
|
||||||
*/
|
*/
|
||||||
_getListings(url) {
|
async _getListings(url) {
|
||||||
const extractor = new Extractor({ ...this._providerConfig.puppeteerOptions, browser: this._browser });
|
const extractor = new Extractor({ ...this._providerConfig.puppeteerOptions, browser: this._browser });
|
||||||
return new Promise((resolve, reject) => {
|
await extractor.execute(url, this._providerConfig.waitForSelector, this._providerId);
|
||||||
extractor
|
const listings = extractor.parseResponseText(
|
||||||
.execute(url, this._providerConfig.waitForSelector, this._jobKey)
|
this._providerConfig.crawlContainer,
|
||||||
.then(() => {
|
this._providerConfig.crawlFields,
|
||||||
const listings = extractor.parseResponseText(
|
url,
|
||||||
this._providerConfig.crawlContainer,
|
);
|
||||||
this._providerConfig.crawlFields,
|
return listings == null ? [] : listings;
|
||||||
url,
|
|
||||||
);
|
|
||||||
resolve(listings == null ? [] : listings);
|
|
||||||
})
|
|
||||||
.catch((err) => {
|
|
||||||
reject(err);
|
|
||||||
logger.error(err);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -264,15 +260,59 @@ class FredyPipelineExecutioner {
|
|||||||
const requiredKeys = this._providerConfig.requiredFieldNames;
|
const requiredKeys = this._providerConfig.requiredFieldNames;
|
||||||
const requireValues = ['id', 'link', 'title'];
|
const requireValues = ['id', 'link', 'title'];
|
||||||
|
|
||||||
const filteredListings = listings
|
return (
|
||||||
// this should never filter some listings out, because the normalize function should always extract all fields.
|
listings
|
||||||
.filter((item) => requiredKeys.every((key) => key in item))
|
// this should never filter some listings out, because the normalize function should always extract all fields.
|
||||||
// TODO: move blacklist filter to this file, so it will handle for all providers in same way.
|
.filter((item) => requiredKeys.every((key) => key in item))
|
||||||
.filter(this._providerConfig.filter)
|
// Drop listings missing a required identifying field *before* the provider
|
||||||
// filter out listings that are missing required fields
|
// filter runs, so provider filter functions never have to defend against a
|
||||||
.filter((item) => requireValues.every((key) => item[key] != null));
|
// null id/link/title.
|
||||||
|
.filter((item) => requireValues.every((key) => item[key] != null))
|
||||||
|
// TODO: move blacklist filter to this file, so it will handle for all providers in same way.
|
||||||
|
.filter(this._providerConfig.filter)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
return filteredListings;
|
/**
|
||||||
|
* Re-apply the provider's blacklist filter after `_fetchDetails` has had a
|
||||||
|
* chance to enrich the listings (e.g., load the full description from the
|
||||||
|
* detail page). The initial `_filter` step only sees the truncated snippet
|
||||||
|
* exposed on the search results page, so a blacklisted term that lives
|
||||||
|
* deeper in the listing's full description would otherwise slip through.
|
||||||
|
*
|
||||||
|
* Opt-in: gated by the user setting `blacklist_filter_on_provider_details`.
|
||||||
|
* The full detail description tends to contain a lot of boilerplate (legal,
|
||||||
|
* exposé contact info, generic marketing copy) which can accidentally match
|
||||||
|
* a blacklist term and remove otherwise relevant listings. Users who want
|
||||||
|
* the stricter behavior must enable the setting explicitly.
|
||||||
|
*
|
||||||
|
* Throws {@link NoNewListingsWarning} when all listings are filtered out
|
||||||
|
* so the rest of the pipeline (save + notify) is short-circuited.
|
||||||
|
*
|
||||||
|
* @param {ParsedListing[]} listings Enriched listings to re-filter.
|
||||||
|
* @returns {ParsedListing[]} Listings that still pass the provider's filter.
|
||||||
|
* @throws {NoNewListingsWarning} When every listing is filtered out.
|
||||||
|
*/
|
||||||
|
_filterAfterDetails(listings) {
|
||||||
|
if (typeof this._providerConfig.filter !== 'function') {
|
||||||
|
return listings;
|
||||||
|
}
|
||||||
|
const userId = getJob(this._jobKey)?.userId;
|
||||||
|
const enabled = getUserSettings(userId)?.blacklist_filter_on_provider_details === true;
|
||||||
|
if (!enabled) {
|
||||||
|
return listings;
|
||||||
|
}
|
||||||
|
const kept = listings.filter(this._providerConfig.filter);
|
||||||
|
const removed = listings.length - kept.length;
|
||||||
|
if (removed > 0) {
|
||||||
|
logger.debug(
|
||||||
|
`Re-filter after detail enrichment removed ${removed} listing(s) by blacklist (Provider: '${this._providerId}')`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (kept.length === 0) {
|
||||||
|
throw new NoNewListingsWarning();
|
||||||
|
}
|
||||||
|
return kept;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -284,9 +324,9 @@ class FredyPipelineExecutioner {
|
|||||||
*/
|
*/
|
||||||
_findNew(listings) {
|
_findNew(listings) {
|
||||||
logger.debug(`Checking ${listings.length} listings for new entries (Provider: '${this._providerId}')`);
|
logger.debug(`Checking ${listings.length} listings for new entries (Provider: '${this._providerId}')`);
|
||||||
const hashes = getKnownListingHashesForJobAndProvider(this._jobKey, this._providerId) || [];
|
const knownHashes = new Set(getKnownListingHashesForJobAndProvider(this._jobKey, this._providerId) || []);
|
||||||
|
|
||||||
const newListings = listings.filter((o) => !hashes.includes(o.id));
|
const newListings = listings.filter((o) => !knownHashes.has(o.id));
|
||||||
if (newListings.length === 0) {
|
if (newListings.length === 0) {
|
||||||
throw new NoNewListingsWarning();
|
throw new NoNewListingsWarning();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,5 +10,9 @@ export const TRACKING_POIS = {
|
|||||||
JOBS_TABLE_VIEW: 'JOBS_TABLE_VIEW',
|
JOBS_TABLE_VIEW: 'JOBS_TABLE_VIEW',
|
||||||
LISTING_TABLE_VIEW: 'LISTING_TABLE_VIEW',
|
LISTING_TABLE_VIEW: 'LISTING_TABLE_VIEW',
|
||||||
BASE_URL_SETTING: 'BASE_URL_SETTING',
|
BASE_URL_SETTING: 'BASE_URL_SETTING',
|
||||||
|
SET_PROXY_SETTING: 'SET_PROXY_SETTING',
|
||||||
DETECTED_AS_BOT: 'DETECTED_AS_BOT',
|
DETECTED_AS_BOT: 'DETECTED_AS_BOT',
|
||||||
|
NOTES_CREATE: 'NOTES_CREATE',
|
||||||
|
USING_LISTING_STATUS: 'USING_LISTING_STATUS',
|
||||||
|
CHANGE_LANGUAGE: 'CHANGE_LANGUAGE',
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ import userSettingsPlugin from './routes/userSettingsRoute.js';
|
|||||||
import trackingPlugin from './routes/trackingRoute.js';
|
import trackingPlugin from './routes/trackingRoute.js';
|
||||||
import generalSettingsPlugin from './routes/generalSettingsRoute.js';
|
import generalSettingsPlugin from './routes/generalSettingsRoute.js';
|
||||||
import backupPlugin from './routes/backupRouter.js';
|
import backupPlugin from './routes/backupRouter.js';
|
||||||
|
import debugPlugin, { registerDebugPublicProbe } from './routes/debugRouter.js';
|
||||||
import userPlugin from './routes/userRoute.js';
|
import userPlugin from './routes/userRoute.js';
|
||||||
import notificationAdapterPlugin from './routes/notificationAdapterRouter.js';
|
import notificationAdapterPlugin from './routes/notificationAdapterRouter.js';
|
||||||
import providerPlugin from './routes/providerRouter.js';
|
import providerPlugin from './routes/providerRouter.js';
|
||||||
@@ -77,6 +78,16 @@ fastify.register(async (app) => {
|
|||||||
app.register(userSettingsPlugin, { prefix: '/api/user/settings' });
|
app.register(userSettingsPlugin, { prefix: '/api/user/settings' });
|
||||||
app.register(trackingPlugin, { prefix: '/api/tracking' });
|
app.register(trackingPlugin, { prefix: '/api/tracking' });
|
||||||
app.register(generalSettingsPlugin, { prefix: '/api/admin/generalSettings' });
|
app.register(generalSettingsPlugin, { prefix: '/api/admin/generalSettings' });
|
||||||
|
// The lightweight /api/debug/active probe used by the app-wide red banner. Lives
|
||||||
|
// here (under authHook, NOT adminHook) so non-admin users also see the warning
|
||||||
|
// banner when an admin has enabled the feature, without exposing the rest of the
|
||||||
|
// settings payload.
|
||||||
|
app.register(
|
||||||
|
async (sub) => {
|
||||||
|
registerDebugPublicProbe(sub);
|
||||||
|
},
|
||||||
|
{ prefix: '/api/debug' },
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
// Admin-only routes
|
// Admin-only routes
|
||||||
@@ -84,6 +95,7 @@ fastify.register(async (app) => {
|
|||||||
app.addHook('preHandler', authHook);
|
app.addHook('preHandler', authHook);
|
||||||
app.addHook('preHandler', adminHook);
|
app.addHook('preHandler', adminHook);
|
||||||
app.register(backupPlugin, { prefix: '/api/admin/backup' });
|
app.register(backupPlugin, { prefix: '/api/admin/backup' });
|
||||||
|
app.register(debugPlugin, { prefix: '/api/admin/debug' });
|
||||||
app.register(userPlugin, { prefix: '/api/admin/users' });
|
app.register(userPlugin, { prefix: '/api/admin/users' });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -20,6 +20,28 @@ function cap(val) {
|
|||||||
return String(val).charAt(0).toUpperCase() + String(val).slice(1);
|
return String(val).charAt(0).toUpperCase() + String(val).slice(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compute the most recent job trigger timestamp across the given jobs.
|
||||||
|
*
|
||||||
|
* Returns `null` when none of the jobs has ever been triggered. The value is
|
||||||
|
* persisted per-job via `jobs.last_run_at`, so the dashboard reflects the
|
||||||
|
* scope visible to the current user (own + shared, or all for admins) rather
|
||||||
|
* than a process-wide in-memory value.
|
||||||
|
*
|
||||||
|
* @param {Array<{lastRunAt?: number|null}>} jobs
|
||||||
|
* @returns {number|null}
|
||||||
|
*/
|
||||||
|
function computeLastRun(jobs) {
|
||||||
|
let lastRun = null;
|
||||||
|
for (const job of jobs) {
|
||||||
|
const ts = job.lastRunAt;
|
||||||
|
if (typeof ts === 'number' && (lastRun == null || ts > lastRun)) {
|
||||||
|
lastRun = ts;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return lastRun;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param {import('fastify').FastifyInstance} fastify
|
* @param {import('fastify').FastifyInstance} fastify
|
||||||
*/
|
*/
|
||||||
@@ -46,11 +68,13 @@ export default async function dashboardPlugin(fastify) {
|
|||||||
}
|
}
|
||||||
: { labels: [], values: [] };
|
: { labels: [], values: [] };
|
||||||
|
|
||||||
|
const lastRun = computeLastRun(jobs);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
general: {
|
general: {
|
||||||
interval: settings.interval,
|
interval: settings.interval,
|
||||||
lastRun: settings.lastRun || null,
|
lastRun,
|
||||||
nextRun: settings.lastRun == null ? 0 : settings.lastRun + settings.interval * 60000,
|
nextRun: lastRun == null ? 0 : lastRun + settings.interval * 60000,
|
||||||
},
|
},
|
||||||
kpis: {
|
kpis: {
|
||||||
totalJobs,
|
totalJobs,
|
||||||
|
|||||||
93
lib/api/routes/debugRouter.js
Normal file
93
lib/api/routes/debugRouter.js
Normal file
@@ -0,0 +1,93 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (c) 2026 by Christian Kellner.
|
||||||
|
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||||
|
*/
|
||||||
|
|
||||||
|
import {
|
||||||
|
isEnabled,
|
||||||
|
enableDebugLogging,
|
||||||
|
disableDebugLogging,
|
||||||
|
getCurrentSize,
|
||||||
|
getMaxSize,
|
||||||
|
hasAnyLogs,
|
||||||
|
wasEverEnabled,
|
||||||
|
clearAllDebugLogs,
|
||||||
|
} from '../../services/debug/debugLogStorage.js';
|
||||||
|
import { buildDebugBundleFileName, buildDebugBundleZip } from '../../services/debug/debugBundleService.js';
|
||||||
|
import { getSettings } from '../../services/storage/settingsStorage.js';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build the JSON status payload returned by /status and after each enable/disable.
|
||||||
|
* @returns {Promise<{enabled:boolean, size:number, max:number, hasLogs:boolean, everEnabled:boolean}>}
|
||||||
|
*/
|
||||||
|
async function buildStatus() {
|
||||||
|
return {
|
||||||
|
enabled: isEnabled(),
|
||||||
|
size: await getCurrentSize(),
|
||||||
|
max: getMaxSize(),
|
||||||
|
hasLogs: hasAnyLogs(),
|
||||||
|
everEnabled: await wasEverEnabled(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Register the lightweight /active probe used by the app-wide red banner. Exposed
|
||||||
|
* to every authenticated user (not just admins) so non-admin users see the warning
|
||||||
|
* banner too. Returns only a single boolean so it cannot be repurposed to leak any
|
||||||
|
* other state.
|
||||||
|
*
|
||||||
|
* @param {import('fastify').FastifyInstance} fastify
|
||||||
|
*/
|
||||||
|
export async function registerDebugPublicProbe(fastify) {
|
||||||
|
fastify.get('/active', async () => ({ enabled: isEnabled() }));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Admin-only debug logging endpoints.
|
||||||
|
*
|
||||||
|
* Routes (all relative to the registered prefix /api/admin/debug):
|
||||||
|
* GET /status → current feature status (used by the UI polling).
|
||||||
|
* POST /enable → turn debug logging on. Body: { clearPrevious?:boolean }.
|
||||||
|
* POST /disable → turn debug logging off (existing logs are kept on disk).
|
||||||
|
* GET /download → ZIP with logs.txt + sys.txt. 409 when the feature has
|
||||||
|
* never been enabled OR there are no logs to export.
|
||||||
|
* DELETE /logs → drop every stored debug log row (does NOT change the
|
||||||
|
* enabled flag — useful to free space while keeping
|
||||||
|
* recording on).
|
||||||
|
*
|
||||||
|
* @param {import('fastify').FastifyInstance} fastify
|
||||||
|
*/
|
||||||
|
export default async function debugPlugin(fastify) {
|
||||||
|
fastify.get('/status', async () => buildStatus());
|
||||||
|
|
||||||
|
fastify.post('/enable', async (request) => {
|
||||||
|
const clearPrevious = request.body?.clearPrevious === true;
|
||||||
|
await enableDebugLogging({ clearPrevious });
|
||||||
|
return buildStatus();
|
||||||
|
});
|
||||||
|
|
||||||
|
fastify.post('/disable', async () => {
|
||||||
|
await disableDebugLogging();
|
||||||
|
return buildStatus();
|
||||||
|
});
|
||||||
|
|
||||||
|
fastify.delete('/logs', async () => {
|
||||||
|
clearAllDebugLogs();
|
||||||
|
return buildStatus();
|
||||||
|
});
|
||||||
|
|
||||||
|
fastify.get('/download', async (request, reply) => {
|
||||||
|
const ever = await wasEverEnabled();
|
||||||
|
if (!ever || !hasAnyLogs()) {
|
||||||
|
return reply.code(409).send({
|
||||||
|
error: 'Debug logging has never produced any data on this Fredy installation.',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const settings = await getSettings();
|
||||||
|
const zipBuffer = await buildDebugBundleZip({ settings });
|
||||||
|
const fileName = await buildDebugBundleFileName();
|
||||||
|
reply.header('Content-Type', 'application/zip');
|
||||||
|
reply.header('Content-Disposition', `attachment; filename="${fileName}"`);
|
||||||
|
return reply.send(zipBuffer);
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -44,6 +44,9 @@ export default async function generalSettingsPlugin(fastify) {
|
|||||||
if (appSettings.baseUrl != null) {
|
if (appSettings.baseUrl != null) {
|
||||||
await trackPoi(TRACKING_POIS.BASE_URL_SETTING);
|
await trackPoi(TRACKING_POIS.BASE_URL_SETTING);
|
||||||
}
|
}
|
||||||
|
if (appSettings.proxyUrl != null) {
|
||||||
|
await trackPoi(TRACKING_POIS.SET_PROXY_SETTING);
|
||||||
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
logger.error(err);
|
logger.error(err);
|
||||||
return reply.code(500).send({ error: 'Error while trying to write settings.' });
|
return reply.code(500).send({ error: 'Error while trying to write settings.' });
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ export default async function jobPlugin(fastify) {
|
|||||||
fastify.get('/', async (request) => {
|
fastify.get('/', async (request) => {
|
||||||
const isUserAdmin = isAdmin(request);
|
const isUserAdmin = isAdmin(request);
|
||||||
return jobStorage
|
return jobStorage
|
||||||
.getJobs()
|
.getJobs({ includeDisabled: true })
|
||||||
.filter(
|
.filter(
|
||||||
(job) =>
|
(job) =>
|
||||||
isUserAdmin ||
|
isUserAdmin ||
|
||||||
@@ -195,6 +195,9 @@ export default async function jobPlugin(fastify) {
|
|||||||
const settings = await getSettings();
|
const settings = await getSettings();
|
||||||
try {
|
try {
|
||||||
const job = jobStorage.getJob(jobId);
|
const job = jobStorage.getJob(jobId);
|
||||||
|
if (!job) {
|
||||||
|
return reply.code(404).send({ error: 'Job not found' });
|
||||||
|
}
|
||||||
if (settings.demoMode && !isAdmin(request) && job.name === DEMO_JOB_NAME) {
|
if (settings.demoMode && !isAdmin(request) && job.name === DEMO_JOB_NAME) {
|
||||||
return reply.code(403).send({ error: 'Sorry, but you cannot remove the Demo Job ;)' });
|
return reply.code(403).send({ error: 'Sorry, but you cannot remove the Demo Job ;)' });
|
||||||
}
|
}
|
||||||
@@ -216,6 +219,9 @@ export default async function jobPlugin(fastify) {
|
|||||||
const settings = await getSettings();
|
const settings = await getSettings();
|
||||||
try {
|
try {
|
||||||
const job = jobStorage.getJob(jobId);
|
const job = jobStorage.getJob(jobId);
|
||||||
|
if (!job) {
|
||||||
|
return reply.code(404).send({ error: 'Job not found' });
|
||||||
|
}
|
||||||
|
|
||||||
if (settings.demoMode && !isAdmin(request) && job.name === DEMO_JOB_NAME) {
|
if (settings.demoMode && !isAdmin(request) && job.name === DEMO_JOB_NAME) {
|
||||||
return reply.code(403).send({ error: 'Sorry, but you cannot change the Status of our Demo Job ;)' });
|
return reply.code(403).send({ error: 'Sorry, but you cannot change the Status of our Demo Job ;)' });
|
||||||
|
|||||||
@@ -8,8 +8,10 @@ import * as watchListStorage from '../../services/storage/watchListStorage.js';
|
|||||||
import { isAdmin as isAdminFn } from '../security.js';
|
import { isAdmin as isAdminFn } from '../security.js';
|
||||||
import logger from '../../services/logger.js';
|
import logger from '../../services/logger.js';
|
||||||
import { nullOrEmpty } from '../../utils.js';
|
import { nullOrEmpty } from '../../utils.js';
|
||||||
import { getJobs } from '../../services/storage/jobStorage.js';
|
import { getJob } from '../../services/storage/jobStorage.js';
|
||||||
import { getSettings } from '../../services/storage/settingsStorage.js';
|
import { getSettings } from '../../services/storage/settingsStorage.js';
|
||||||
|
import { trackPoi } from '../../services/tracking/Tracker.js';
|
||||||
|
import { TRACKING_POIS } from '../../TRACKING_POIS.js';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param {import('fastify').FastifyInstance} fastify
|
* @param {import('fastify').FastifyInstance} fastify
|
||||||
@@ -23,6 +25,8 @@ export default async function listingsPlugin(fastify) {
|
|||||||
jobNameFilter,
|
jobNameFilter,
|
||||||
providerFilter,
|
providerFilter,
|
||||||
watchListFilter,
|
watchListFilter,
|
||||||
|
statusFilter,
|
||||||
|
hiddenOnly,
|
||||||
sortfield = null,
|
sortfield = null,
|
||||||
sortdir = 'asc',
|
sortdir = 'asc',
|
||||||
freeTextFilter,
|
freeTextFilter,
|
||||||
@@ -35,12 +39,17 @@ export default async function listingsPlugin(fastify) {
|
|||||||
};
|
};
|
||||||
const normalizedActivity = toBool(activityFilter);
|
const normalizedActivity = toBool(activityFilter);
|
||||||
const normalizedWatch = toBool(watchListFilter);
|
const normalizedWatch = toBool(watchListFilter);
|
||||||
|
const normalizedHidden = toBool(hiddenOnly) === true;
|
||||||
|
const allowedStatuses = ['applied', 'rejected', 'accepted', 'none'];
|
||||||
|
const normalizedStatus =
|
||||||
|
typeof statusFilter === 'string' && allowedStatuses.includes(statusFilter.toLowerCase())
|
||||||
|
? statusFilter.toLowerCase()
|
||||||
|
: undefined;
|
||||||
|
|
||||||
let jobFilter = null;
|
let jobFilter = null;
|
||||||
let jobIdFilter = null;
|
let jobIdFilter = null;
|
||||||
const jobs = getJobs();
|
|
||||||
if (!nullOrEmpty(jobNameFilter)) {
|
if (!nullOrEmpty(jobNameFilter)) {
|
||||||
const job = jobs.find((j) => j.id === jobNameFilter);
|
const job = getJob(jobNameFilter);
|
||||||
jobFilter = job != null ? job.name : null;
|
jobFilter = job != null ? job.name : null;
|
||||||
jobIdFilter = job != null ? job.id : null;
|
jobIdFilter = job != null ? job.id : null;
|
||||||
}
|
}
|
||||||
@@ -54,6 +63,8 @@ export default async function listingsPlugin(fastify) {
|
|||||||
jobIdFilter: jobIdFilter,
|
jobIdFilter: jobIdFilter,
|
||||||
providerFilter,
|
providerFilter,
|
||||||
watchListFilter: normalizedWatch,
|
watchListFilter: normalizedWatch,
|
||||||
|
statusFilter: normalizedStatus,
|
||||||
|
hiddenOnly: normalizedHidden,
|
||||||
sortField: sortfield || null,
|
sortField: sortfield || null,
|
||||||
sortDir: sortdir === 'desc' ? 'desc' : 'asc',
|
sortDir: sortdir === 'desc' ? 'desc' : 'asc',
|
||||||
userId: request.session.currentUser,
|
userId: request.session.currentUser,
|
||||||
@@ -94,6 +105,55 @@ export default async function listingsPlugin(fastify) {
|
|||||||
return reply.send();
|
return reply.send();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
fastify.post('/:listingId/notes', async (request, reply) => {
|
||||||
|
const { listingId } = request.params || {};
|
||||||
|
const { notes } = request.body || {};
|
||||||
|
const userId = request.session?.currentUser;
|
||||||
|
if (!listingId || !userId) {
|
||||||
|
return reply.code(400).send({ message: 'listingId or user not provided' });
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const changes = listingStorage.setListingNotes(listingId, typeof notes === 'string' ? notes : null);
|
||||||
|
if (changes === 0) {
|
||||||
|
return reply.code(404).send({ message: 'Listing not found' });
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
logger.error(error);
|
||||||
|
return reply.code(500).send({ message: 'Failed to update listing notes' });
|
||||||
|
}
|
||||||
|
|
||||||
|
await trackPoi(TRACKING_POIS.NOTES_CREATE);
|
||||||
|
return reply.send();
|
||||||
|
});
|
||||||
|
|
||||||
|
fastify.post('/:listingId/status', async (request, reply) => {
|
||||||
|
const { listingId } = request.params || {};
|
||||||
|
const { status } = request.body || {};
|
||||||
|
const userId = request.session?.currentUser;
|
||||||
|
if (!listingId || !userId) {
|
||||||
|
return reply.code(400).send({ message: 'listingId or user not provided' });
|
||||||
|
}
|
||||||
|
const allowed = ['applied', 'rejected', 'accepted'];
|
||||||
|
const normalized = status == null ? null : String(status).toLowerCase();
|
||||||
|
if (normalized != null && !allowed.includes(normalized)) {
|
||||||
|
return reply.code(400).send({ message: `Invalid status: ${status}` });
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const changes = listingStorage.setListingStatus(listingId, normalized);
|
||||||
|
await trackPoi(TRACKING_POIS.USING_LISTING_STATUS);
|
||||||
|
if (changes === 0) {
|
||||||
|
return reply.code(404).send({ message: 'Listing not found' });
|
||||||
|
}
|
||||||
|
if (normalized != null) {
|
||||||
|
watchListStorage.ensureWatch(listingId, userId);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
logger.error(error);
|
||||||
|
return reply.code(500).send({ message: 'Failed to update listing status' });
|
||||||
|
}
|
||||||
|
return reply.send();
|
||||||
|
});
|
||||||
|
|
||||||
fastify.delete('/job', async (request, reply) => {
|
fastify.delete('/job', async (request, reply) => {
|
||||||
const { jobId, hardDelete = false } = request.body;
|
const { jobId, hardDelete = false } = request.body;
|
||||||
const settings = await getSettings();
|
const settings = await getSettings();
|
||||||
@@ -101,6 +161,16 @@ export default async function listingsPlugin(fastify) {
|
|||||||
if (settings.demoMode && !isAdminFn(request)) {
|
if (settings.demoMode && !isAdminFn(request)) {
|
||||||
return reply.code(403).send({ error: 'Sorry, but you cannot remove listings in demo mode ;)' });
|
return reply.code(403).send({ error: 'Sorry, but you cannot remove listings in demo mode ;)' });
|
||||||
}
|
}
|
||||||
|
const job = getJob(jobId);
|
||||||
|
if (!job) {
|
||||||
|
return reply.code(404).send({ error: 'Job not found' });
|
||||||
|
}
|
||||||
|
const userId = request.session.currentUser;
|
||||||
|
if (!isAdminFn(request) && job.userId !== userId && !job.shared_with_user.includes(userId)) {
|
||||||
|
return reply
|
||||||
|
.code(403)
|
||||||
|
.send({ error: 'You are trying to remove listings for a job that is not associated to your user' });
|
||||||
|
}
|
||||||
listingStorage.deleteListingsByJobId(jobId, hardDelete);
|
listingStorage.deleteListingsByJobId(jobId, hardDelete);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error(error);
|
logger.error(error);
|
||||||
@@ -111,7 +181,11 @@ export default async function listingsPlugin(fastify) {
|
|||||||
|
|
||||||
fastify.delete('/', async (request, reply) => {
|
fastify.delete('/', async (request, reply) => {
|
||||||
const { ids, hardDelete = false } = request.body;
|
const { ids, hardDelete = false } = request.body;
|
||||||
|
const settings = await getSettings();
|
||||||
try {
|
try {
|
||||||
|
if (settings.demoMode && !isAdminFn(request)) {
|
||||||
|
return reply.code(403).send({ error: 'Sorry, but you cannot remove listings in demo mode ;)' });
|
||||||
|
}
|
||||||
if (Array.isArray(ids) && ids.length > 0) {
|
if (Array.isArray(ids) && ids.length > 0) {
|
||||||
listingStorage.deleteListingsById(ids, hardDelete);
|
listingStorage.deleteListingsById(ids, hardDelete);
|
||||||
}
|
}
|
||||||
@@ -121,4 +195,21 @@ export default async function listingsPlugin(fastify) {
|
|||||||
}
|
}
|
||||||
return reply.send();
|
return reply.send();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
fastify.post('/restore', async (request, reply) => {
|
||||||
|
const { ids } = request.body || {};
|
||||||
|
const settings = await getSettings();
|
||||||
|
try {
|
||||||
|
if (settings.demoMode && !isAdminFn(request)) {
|
||||||
|
return reply.code(403).send({ error: 'Sorry, but you cannot restore listings in demo mode ;)' });
|
||||||
|
}
|
||||||
|
if (Array.isArray(ids) && ids.length > 0) {
|
||||||
|
listingStorage.restoreListingsById(ids);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
logger.error(error);
|
||||||
|
return reply.code(500).send({ error: error.message });
|
||||||
|
}
|
||||||
|
return reply.send();
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,6 +20,9 @@ function getClientIp(request) {
|
|||||||
|
|
||||||
function isRateLimited(ip) {
|
function isRateLimited(ip) {
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
|
for (const [key, rec] of loginAttempts) {
|
||||||
|
if (now - rec.firstAttempt > LOGIN_WINDOW_MS) loginAttempts.delete(key);
|
||||||
|
}
|
||||||
const record = loginAttempts.get(ip);
|
const record = loginAttempts.get(ip);
|
||||||
if (!record || now - record.firstAttempt > LOGIN_WINDOW_MS) {
|
if (!record || now - record.firstAttempt > LOGIN_WINDOW_MS) {
|
||||||
loginAttempts.set(ip, { count: 1, firstAttempt: now });
|
loginAttempts.set(ip, { count: 1, firstAttempt: now });
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ const notificationAdapter = await Promise.all(
|
|||||||
*/
|
*/
|
||||||
export default async function notificationAdapterPlugin(fastify) {
|
export default async function notificationAdapterPlugin(fastify) {
|
||||||
fastify.get('/', async () => {
|
fastify.get('/', async () => {
|
||||||
return notificationAdapter.map((adapter) => adapter.config);
|
return notificationAdapter.map((adapter) => adapter.config).filter(Boolean);
|
||||||
});
|
});
|
||||||
|
|
||||||
fastify.post('/try', async (request, reply) => {
|
fastify.post('/try', async (request, reply) => {
|
||||||
|
|||||||
@@ -3,13 +3,11 @@
|
|||||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import SqliteConnection from '../../services/storage/SqliteConnection.js';
|
import { getSettings, getUserSettings, upsertSettings } from '../../services/storage/settingsStorage.js';
|
||||||
import { getSettings, upsertSettings } from '../../services/storage/settingsStorage.js';
|
|
||||||
import { isAdmin } from '../security.js';
|
import { isAdmin } from '../security.js';
|
||||||
import { resetGeocoordinatesAndDistanceForUser } from '../../services/storage/listingsStorage.js';
|
import { resetGeocoordinatesAndDistanceForUser } from '../../services/storage/listingsStorage.js';
|
||||||
import { geocodeAddress } from '../../services/geocoding/geoCodingService.js';
|
import { geocodeAddress } from '../../services/geocoding/geoCodingService.js';
|
||||||
import { autocompleteAddress } from '../../services/geocoding/autocompleteService.js';
|
import { autocompleteAddress } from '../../services/geocoding/autocompleteService.js';
|
||||||
import { fromJson } from '../../utils.js';
|
|
||||||
import { trackPoi } from '../../services/tracking/Tracker.js';
|
import { trackPoi } from '../../services/tracking/Tracker.js';
|
||||||
import { TRACKING_POIS } from '../../TRACKING_POIS.js';
|
import { TRACKING_POIS } from '../../TRACKING_POIS.js';
|
||||||
import logger from '../../services/logger.js';
|
import logger from '../../services/logger.js';
|
||||||
@@ -21,12 +19,7 @@ import { runGeoCordTask } from '../../services/crons/geocoding-cron.js';
|
|||||||
export default async function userSettingsPlugin(fastify) {
|
export default async function userSettingsPlugin(fastify) {
|
||||||
fastify.get('/', async (request) => {
|
fastify.get('/', async (request) => {
|
||||||
const userId = request.session.currentUser;
|
const userId = request.session.currentUser;
|
||||||
const rows = SqliteConnection.query('SELECT name, value FROM settings WHERE user_id = @userId', { userId });
|
return getUserSettings(userId);
|
||||||
const settings = {};
|
|
||||||
for (const r of rows) {
|
|
||||||
settings[r.name] = fromJson(r.value, null);
|
|
||||||
}
|
|
||||||
return settings;
|
|
||||||
});
|
});
|
||||||
|
|
||||||
fastify.get('/autocomplete', async (request, reply) => {
|
fastify.get('/autocomplete', async (request, reply) => {
|
||||||
@@ -110,6 +103,28 @@ export default async function userSettingsPlugin(fastify) {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
fastify.post('/blacklist-filter-on-details', async (request, reply) => {
|
||||||
|
const userId = request.session.currentUser;
|
||||||
|
const { blacklist_filter_on_provider_details } = request.body;
|
||||||
|
|
||||||
|
const globalSettings = await getSettings();
|
||||||
|
if (globalSettings.demoMode && !isAdmin(request)) {
|
||||||
|
return reply.code(403).send({ error: 'In demo mode, it is not allowed to change settings.' });
|
||||||
|
}
|
||||||
|
|
||||||
|
if (typeof blacklist_filter_on_provider_details !== 'boolean') {
|
||||||
|
return reply.code(400).send({ error: 'blacklist_filter_on_provider_details must be a boolean.' });
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
upsertSettings({ blacklist_filter_on_provider_details }, userId);
|
||||||
|
return { success: true };
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Error updating blacklist-filter-on-details setting', error);
|
||||||
|
return reply.code(500).send({ error: error.message });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
fastify.post('/listings-view-mode', async (request, reply) => {
|
fastify.post('/listings-view-mode', async (request, reply) => {
|
||||||
const userId = request.session.currentUser;
|
const userId = request.session.currentUser;
|
||||||
const { listings_view_mode } = request.body;
|
const { listings_view_mode } = request.body;
|
||||||
@@ -151,4 +166,46 @@ export default async function userSettingsPlugin(fastify) {
|
|||||||
return reply.code(500).send({ error: error.message });
|
return reply.code(500).send({ error: error.message });
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
fastify.post('/listing-deletion-preference', async (request, reply) => {
|
||||||
|
const userId = request.session.currentUser;
|
||||||
|
const { listing_deletion_preference } = request.body;
|
||||||
|
|
||||||
|
const globalSettings = await getSettings();
|
||||||
|
if (globalSettings.demoMode && !isAdmin(request)) {
|
||||||
|
return reply.code(403).send({ error: 'In demo mode, it is not allowed to change settings.' });
|
||||||
|
}
|
||||||
|
|
||||||
|
if (listing_deletion_preference == null) {
|
||||||
|
return reply.code(400).send({ error: 'listing_deletion_preference is required.' });
|
||||||
|
}
|
||||||
|
|
||||||
|
const { skipPrompt, hardDelete } = listing_deletion_preference;
|
||||||
|
|
||||||
|
try {
|
||||||
|
upsertSettings({ listing_deletion_preference: { skipPrompt, hardDelete } }, userId);
|
||||||
|
return { success: true };
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Error updating listing deletion preference', error);
|
||||||
|
return reply.code(500).send({ error: error.message });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
fastify.post('/language', async (request, reply) => {
|
||||||
|
const userId = request.session.currentUser;
|
||||||
|
const { language } = request.body;
|
||||||
|
|
||||||
|
if (typeof language !== 'string' || language.trim() === '') {
|
||||||
|
return reply.code(400).send({ error: 'language must be a non-empty string.' });
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
upsertSettings({ language }, userId);
|
||||||
|
await trackPoi(TRACKING_POIS.CHANGE_LANGUAGE);
|
||||||
|
return { success: true };
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Error updating language setting', error);
|
||||||
|
return reply.code(500).send({ error: error.message });
|
||||||
|
}
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -155,6 +155,12 @@ export function createMcpServer() {
|
|||||||
),
|
),
|
||||||
sortField: z.string().optional().describe('Sort by: created_at, price, size, provider, title, is_active'),
|
sortField: z.string().optional().describe('Sort by: created_at, price, size, provider, title, is_active'),
|
||||||
sortDir: z.string().optional().describe('Sort direction: asc or desc'),
|
sortDir: z.string().optional().describe('Sort direction: asc or desc'),
|
||||||
|
status: z
|
||||||
|
.enum(['applied', 'rejected', 'accepted', 'none'])
|
||||||
|
.optional()
|
||||||
|
.describe(
|
||||||
|
'Filter by user-set status. "applied", "rejected", or "accepted" return only listings with that status; "none" returns only listings without a status set.',
|
||||||
|
),
|
||||||
},
|
},
|
||||||
async (
|
async (
|
||||||
{
|
{
|
||||||
@@ -170,6 +176,7 @@ export function createMcpServer() {
|
|||||||
maxPrice,
|
maxPrice,
|
||||||
sortField,
|
sortField,
|
||||||
sortDir,
|
sortDir,
|
||||||
|
status,
|
||||||
},
|
},
|
||||||
extra,
|
extra,
|
||||||
) => {
|
) => {
|
||||||
@@ -192,6 +199,7 @@ export function createMcpServer() {
|
|||||||
maxPrice: maxPrice ?? null,
|
maxPrice: maxPrice ?? null,
|
||||||
sortField: sortField ?? null,
|
sortField: sortField ?? null,
|
||||||
sortDir: sortDir ?? 'desc',
|
sortDir: sortDir ?? 'desc',
|
||||||
|
statusFilter: status,
|
||||||
userId: user.id,
|
userId: user.id,
|
||||||
isAdmin: user.isAdmin,
|
isAdmin: user.isAdmin,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -124,10 +124,10 @@ export function normalizeListListings(queryResult, { page, pageSize }) {
|
|||||||
md += '\n\n';
|
md += '\n\n';
|
||||||
|
|
||||||
if (listings.length > 0) {
|
if (listings.length > 0) {
|
||||||
md += `| ID | Title | Address | Price | Size | Provider | Active | Created | Job |\n`;
|
md += `| ID | Title | Address | Price | Size | Provider | Active | Status | Created | Job |\n`;
|
||||||
md += `|----|-------|---------|-------|------|----------|--------|---------|-----|\n`;
|
md += `|----|-------|---------|-------|------|----------|--------|--------|---------|-----|\n`;
|
||||||
for (const l of listings) {
|
for (const l of listings) {
|
||||||
md += `| ${cell(l.id)} | ${cell(l.title)} | ${cell(l.address)} | ${cell(l.price)} | ${cell(l.size)} | ${cell(l.provider)} | ${l.is_active ? 'yes' : 'no'} | ${formatDate(l.created_at)} | ${cell(l.job_name)} |\n`;
|
md += `| ${cell(l.id)} | ${cell(l.title)} | ${cell(l.address)} | ${cell(l.price)} | ${cell(l.size)} | ${cell(l.provider)} | ${l.is_active ? 'yes' : 'no'} | ${cell(l.status?.status)} | ${formatDate(l.created_at)} | ${cell(l.job_name)} |\n`;
|
||||||
}
|
}
|
||||||
md += `\nUse **get_listing** with an ID for full details (description, link, image).\n`;
|
md += `\nUse **get_listing** with an ID for full details (description, link, image).\n`;
|
||||||
} else {
|
} else {
|
||||||
@@ -156,6 +156,10 @@ export function normalizeGetListing(listing) {
|
|||||||
md += `- **Link:** ${listing.link || '–'}\n`;
|
md += `- **Link:** ${listing.link || '–'}\n`;
|
||||||
md += `- **Image:** ${listing.image_url || '–'}\n`;
|
md += `- **Image:** ${listing.image_url || '–'}\n`;
|
||||||
md += `- **Active:** ${listing.is_active ? 'yes' : 'no'}\n`;
|
md += `- **Active:** ${listing.is_active ? 'yes' : 'no'}\n`;
|
||||||
|
md += `- **Status:** ${listing.status?.status || '–'}\n`;
|
||||||
|
if (listing.status?.setAt) {
|
||||||
|
md += `- **Status set at:** ${formatDate(listing.status.setAt)}\n`;
|
||||||
|
}
|
||||||
md += `- **Created:** ${formatDate(listing.created_at)}\n`;
|
md += `- **Created:** ${formatDate(listing.created_at)}\n`;
|
||||||
md += `- **Job:** ${listing.job_name || '–'}\n`;
|
md += `- **Job:** ${listing.job_name || '–'}\n`;
|
||||||
if (listing.latitude != null && listing.longitude != null) {
|
if (listing.latitude != null && listing.longitude != null) {
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey, bas
|
|||||||
const jobName = job == null ? jobKey : job.name;
|
const jobName = job == null ? jobKey : job.name;
|
||||||
const promises = newListings.map((newListing) => {
|
const promises = newListings.map((newListing) => {
|
||||||
const title = `${jobName} at ${serviceName}: ${newListing.title}`;
|
const title = `${jobName} at ${serviceName}: ${newListing.title}`;
|
||||||
const fredyLine = baseUrl && newListing.id ? `\nOpen in Fredy: ${baseUrl}/listings/listing/${newListing.id}` : '';
|
const fredyLine = baseUrl && newListing.id ? `\nOpen in Fredy: ${baseUrl}/#/listings/listing/${newListing.id}` : '';
|
||||||
const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nLink: ${newListing.link}${fredyLine}`;
|
const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nLink: ${newListing.link}${fredyLine}`;
|
||||||
return fetch(server, {
|
return fetch(server, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import { markdown2Html } from '../../services/markdown.js';
|
|||||||
|
|
||||||
export const send = ({ serviceName, newListings, jobKey, baseUrl }) => {
|
export const send = ({ serviceName, newListings, jobKey, baseUrl }) => {
|
||||||
/* eslint-disable no-console */
|
/* eslint-disable no-console */
|
||||||
const fredyLinks = baseUrl ? newListings.map((l) => `${baseUrl}/listings/listing/${l.id}`).join(', ') : null;
|
const fredyLinks = baseUrl ? newListings.map((l) => `${baseUrl}/#/listings/listing/${l.id}`).join(', ') : null;
|
||||||
return [
|
return [
|
||||||
Promise.resolve(
|
Promise.resolve(
|
||||||
console.info(
|
console.info(
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import fetch from 'node-fetch';
|
|||||||
import { getJob } from '../../services/storage/jobStorage.js';
|
import { getJob } from '../../services/storage/jobStorage.js';
|
||||||
import { markdown2Html } from '../../services/markdown.js';
|
import { markdown2Html } from '../../services/markdown.js';
|
||||||
import { normalizeImageUrl } from '../../utils.js';
|
import { normalizeImageUrl } from '../../utils.js';
|
||||||
|
import logger from '../../services/logger.js';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Generates an idempotent decimal color code. The input string-based color code is
|
* Generates an idempotent decimal color code. The input string-based color code is
|
||||||
@@ -67,11 +68,19 @@ const buildEmbed = (jobKey, listing, baseUrl) => {
|
|||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
|
if (baseUrl && listing.id) {
|
||||||
|
fields.push({
|
||||||
|
name: 'Open in Fredy',
|
||||||
|
value: `[Open in Fredy](${baseUrl}/#/listings/listing/${listing.id})`,
|
||||||
|
inline: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
const embed = {
|
const embed = {
|
||||||
title: title,
|
title: title,
|
||||||
color: generateColorFromString(jobKey),
|
color: generateColorFromString(jobKey),
|
||||||
url: listing.link,
|
url: listing.link,
|
||||||
fields: fields,
|
fields,
|
||||||
};
|
};
|
||||||
|
|
||||||
if (listing.image) {
|
if (listing.image) {
|
||||||
@@ -80,14 +89,6 @@ const buildEmbed = (jobKey, listing, baseUrl) => {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
if (baseUrl && listing.id) {
|
|
||||||
fields.push({
|
|
||||||
name: 'Open in Fredy',
|
|
||||||
value: `[Open in Fredy](${baseUrl}/listings/listing/${listing.id})`,
|
|
||||||
inline: false,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
return embed;
|
return embed;
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -119,7 +120,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey, bas
|
|||||||
headers: { 'Content-Type': 'application/json' },
|
headers: { 'Content-Type': 'application/json' },
|
||||||
body,
|
body,
|
||||||
}).catch((error) => {
|
}).catch((error) => {
|
||||||
console.error(`Error sending Discord webhook for chunk starting at ${i}:`, error);
|
logger.error(`Error sending Discord webhook for chunk starting at ${i}:`, error);
|
||||||
return Promise.reject(new Error(`Webhook failed: ${error.message}`));
|
return Promise.reject(new Error(`Webhook failed: ${error.message}`));
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ const mapListing = (listing, baseUrl) => ({
|
|||||||
size: listing.size,
|
size: listing.size,
|
||||||
title: listing.title,
|
title: listing.title,
|
||||||
url: listing.link,
|
url: listing.link,
|
||||||
fredyUrl: baseUrl && listing.id ? `${baseUrl}/listings/listing/${listing.id}` : null,
|
fredyUrl: baseUrl && listing.id ? `${baseUrl}/#/listings/listing/${listing.id}` : null,
|
||||||
});
|
});
|
||||||
|
|
||||||
export const send = async ({ serviceName, newListings, notificationConfig, jobKey, baseUrl }) => {
|
export const send = async ({ serviceName, newListings, notificationConfig, jobKey, baseUrl }) => {
|
||||||
|
|||||||
@@ -53,7 +53,7 @@ const mapListingsWithCid = async (serviceName, jobKey, listings, baseUrl) => {
|
|||||||
jobKey,
|
jobKey,
|
||||||
hasImage: false,
|
hasImage: false,
|
||||||
imageCid: '',
|
imageCid: '',
|
||||||
fredyUrl: baseUrl && l.id ? `${baseUrl}/listings/listing/${l.id}` : null,
|
fredyUrl: baseUrl && l.id ? `${baseUrl}/#/listings/listing/${l.id}` : null,
|
||||||
};
|
};
|
||||||
|
|
||||||
if (imgUrl) {
|
if (imgUrl) {
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey, bas
|
|||||||
let message = `### *${jobName}* (${serviceName}) found **${newListings.length}** new listings:\n\n`;
|
let message = `### *${jobName}* (${serviceName}) found **${newListings.length}** new listings:\n\n`;
|
||||||
message += `| Title | Address | Size | Price |${baseUrl ? ' Open in Fredy |' : ''}\n|:----|:----|:----|:----|${baseUrl ? ':----|\n' : '\n'}`;
|
message += `| Title | Address | Size | Price |${baseUrl ? ' Open in Fredy |' : ''}\n|:----|:----|:----|:----|${baseUrl ? ':----|\n' : '\n'}`;
|
||||||
message += newListings.map((o) => {
|
message += newListings.map((o) => {
|
||||||
const fredyCell = baseUrl && o.id ? ` [Open in Fredy](${baseUrl}/listings/listing/${o.id}) |` : '';
|
const fredyCell = baseUrl && o.id ? ` [Open in Fredy](${baseUrl}/#/listings/listing/${o.id}) |` : '';
|
||||||
return (
|
return (
|
||||||
`| [${o.title}](${o.link}) | ` +
|
`| [${o.title}](${o.link}) | ` +
|
||||||
[o.address, o.size.replace(/2m/g, '$m^2$'), o.price].join(' | ') +
|
[o.address, o.size.replace(/2m/g, '$m^2$'), o.price].join(' | ') +
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey, bas
|
|||||||
const jobName = job == null ? jobKey : job.name;
|
const jobName = job == null ? jobKey : job.name;
|
||||||
|
|
||||||
const promises = newListings.map((newListing) => {
|
const promises = newListings.map((newListing) => {
|
||||||
const fredyLine = baseUrl && newListing.id ? `\nOpen in Fredy: ${baseUrl}/listings/listing/${newListing.id}` : '';
|
const fredyLine = baseUrl && newListing.id ? `\nOpen in Fredy: ${baseUrl}/#/listings/listing/${newListing.id}` : '';
|
||||||
const message = `
|
const message = `
|
||||||
Address: ${newListing.address}
|
Address: ${newListing.address}
|
||||||
Size: ${newListing.size == null ? 'N/A' : newListing.size.replace(/2m/g, '$m^2$')}
|
Size: ${newListing.size == null ? 'N/A' : newListing.size.replace(/2m/g, '$m^2$')}
|
||||||
|
|||||||
@@ -15,7 +15,8 @@ export const send = async ({ serviceName, newListings, notificationConfig, jobKe
|
|||||||
const results = await Promise.all(
|
const results = await Promise.all(
|
||||||
newListings.map(async (newListing) => {
|
newListings.map(async (newListing) => {
|
||||||
const title = `${jobName} at ${serviceName}: ${newListing.title}`;
|
const title = `${jobName} at ${serviceName}: ${newListing.title}`;
|
||||||
const fredyLine = baseUrl && newListing.id ? `\nOpen in Fredy: ${baseUrl}/listings/listing/${newListing.id}` : '';
|
const fredyLine =
|
||||||
|
baseUrl && newListing.id ? `\nOpen in Fredy: ${baseUrl}/#/listings/listing/${newListing.id}` : '';
|
||||||
const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nLink: ${newListing.link}${fredyLine}`;
|
const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nLink: ${newListing.link}${fredyLine}`;
|
||||||
|
|
||||||
const form = new FormData();
|
const form = new FormData();
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ const mapListings = (serviceName, jobKey, listings, baseUrl) =>
|
|||||||
price: l.price || '',
|
price: l.price || '',
|
||||||
image,
|
image,
|
||||||
hasImage: Boolean(image),
|
hasImage: Boolean(image),
|
||||||
fredyUrl: baseUrl && l.id ? `${baseUrl}/listings/listing/${l.id}` : null,
|
fredyUrl: baseUrl && l.id ? `${baseUrl}/#/listings/listing/${l.id}` : null,
|
||||||
serviceName,
|
serviceName,
|
||||||
jobKey,
|
jobKey,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ const mapListings = (serviceName, jobKey, listings, baseUrl) =>
|
|||||||
hasImage: Boolean(image),
|
hasImage: Boolean(image),
|
||||||
// optional plain text snippet
|
// optional plain text snippet
|
||||||
snippet: [l.address, l.price, l.size].filter(Boolean).join(' | '),
|
snippet: [l.address, l.price, l.size].filter(Boolean).join(' | '),
|
||||||
fredyUrl: baseUrl && l.id ? `${baseUrl}/listings/listing/${l.id}` : null,
|
fredyUrl: baseUrl && l.id ? `${baseUrl}/#/listings/listing/${l.id}` : null,
|
||||||
serviceName,
|
serviceName,
|
||||||
jobKey,
|
jobKey,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -39,7 +39,7 @@ const buildBlocks = (serviceName, jobKey, p, baseUrl) => {
|
|||||||
if (baseUrl && p.id) {
|
if (baseUrl && p.id) {
|
||||||
blocks.push({
|
blocks.push({
|
||||||
type: 'section',
|
type: 'section',
|
||||||
text: { type: 'mrkdwn', text: `<${baseUrl}/listings/listing/${p.id}|Open in Fredy>` },
|
text: { type: 'mrkdwn', text: `<${baseUrl}/#/listings/listing/${p.id}|Open in Fredy>` },
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -39,7 +39,7 @@ const buildBlocks = (serviceName, jobKey, p, baseUrl) => {
|
|||||||
if (baseUrl && p.id) {
|
if (baseUrl && p.id) {
|
||||||
blocks.push({
|
blocks.push({
|
||||||
type: 'section',
|
type: 'section',
|
||||||
text: { type: 'mrkdwn', text: `<${baseUrl}/listings/listing/${p.id}|Open in Fredy>` },
|
text: { type: 'mrkdwn', text: `<${baseUrl}/#/listings/listing/${p.id}|Open in Fredy>` },
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ const mapListings = (serviceName, jobKey, listings, baseUrl) =>
|
|||||||
price: l.price || '',
|
price: l.price || '',
|
||||||
image,
|
image,
|
||||||
hasImage: Boolean(image),
|
hasImage: Boolean(image),
|
||||||
fredyUrl: baseUrl && l.id ? `${baseUrl}/listings/listing/${l.id}` : null,
|
fredyUrl: baseUrl && l.id ? `${baseUrl}/#/listings/listing/${l.id}` : null,
|
||||||
serviceName,
|
serviceName,
|
||||||
jobKey,
|
jobKey,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -9,43 +9,48 @@ import fetch from 'node-fetch';
|
|||||||
import pThrottle from 'p-throttle';
|
import pThrottle from 'p-throttle';
|
||||||
import { normalizeImageUrl } from '../../utils.js';
|
import { normalizeImageUrl } from '../../utils.js';
|
||||||
import logger from '../../services/logger.js';
|
import logger from '../../services/logger.js';
|
||||||
|
import { shouldUseMultipart, buildPhotoFormData } from './telegramPhotoUploader.js';
|
||||||
|
|
||||||
const RATE_LIMIT_INTERVAL = 1000;
|
const RATE_LIMIT_INTERVAL = 1000;
|
||||||
|
const THROTTLE_MAX_IDLE_MS = RATE_LIMIT_INTERVAL + 2000;
|
||||||
const chatThrottleMap = new Map();
|
const chatThrottleMap = new Map();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Removes stale throttled call entries to keep memory bounded.
|
* Removes stale throttled call entries to keep memory bounded.
|
||||||
|
* An entry is stale when no API call has fired for longer than THROTTLE_MAX_IDLE_MS.
|
||||||
*/
|
*/
|
||||||
function cleanupOldThrottles() {
|
function cleanupOldThrottles() {
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
const maxAge = RATE_LIMIT_INTERVAL + 1000;
|
|
||||||
const toBeDeleted = [];
|
|
||||||
for (const [chatId, chatThrottle] of chatThrottleMap.entries()) {
|
for (const [chatId, chatThrottle] of chatThrottleMap.entries()) {
|
||||||
if (now - chatThrottle.lastUsedAt > maxAge) toBeDeleted.push(chatId);
|
if (now - chatThrottle.lastUsedAt > THROTTLE_MAX_IDLE_MS) chatThrottleMap.delete(chatId);
|
||||||
}
|
}
|
||||||
for (const chatId of toBeDeleted) chatThrottleMap.delete(chatId);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Return a throttled wrapper for a chatId to limit Telegram API calls.
|
* Return a throttled wrapper for a chatId to limit Telegram API calls.
|
||||||
* Uses p-throttle with 1 request per RATE_LIMIT_INTERVAL per chat.
|
* Uses p-throttle with 1 request per RATE_LIMIT_INTERVAL per chat.
|
||||||
|
* `lastUsedAt` is refreshed on every actual API call so that the idle window
|
||||||
|
* starts from the last fired call, not from when send() was invoked.
|
||||||
*
|
*
|
||||||
* @template {Function} T
|
|
||||||
* @param {string|number} chatId
|
* @param {string|number} chatId
|
||||||
* @param {T} call - async function (endpoint: string, body: any) => Promise<Response>
|
* @param {Function} call - async function (endpoint: string, body: any) => Promise<Response>
|
||||||
* @returns {T}
|
* @returns {Function}
|
||||||
*/
|
*/
|
||||||
function getThrottled(chatId, call) {
|
function getThrottled(chatId, call) {
|
||||||
cleanupOldThrottles();
|
cleanupOldThrottles();
|
||||||
const now = Date.now();
|
const existing = chatThrottleMap.get(chatId);
|
||||||
const chatThrottle = chatThrottleMap.get(chatId);
|
if (existing) {
|
||||||
if (chatThrottle) {
|
existing.lastUsedAt = Date.now();
|
||||||
chatThrottle.lastUsedAt = now;
|
return existing.throttled;
|
||||||
return chatThrottle.throttled;
|
|
||||||
}
|
}
|
||||||
const throttled = pThrottle({ limit: 1, interval: RATE_LIMIT_INTERVAL })(call);
|
const entry = { lastUsedAt: Date.now(), throttled: null };
|
||||||
chatThrottleMap.set(chatId, { lastUsedAt: now, throttled });
|
chatThrottleMap.set(chatId, entry);
|
||||||
return throttled;
|
entry.throttled = pThrottle({ limit: 1, interval: RATE_LIMIT_INTERVAL })(async (endpoint, body) => {
|
||||||
|
const e = chatThrottleMap.get(chatId);
|
||||||
|
if (e) e.lastUsedAt = Date.now();
|
||||||
|
return call(endpoint, body);
|
||||||
|
});
|
||||||
|
return entry.throttled;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -69,39 +74,20 @@ function escapeHtml(s = '') {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Build a Telegram photo caption (max 1024 characters) using HTML parse mode.
|
* Build a Telegram HTML-formatted message body.
|
||||||
|
* Suitable for both sendMessage (uncapped) and sendPhoto captions (caller must slice to 1024).
|
||||||
|
*
|
||||||
* @param {string} jobName
|
* @param {string} jobName
|
||||||
* @param {string} serviceName
|
* @param {string} serviceName
|
||||||
* @param {Object} o - Listing object
|
* @param {Object} o - Listing object
|
||||||
* @param {string} [o.title]
|
* @param {string} [baseUrl]
|
||||||
* @param {string} [o.address]
|
|
||||||
* @param {string|number} [o.price]
|
|
||||||
* @param {string|number} [o.size]
|
|
||||||
* @param {string} [o.link]
|
|
||||||
* @returns {string}
|
* @returns {string}
|
||||||
*/
|
*/
|
||||||
function buildCaption(jobName, serviceName, o, baseUrl) {
|
function buildHtmlBody(jobName, serviceName, o, baseUrl) {
|
||||||
const title = shorten((o.title || '').replace(/\*/g, ''), 90);
|
const title = shorten((o.title || '').replace(/\*/g, ''), 90);
|
||||||
const meta = [o.address, o.price, o.size].filter(Boolean).join(' | ');
|
const meta = [o.address, o.price, o.size].filter(Boolean).join(' | ');
|
||||||
const fredyLink =
|
const fredyLink =
|
||||||
baseUrl && o.id ? `\n<a href='${escapeHtml(`${baseUrl}/listings/listing/${o.id}`)}'>Open in Fredy</a>` : '';
|
baseUrl && o.id ? `\n<a href='${escapeHtml(`${baseUrl}/#/listings/listing/${o.id}`)}'>Open in Fredy</a>` : '';
|
||||||
return `<i>${escapeHtml(jobName)}</i> (${escapeHtml(serviceName)})\n<a href='${escapeHtml(
|
|
||||||
o.link || '',
|
|
||||||
)}'><b>${escapeHtml(title)}</b></a>\n${escapeHtml(meta)}${fredyLink}`.slice(0, 1024);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Build a Telegram message text using HTML parse mode.
|
|
||||||
* @param {string} jobName
|
|
||||||
* @param {string} serviceName
|
|
||||||
* @param {Object} o - Listing object
|
|
||||||
* @returns {string}
|
|
||||||
*/
|
|
||||||
function buildText(jobName, serviceName, o, baseUrl) {
|
|
||||||
const title = shorten((o.title || '').replace(/\*/g, ''), 90);
|
|
||||||
const meta = [o.address, o.price, o.size].filter(Boolean).join(' | ');
|
|
||||||
const fredyLink =
|
|
||||||
baseUrl && o.id ? `\n<a href='${escapeHtml(`${baseUrl}/listings/listing/${o.id}`)}'>Open in Fredy</a>` : '';
|
|
||||||
return (
|
return (
|
||||||
`<i>${escapeHtml(jobName)}</i> (${escapeHtml(serviceName)})\n` +
|
`<i>${escapeHtml(jobName)}</i> (${escapeHtml(serviceName)})\n` +
|
||||||
`<a href='${escapeHtml(o.link || '')}'><b>${escapeHtml(title)}</b></a>\n` +
|
`<a href='${escapeHtml(o.link || '')}'><b>${escapeHtml(title)}</b></a>\n` +
|
||||||
@@ -110,34 +96,128 @@ function buildText(jobName, serviceName, o, baseUrl) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Build a plain text Telegram photo caption (max 4096 characters).
|
* Build a plain-text Telegram photo caption (max 4096 characters).
|
||||||
|
* Meta appears before the link so the most relevant info is visible within the cap.
|
||||||
|
*
|
||||||
* @param {string} jobName
|
* @param {string} jobName
|
||||||
* @param {string} serviceName
|
* @param {string} serviceName
|
||||||
* @param {Object} o - Listing object
|
* @param {Object} o - Listing object
|
||||||
* @param baseUrl
|
* @param {string} [baseUrl]
|
||||||
* @returns {string}
|
* @returns {string}
|
||||||
*/
|
*/
|
||||||
function buildCaptionPlain(jobName, serviceName, o, baseUrl) {
|
function buildPlainCaption(jobName, serviceName, o, baseUrl) {
|
||||||
const title = shorten((o.title || '').replace(/\*/g, ''), 90);
|
const title = shorten((o.title || '').replace(/\*/g, ''), 90);
|
||||||
const meta = [o.address, o.price, o.size].filter(Boolean).join(' | ');
|
const meta = [o.address, o.price, o.size].filter(Boolean).join(' | ');
|
||||||
const fredyLine = baseUrl && o.id ? `\nOpen in Fredy: ${baseUrl}/listings/listing/${o.id}` : '';
|
const fredyLine = baseUrl && o.id ? `\nOpen in Fredy: ${baseUrl}/#/listings/listing/${o.id}` : '';
|
||||||
return `${jobName} (${serviceName})\n${title}\n${meta}\n\n${o.link || ''}${fredyLine}`.slice(0, 4096);
|
return `${jobName} (${serviceName})\n${title}\n${meta}\n\n${o.link || ''}${fredyLine}`.slice(0, 4096);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Build a plain text Telegram message.
|
* Build a plain-text Telegram message body.
|
||||||
|
* Link appears early so it is tappable without scrolling.
|
||||||
|
*
|
||||||
* @param {string} jobName
|
* @param {string} jobName
|
||||||
* @param {string} serviceName
|
* @param {string} serviceName
|
||||||
* @param {Object} o - Listing object
|
* @param {Object} o - Listing object
|
||||||
|
* @param {string} [baseUrl]
|
||||||
* @returns {string}
|
* @returns {string}
|
||||||
*/
|
*/
|
||||||
function buildTextPlain(jobName, serviceName, o, baseUrl) {
|
function buildPlainText(jobName, serviceName, o, baseUrl) {
|
||||||
const title = shorten((o.title || '').replace(/\*/g, ''), 90);
|
const title = shorten((o.title || '').replace(/\*/g, ''), 90);
|
||||||
const meta = [o.address, o.price, o.size].filter(Boolean).join(' | ');
|
const meta = [o.address, o.price, o.size].filter(Boolean).join(' | ');
|
||||||
const fredyLine = baseUrl && o.id ? `\nOpen in Fredy: ${baseUrl}/listings/listing/${o.id}` : '';
|
const fredyLine = baseUrl && o.id ? `\nOpen in Fredy: ${baseUrl}/#/listings/listing/${o.id}` : '';
|
||||||
return `${jobName} (${serviceName})\n${title}\n${o.link || ''}\n${meta}${fredyLine}`;
|
return `${jobName} (${serviceName})\n${title}\n${o.link || ''}\n${meta}${fredyLine}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create the raw Telegram API caller for a given bot token.
|
||||||
|
* Handles JSON and multipart (FormData) bodies.
|
||||||
|
*
|
||||||
|
* @param {string} token - Telegram bot token.
|
||||||
|
* @param {string} jobName - Used in error messages.
|
||||||
|
* @returns {(endpoint: string, body: object|FormData) => Promise<Response>}
|
||||||
|
*/
|
||||||
|
function makeTelegramCaller(token, jobName) {
|
||||||
|
return async function (endpoint, body) {
|
||||||
|
const isFormData = body instanceof FormData;
|
||||||
|
const opts = isFormData
|
||||||
|
? { method: 'post', body }
|
||||||
|
: { method: 'post', body: JSON.stringify(body), headers: { 'Content-Type': 'application/json' } };
|
||||||
|
const res = await fetch(`https://api.telegram.org/bot${token}/${endpoint}`, opts);
|
||||||
|
if (!res.ok) {
|
||||||
|
const errorBody = await res.text();
|
||||||
|
throw new Error(`API error for '${jobName}'. '${endpoint}' returned ${errorBody}`);
|
||||||
|
}
|
||||||
|
return res;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Send a single listing to a single Telegram chat, with photo-then-text fallback.
|
||||||
|
*
|
||||||
|
* @param {Function} throttledCall - Throttled Telegram API caller for this chat.
|
||||||
|
* @param {Object} listing - Listing object.
|
||||||
|
* @param {string|number} chatId
|
||||||
|
* @param {Object} opts
|
||||||
|
* @param {string} opts.jobName
|
||||||
|
* @param {string} opts.serviceName
|
||||||
|
* @param {string} opts.baseUrl
|
||||||
|
* @param {boolean} opts.plainText
|
||||||
|
* @param {number|undefined} opts.message_thread_id
|
||||||
|
* @returns {Promise<void>}
|
||||||
|
*/
|
||||||
|
async function sendListingToChat(
|
||||||
|
throttledCall,
|
||||||
|
listing,
|
||||||
|
chatId,
|
||||||
|
{ jobName, serviceName, baseUrl, plainText, message_thread_id },
|
||||||
|
) {
|
||||||
|
const img = normalizeImageUrl(listing.image);
|
||||||
|
|
||||||
|
const textPayload = {
|
||||||
|
chat_id: chatId,
|
||||||
|
text: plainText
|
||||||
|
? buildPlainText(jobName, serviceName, listing, baseUrl)
|
||||||
|
: buildHtmlBody(jobName, serviceName, listing, baseUrl),
|
||||||
|
...(plainText ? {} : { parse_mode: 'HTML' }),
|
||||||
|
disable_web_page_preview: true,
|
||||||
|
...(message_thread_id ? { message_thread_id } : {}),
|
||||||
|
};
|
||||||
|
|
||||||
|
if (!img) {
|
||||||
|
return throttledCall('sendMessage', textPayload).catch((e) => {
|
||||||
|
logger.error(`Error sending message to Telegram: ${e.message}`);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const caption = plainText
|
||||||
|
? buildPlainCaption(jobName, serviceName, listing, baseUrl)
|
||||||
|
: buildHtmlBody(jobName, serviceName, listing, baseUrl).slice(0, 1024);
|
||||||
|
const parseMode = plainText ? undefined : 'HTML';
|
||||||
|
|
||||||
|
// .webp URLs (Immowelt/Cloudimage) fail Telegram's URL-based sendPhoto with
|
||||||
|
// "failed to get HTTP URL content". Upload the bytes via multipart instead.
|
||||||
|
const photoCall = shouldUseMultipart(img)
|
||||||
|
? buildPhotoFormData({ chatId, imageUrl: img, caption, parseMode, messageThreadId: message_thread_id }).then((fd) =>
|
||||||
|
throttledCall('sendPhoto', fd),
|
||||||
|
)
|
||||||
|
: throttledCall('sendPhoto', {
|
||||||
|
chat_id: chatId,
|
||||||
|
photo: img,
|
||||||
|
caption,
|
||||||
|
...(parseMode ? { parse_mode: parseMode } : {}),
|
||||||
|
...(message_thread_id ? { message_thread_id } : {}),
|
||||||
|
});
|
||||||
|
|
||||||
|
return photoCall.catch(async (e) => {
|
||||||
|
logger.warn(`Error sending photo to Telegram and use a fallback: ${e.message}`);
|
||||||
|
return throttledCall('sendMessage', textPayload).catch((e) => {
|
||||||
|
logger.error(`Error sending message to Telegram: ${e.message}`);
|
||||||
|
throw e;
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Send new listings to Telegram.
|
* Send new listings to Telegram.
|
||||||
* - Respects per-chat Telegram rate limits using a lightweight throttle cache.
|
* - Respects per-chat Telegram rate limits using a lightweight throttle cache.
|
||||||
@@ -160,6 +240,11 @@ export const send = ({ serviceName, newListings = [], notificationConfig, jobKey
|
|||||||
throw new Error("Telegram 'token' and 'chatId' must be provided in notification config");
|
throw new Error("Telegram 'token' and 'chatId' must be provided in notification config");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const chatIds = String(chatId)
|
||||||
|
.split(',')
|
||||||
|
.map((s) => s.trim())
|
||||||
|
.filter(Boolean);
|
||||||
|
|
||||||
// Optional Telegram topic/thread support (supergroups)
|
// Optional Telegram topic/thread support (supergroups)
|
||||||
let message_thread_id;
|
let message_thread_id;
|
||||||
if (messageThreadId !== undefined && messageThreadId !== null && `${messageThreadId}`.trim() !== '') {
|
if (messageThreadId !== undefined && messageThreadId !== null && `${messageThreadId}`.trim() !== '') {
|
||||||
@@ -176,56 +261,16 @@ export const send = ({ serviceName, newListings = [], notificationConfig, jobKey
|
|||||||
const job = getJob(jobKey);
|
const job = getJob(jobKey);
|
||||||
const jobName = job == null ? jobKey : job.name;
|
const jobName = job == null ? jobKey : job.name;
|
||||||
|
|
||||||
const throttledCall = getThrottled(chatId, async function (endpoint, body) {
|
|
||||||
const res = await fetch(`https://api.telegram.org/bot${token}/${endpoint}`, {
|
|
||||||
method: 'post',
|
|
||||||
body: JSON.stringify(body),
|
|
||||||
headers: { 'Content-Type': 'application/json' },
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!res.ok) {
|
|
||||||
const errorBody = await res.text();
|
|
||||||
throw new Error(`API error for '${jobName}'. '${endpoint}' returned ${errorBody}`);
|
|
||||||
}
|
|
||||||
return res;
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!Array.isArray(newListings) || newListings.length === 0) return Promise.resolve([]);
|
if (!Array.isArray(newListings) || newListings.length === 0) return Promise.resolve([]);
|
||||||
|
|
||||||
const promises = newListings.map(async (o) => {
|
const allPromises = chatIds.flatMap((id) => {
|
||||||
const img = normalizeImageUrl(o.image);
|
const caller = makeTelegramCaller(token, jobName);
|
||||||
const textPayload = {
|
const throttledCall = getThrottled(id, caller);
|
||||||
chat_id: chatId,
|
const opts = { jobName, serviceName, baseUrl, plainText, message_thread_id };
|
||||||
text: plainText ? buildTextPlain(jobName, serviceName, o, baseUrl) : buildText(jobName, serviceName, o, baseUrl),
|
return newListings.map((listing) => sendListingToChat(throttledCall, listing, id, opts));
|
||||||
...(plainText ? {} : { parse_mode: 'HTML' }),
|
|
||||||
disable_web_page_preview: true,
|
|
||||||
...(message_thread_id ? { message_thread_id } : {}),
|
|
||||||
};
|
|
||||||
|
|
||||||
if (!img) {
|
|
||||||
return await throttledCall('sendMessage', textPayload).catch(async (e) => {
|
|
||||||
logger.error(`Error sending message to Telegram: ${e.message}`);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
return await throttledCall('sendPhoto', {
|
|
||||||
chat_id: chatId,
|
|
||||||
photo: img,
|
|
||||||
caption: plainText
|
|
||||||
? buildCaptionPlain(jobName, serviceName, o, baseUrl)
|
|
||||||
: buildCaption(jobName, serviceName, o, baseUrl),
|
|
||||||
...(plainText ? {} : { parse_mode: 'HTML' }),
|
|
||||||
...(message_thread_id ? { message_thread_id } : {}),
|
|
||||||
}).catch(async (e) => {
|
|
||||||
logger.error(`Error sending photo to Telegram and use a fallback: ${e.message}`);
|
|
||||||
return await throttledCall('sendMessage', textPayload).catch((e) => {
|
|
||||||
logger.error(`Error sending message to Telegram: ${e.message}`);
|
|
||||||
throw e;
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
|
|
||||||
return Promise.all(promises);
|
return Promise.all(allPromises);
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -246,7 +291,8 @@ export const config = {
|
|||||||
chatId: {
|
chatId: {
|
||||||
type: 'chatId',
|
type: 'chatId',
|
||||||
label: 'Chat Id',
|
label: 'Chat Id',
|
||||||
description: 'The chat id to send messages to you.',
|
description:
|
||||||
|
'The chat ID to send messages to. Separate multiple IDs with commas to notify several recipients (e.g. 123456789, 987654321).',
|
||||||
},
|
},
|
||||||
messageThreadId: {
|
messageThreadId: {
|
||||||
type: 'text',
|
type: 'text',
|
||||||
|
|||||||
@@ -21,6 +21,8 @@ Steps:
|
|||||||
- Private chats: `chat.id` is a positive number
|
- Private chats: `chat.id` is a positive number
|
||||||
- Groups/supergroups: `chat.id` is a negative number
|
- Groups/supergroups: `chat.id` is a negative number
|
||||||
|
|
||||||
|
**Multiple recipients:** To notify several users individually, enter a comma-separated list of chat IDs in the Chat Id field, e.g. `123456789, 987654321`. Each recipient receives the same messages and gets its own independent rate-limit window. This avoids having to create a group and add the bot to it.
|
||||||
|
|
||||||
Keep your bot token secret. If `getUpdates` returns an empty list, send a new message and try again, or make sure your bot’s privacy settings allow it to see group messages when used in groups.
|
Keep your bot token secret. If `getUpdates` returns an empty list, send a new message and try again, or make sure your bot’s privacy settings allow it to see group messages when used in groups.
|
||||||
|
|
||||||
#### Getting the thread ID (this is optional to be used for forum topics)
|
#### Getting the thread ID (this is optional to be used for forum topics)
|
||||||
|
|||||||
106
lib/notification/adapter/telegramPhotoUploader.js
Normal file
106
lib/notification/adapter/telegramPhotoUploader.js
Normal file
@@ -0,0 +1,106 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (c) 2026 by Christian Kellner.
|
||||||
|
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Helpers for sending photos to Telegram via `multipart/form-data` instead of
|
||||||
|
* the HTTP-URL path. Used when the URL is one that Telegram's URL-fetcher will
|
||||||
|
* reject - notably `.webp` images from Cloudimage (mms.immowelt.de), which
|
||||||
|
* Telegram refuses with "Bad Request: failed to get HTTP URL content".
|
||||||
|
*
|
||||||
|
* The HTTP-URL path is faster and is still the default in telegram.js; this
|
||||||
|
* module is the fallback for URLs whose extension makes Telegram fail.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/** Telegram's sendPhoto limit when uploading bytes via multipart/form-data. */
|
||||||
|
const TELEGRAM_MULTIPART_MAX_BYTES = 10 * 1024 * 1024;
|
||||||
|
|
||||||
|
/** Accept header used when re-fetching the image ourselves.
|
||||||
|
* Deliberately excludes `image/webp` so CDNs that content-negotiate
|
||||||
|
* (like Cloudimage on mms.immowelt.de) transcode WEBP to JPEG. */
|
||||||
|
const NON_WEBP_ACCEPT = 'image/jpeg,image/png,image/*;q=0.8';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns true if the URL's path ends in a `.webp` extension. Such URLs need
|
||||||
|
* multipart upload because Telegram identifies media types from the URL path
|
||||||
|
* and rejects `.webp` in sendPhoto via HTTP URL.
|
||||||
|
*
|
||||||
|
* Conservative: returns false for null/empty/non-string input, malformed URLs,
|
||||||
|
* and non-https schemes.
|
||||||
|
*
|
||||||
|
* @param {string|null|undefined} url
|
||||||
|
* @returns {boolean}
|
||||||
|
*/
|
||||||
|
export function shouldUseMultipart(url) {
|
||||||
|
if (typeof url !== 'string' || url.length === 0) return false;
|
||||||
|
let parsed;
|
||||||
|
try {
|
||||||
|
parsed = new URL(url);
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (parsed.protocol !== 'https:') return false;
|
||||||
|
return /\.webp$/i.test(parsed.pathname);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fetch an image from `imageUrl` and build a `FormData` body suitable for
|
||||||
|
* POSTing to `https://api.telegram.org/bot<token>/sendPhoto`.
|
||||||
|
*
|
||||||
|
* - Sends an `Accept` header that excludes `image/webp` so origin/CDN servers
|
||||||
|
* that content-negotiate return JPEG bytes.
|
||||||
|
* - Rejects images larger than Telegram's 10 MB multipart limit, both
|
||||||
|
* advertised via `Content-Length` and (defensively) after download.
|
||||||
|
* - The `photo` field is named with a `.jpg` extension because Telegram
|
||||||
|
* identifies file type from the filename.
|
||||||
|
*
|
||||||
|
* Throws if the image fetch fails, the size limit is exceeded, or the URL is
|
||||||
|
* unreachable. The caller is responsible for catching and falling back.
|
||||||
|
*
|
||||||
|
* @param {Object} args
|
||||||
|
* @param {string|number} args.chatId
|
||||||
|
* @param {string} args.imageUrl
|
||||||
|
* @param {string} args.caption
|
||||||
|
* @param {string} [args.parseMode] - Telegram parse_mode, e.g. 'HTML'.
|
||||||
|
* @param {number} [args.messageThreadId] - Telegram supergroup topic id.
|
||||||
|
* @returns {Promise<FormData>}
|
||||||
|
*/
|
||||||
|
export async function buildPhotoFormData({ chatId, imageUrl, caption, parseMode, messageThreadId }) {
|
||||||
|
const res = await fetch(imageUrl, {
|
||||||
|
method: 'GET',
|
||||||
|
headers: { Accept: NON_WEBP_ACCEPT },
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!res.ok) {
|
||||||
|
throw new Error(`Failed to fetch image for multipart upload (${res.status}): ${imageUrl}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const advertised = Number(res.headers.get('content-length'));
|
||||||
|
if (Number.isFinite(advertised) && advertised > TELEGRAM_MULTIPART_MAX_BYTES) {
|
||||||
|
throw new Error(
|
||||||
|
`Image exceeds Telegram multipart size limit (advertised ${advertised} bytes, max ${TELEGRAM_MULTIPART_MAX_BYTES}): ${imageUrl}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const buf = await res.arrayBuffer();
|
||||||
|
if (buf.byteLength > TELEGRAM_MULTIPART_MAX_BYTES) {
|
||||||
|
throw new Error(
|
||||||
|
`Image exceeds Telegram multipart size limit (downloaded ${buf.byteLength} bytes, max ${TELEGRAM_MULTIPART_MAX_BYTES}): ${imageUrl}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Telegram identifies the media type from the filename extension. We always
|
||||||
|
// upload as .jpg because the Accept header forces JPEG bytes from CDNs that
|
||||||
|
// honor it; for the rare CDN that ignores Accept and still returns WEBP, the
|
||||||
|
// .jpg filename is a small lie but Telegram's image pipeline accepts it.
|
||||||
|
const blob = new Blob([buf], { type: 'image/jpeg' });
|
||||||
|
|
||||||
|
const fd = new FormData();
|
||||||
|
fd.append('chat_id', String(chatId));
|
||||||
|
fd.append('caption', caption);
|
||||||
|
if (parseMode) fd.append('parse_mode', parseMode);
|
||||||
|
if (messageThreadId != null) fd.append('message_thread_id', String(messageThreadId));
|
||||||
|
fd.append('photo', blob, 'photo.jpg');
|
||||||
|
return fd;
|
||||||
|
}
|
||||||
@@ -4,6 +4,7 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
import fs from 'fs';
|
import fs from 'fs';
|
||||||
|
import logger from '../services/logger.js';
|
||||||
const path = './adapter';
|
const path = './adapter';
|
||||||
|
|
||||||
/** Read every integration existing in ./adapter **/
|
/** Read every integration existing in ./adapter **/
|
||||||
@@ -23,7 +24,13 @@ const findAdapter = (notificationAdapter) => {
|
|||||||
export const send = (serviceName, newListings, notificationConfig, jobKey, baseUrl) => {
|
export const send = (serviceName, newListings, notificationConfig, jobKey, baseUrl) => {
|
||||||
//this is not being used in tests, therefore adapter are always set
|
//this is not being used in tests, therefore adapter are always set
|
||||||
return notificationConfig
|
return notificationConfig
|
||||||
.filter((notificationAdapter) => findAdapter(notificationAdapter) != null)
|
.map((notificationAdapter) => {
|
||||||
.map((notificationAdapter) => findAdapter(notificationAdapter))
|
const found = findAdapter(notificationAdapter);
|
||||||
|
if (!found) {
|
||||||
|
logger.warn(`Notification adapter '${notificationAdapter.id}' not found for job '${jobKey || ''}'`);
|
||||||
|
}
|
||||||
|
return found;
|
||||||
|
})
|
||||||
|
.filter(Boolean)
|
||||||
.map((a) => a.send({ serviceName, newListings, notificationConfig, jobKey, baseUrl }));
|
.map((a) => a.send({ serviceName, newListings, notificationConfig, jobKey, baseUrl }));
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ function normalize(o) {
|
|||||||
const link = `${baseUrl}/expose/${o.id}.html`;
|
const link = `${baseUrl}/expose/${o.id}.html`;
|
||||||
const price = normalizePrice(o.price);
|
const price = normalizePrice(o.price);
|
||||||
const id = buildHash(o.id, price);
|
const id = buildHash(o.id, price);
|
||||||
const image = baseUrl + o.image;
|
const image = o.image == null ? null : baseUrl + o.image;
|
||||||
const address = o.address == null ? null : o.address.trim().replaceAll('/', ',');
|
const address = o.address == null ? null : o.address.trim().replaceAll('/', ',');
|
||||||
return {
|
return {
|
||||||
id,
|
id,
|
||||||
|
|||||||
@@ -198,7 +198,9 @@ function normalize(o) {
|
|||||||
* @returns {boolean}
|
* @returns {boolean}
|
||||||
*/
|
*/
|
||||||
function applyBlacklist(o) {
|
function applyBlacklist(o) {
|
||||||
return !isOneOf(o.title, appliedBlackList);
|
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
|
||||||
|
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
|
||||||
|
return titleNotBlacklisted && descNotBlacklisted;
|
||||||
}
|
}
|
||||||
/** @type {ProviderConfig} */
|
/** @type {ProviderConfig} */
|
||||||
const config = {
|
const config = {
|
||||||
|
|||||||
@@ -196,8 +196,8 @@ const config = {
|
|||||||
id: '.aditem@data-adid',
|
id: '.aditem@data-adid',
|
||||||
price: '.aditem-main--middle--price-shipping--price | removeNewline | trim',
|
price: '.aditem-main--middle--price-shipping--price | removeNewline | trim',
|
||||||
tags: '.aditem-main--middle--tags | removeNewline | trim',
|
tags: '.aditem-main--middle--tags | removeNewline | trim',
|
||||||
title: '.aditem-main .text-module-begin a | removeNewline | trim',
|
title: '.aditem-main .text-module-begin | removeNewline | trim',
|
||||||
link: '.aditem-main .text-module-begin a@href | removeNewline | trim',
|
link: '.aditem@data-href',
|
||||||
description: '.aditem-main .aditem-main--middle--description | removeNewline | trim',
|
description: '.aditem-main .aditem-main--middle--description | removeNewline | trim',
|
||||||
address: '.aditem-main--top--left | trim | removeNewline',
|
address: '.aditem-main--top--left | trim | removeNewline',
|
||||||
image: 'img@src',
|
image: 'img@src',
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ function normalize(o) {
|
|||||||
const originalId = o.id.split('/').pop();
|
const originalId = o.id.split('/').pop();
|
||||||
const id = buildHash(originalId, o.price);
|
const id = buildHash(originalId, o.price);
|
||||||
const link = o.link != null ? `https://www.mcmakler.de${o.link}` : o.link;
|
const link = o.link != null ? `https://www.mcmakler.de${o.link}` : o.link;
|
||||||
const [rooms, size] = o.tags.split(' | ');
|
const [rooms, size] = (o.tags || '').split(' | ');
|
||||||
const address = o.address?.replace(' / ', ' ') || null;
|
const address = o.address?.replace(' / ', ' ') || null;
|
||||||
return {
|
return {
|
||||||
id,
|
id,
|
||||||
|
|||||||
@@ -42,7 +42,9 @@ function normalize(o) {
|
|||||||
* @returns {boolean}
|
* @returns {boolean}
|
||||||
*/
|
*/
|
||||||
function applyBlacklist(o) {
|
function applyBlacklist(o) {
|
||||||
return !isOneOf(o.title, appliedBlackList);
|
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
|
||||||
|
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
|
||||||
|
return titleNotBlacklisted && descNotBlacklisted;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @type {ProviderConfig} */
|
/** @type {ProviderConfig} */
|
||||||
|
|||||||
@@ -21,7 +21,8 @@ function normalize(o) {
|
|||||||
const link = o.link != null ? decodeURIComponent(o.link) : config.url;
|
const link = o.link != null ? decodeURIComponent(o.link) : config.url;
|
||||||
|
|
||||||
const urlReg = new RegExp(/url\((.*?)\)/gim);
|
const urlReg = new RegExp(/url\((.*?)\)/gim);
|
||||||
const image = o.image != null ? urlReg.exec(o.image)[1] : null;
|
const imageMatch = o.image != null ? urlReg.exec(o.image) : null;
|
||||||
|
const image = imageMatch != null ? imageMatch[1] : null;
|
||||||
return {
|
return {
|
||||||
id,
|
id,
|
||||||
link,
|
link,
|
||||||
|
|||||||
@@ -44,6 +44,7 @@ function normalize(o) {
|
|||||||
const link = `https://www.wg-gesucht.de${o.link}`;
|
const link = `https://www.wg-gesucht.de${o.link}`;
|
||||||
const image = o.image != null ? o.image.replace('small', 'large') : null;
|
const image = o.image != null ? o.image.replace('small', 'large') : null;
|
||||||
const [rooms, city, road] = o.details?.split(' | ') || [];
|
const [rooms, city, road] = o.details?.split(' | ') || [];
|
||||||
|
const address = [city, road].filter(Boolean).join(', ') || null;
|
||||||
return {
|
return {
|
||||||
id,
|
id,
|
||||||
link,
|
link,
|
||||||
@@ -51,7 +52,7 @@ function normalize(o) {
|
|||||||
price: extractNumber(o.price),
|
price: extractNumber(o.price),
|
||||||
size: extractNumber(o.size),
|
size: extractNumber(o.size),
|
||||||
rooms: extractNumber(rooms),
|
rooms: extractNumber(rooms),
|
||||||
address: `${city}, ${road}`,
|
address,
|
||||||
image,
|
image,
|
||||||
description: o.description,
|
description: o.description,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ function normalize(o) {
|
|||||||
const [city = '', part = ''] = (o.description || '').split('-').map((v) => v.trim());
|
const [city = '', part = ''] = (o.description || '').split('-').map((v) => v.trim());
|
||||||
const address = `${part}, ${city}`;
|
const address = `${part}, ${city}`;
|
||||||
return {
|
return {
|
||||||
id: o.link.split('/').pop(),
|
id: o.link != null ? o.link.split('/').pop() : null,
|
||||||
link: o.link,
|
link: o.link,
|
||||||
title: o.title || '',
|
title: o.title || '',
|
||||||
price: extractNumber(o.price),
|
price: extractNumber(o.price),
|
||||||
@@ -38,7 +38,7 @@ function normalize(o) {
|
|||||||
function applyBlacklist(o) {
|
function applyBlacklist(o) {
|
||||||
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||||
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||||
return o.id != null && o.title != null && titleNotBlacklisted && descNotBlacklisted && o.link.startsWith(o.link);
|
return o.id != null && o.title != null && o.link != null && titleNotBlacklisted && descNotBlacklisted;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @type {ProviderConfig} */
|
/** @type {ProviderConfig} */
|
||||||
|
|||||||
263
lib/services/debug/debugBundleService.js
Normal file
263
lib/services/debug/debugBundleService.js
Normal file
@@ -0,0 +1,263 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (c) 2026 by Christian Kellner.
|
||||||
|
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||||
|
*/
|
||||||
|
|
||||||
|
import fs from 'fs';
|
||||||
|
import os from 'os';
|
||||||
|
import { getAllDebugLogs } from './debugLogStorage.js';
|
||||||
|
import { getPackageVersion } from '../../utils.js';
|
||||||
|
|
||||||
|
const LOGS_FILE_NAME = 'logs.txt';
|
||||||
|
const SYSTEM_INFO_FILE_NAME = 'sys.txt';
|
||||||
|
const DEBUG_FILE_PREFIX = 'FredyDebug-';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Lazily resolve AdmZip via dynamic import so tests can swap it via globalThis.
|
||||||
|
* Mirrors the pattern used by backupRestoreService.js for consistency.
|
||||||
|
* @returns {Promise<any>}
|
||||||
|
*/
|
||||||
|
let _AdmZipSingleton = null;
|
||||||
|
async function getAdmZip() {
|
||||||
|
if (_AdmZipSingleton) return _AdmZipSingleton;
|
||||||
|
if (globalThis && globalThis.__TEST_ADM_ZIP__) {
|
||||||
|
_AdmZipSingleton = globalThis.__TEST_ADM_ZIP__;
|
||||||
|
return _AdmZipSingleton;
|
||||||
|
}
|
||||||
|
const mod = await import('adm-zip');
|
||||||
|
_AdmZipSingleton = (mod && mod.default) || mod;
|
||||||
|
return _AdmZipSingleton;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Format a Date as YYYY-MM-DD using local time. Used for the download filename.
|
||||||
|
* @param {Date} date
|
||||||
|
* @returns {string}
|
||||||
|
*/
|
||||||
|
function formatDateOnly(date) {
|
||||||
|
const yyyy = date.getFullYear();
|
||||||
|
const mm = String(date.getMonth() + 1).padStart(2, '0');
|
||||||
|
const dd = String(date.getDate()).padStart(2, '0');
|
||||||
|
return `${yyyy}-${mm}-${dd}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build the debug bundle filename, e.g. "2026-06-08-FredyDebug-22.5.0.zip".
|
||||||
|
* @returns {Promise<string>}
|
||||||
|
*/
|
||||||
|
export async function buildDebugBundleFileName() {
|
||||||
|
const version = await getPackageVersion();
|
||||||
|
return `${formatDateOnly(new Date())}-${DEBUG_FILE_PREFIX}${version}.zip`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Format a stored debug_logs row into a single text line. The format mirrors the
|
||||||
|
* console layout from logger.js so support staff sees familiar output:
|
||||||
|
* [YYYY-MM-DD HH:MM:SS] LEVEL: message
|
||||||
|
*
|
||||||
|
* @param {{ts:number, level:string, message:string}} row
|
||||||
|
* @returns {string}
|
||||||
|
*/
|
||||||
|
function formatLogLine(row) {
|
||||||
|
const d = new Date(row.ts);
|
||||||
|
const yyyy = d.getFullYear();
|
||||||
|
const mm = String(d.getMonth() + 1).padStart(2, '0');
|
||||||
|
const dd = String(d.getDate()).padStart(2, '0');
|
||||||
|
const hh = String(d.getHours()).padStart(2, '0');
|
||||||
|
const mi = String(d.getMinutes()).padStart(2, '0');
|
||||||
|
const ss = String(d.getSeconds()).padStart(2, '0');
|
||||||
|
const level = String(row.level || 'info').toUpperCase();
|
||||||
|
return `[${yyyy}-${mm}-${dd} ${hh}:${mi}:${ss}] ${level}: ${row.message}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Render every stored debug log row into a single newline-delimited text blob.
|
||||||
|
* Returns an empty string when there are no rows.
|
||||||
|
*
|
||||||
|
* @returns {string}
|
||||||
|
*/
|
||||||
|
export function renderLogsTxt() {
|
||||||
|
const rows = getAllDebugLogs();
|
||||||
|
if (!rows || rows.length === 0) return '';
|
||||||
|
return rows.map(formatLogLine).join('\n') + '\n';
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Best-effort Docker detection. Used as a context hint in sys.txt so issue triage
|
||||||
|
* knows whether the user runs the official container image.
|
||||||
|
*
|
||||||
|
* @returns {{inDocker:boolean, evidence:string[]}}
|
||||||
|
*/
|
||||||
|
function detectDocker() {
|
||||||
|
const evidence = [];
|
||||||
|
let inDocker = false;
|
||||||
|
|
||||||
|
if (process.env.FREDY_IN_DOCKER === 'true' || process.env.FREDY_IN_DOCKER === '1') {
|
||||||
|
inDocker = true;
|
||||||
|
evidence.push('FREDY_IN_DOCKER env var is set');
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
if (fs.existsSync('/.dockerenv')) {
|
||||||
|
inDocker = true;
|
||||||
|
evidence.push('/.dockerenv exists');
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
if (fs.existsSync('/proc/1/cgroup')) {
|
||||||
|
const cgroup = fs.readFileSync('/proc/1/cgroup', 'utf-8');
|
||||||
|
if (/docker|containerd|kubepods/i.test(cgroup)) {
|
||||||
|
inDocker = true;
|
||||||
|
evidence.push('/proc/1/cgroup mentions a container runtime');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
return { inDocker, evidence };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Strip credentials from URL-like strings so they can safely appear in sys.txt.
|
||||||
|
* Returns the input unchanged for non-URL values.
|
||||||
|
* @param {string} value
|
||||||
|
* @returns {string}
|
||||||
|
*/
|
||||||
|
function sanitizeUrlLike(value) {
|
||||||
|
if (typeof value !== 'string' || value.length === 0) return value;
|
||||||
|
try {
|
||||||
|
const u = new URL(value);
|
||||||
|
if (u.username || u.password) {
|
||||||
|
u.username = '***';
|
||||||
|
u.password = '***';
|
||||||
|
}
|
||||||
|
return u.toString();
|
||||||
|
} catch {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatBytes(bytes) {
|
||||||
|
if (!Number.isFinite(bytes)) return String(bytes);
|
||||||
|
if (bytes < 1024) return `${bytes} B`;
|
||||||
|
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(2)} KiB`;
|
||||||
|
if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(2)} MiB`;
|
||||||
|
return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GiB`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatDuration(seconds) {
|
||||||
|
if (!Number.isFinite(seconds)) return String(seconds);
|
||||||
|
const s = Math.floor(seconds);
|
||||||
|
const days = Math.floor(s / 86400);
|
||||||
|
const hours = Math.floor((s % 86400) / 3600);
|
||||||
|
const minutes = Math.floor((s % 3600) / 60);
|
||||||
|
const secs = s % 60;
|
||||||
|
return `${days}d ${hours}h ${minutes}m ${secs}s`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build a plaintext system / runtime report for inclusion in the debug zip. Settings
|
||||||
|
* are sanitized, proxy URL credentials and session secrets are stripped before
|
||||||
|
* serialization.
|
||||||
|
*
|
||||||
|
* @param {object} [options]
|
||||||
|
* @param {object} [options.settings]
|
||||||
|
* @returns {Promise<string>}
|
||||||
|
*/
|
||||||
|
export async function buildSystemInfo({ settings = null } = {}) {
|
||||||
|
const fredyVersion = await getPackageVersion();
|
||||||
|
const docker = detectDocker();
|
||||||
|
const cpus = os.cpus() || [];
|
||||||
|
const procMem = process.memoryUsage();
|
||||||
|
|
||||||
|
const lines = [];
|
||||||
|
lines.push('# Fredy Debug Report');
|
||||||
|
lines.push(`Generated at: ${new Date().toISOString()}`);
|
||||||
|
lines.push('');
|
||||||
|
|
||||||
|
lines.push('## Application');
|
||||||
|
lines.push(`Fredy version: ${fredyVersion}`);
|
||||||
|
lines.push(`Node.js version: ${process.version}`);
|
||||||
|
lines.push(`Process uptime: ${formatDuration(process.uptime())}`);
|
||||||
|
lines.push(`PID: ${process.pid}`);
|
||||||
|
lines.push(`Env (NODE_ENV): ${process.env.NODE_ENV || 'development'}`);
|
||||||
|
lines.push('');
|
||||||
|
|
||||||
|
lines.push('## Operating System');
|
||||||
|
lines.push(`Platform: ${process.platform}`);
|
||||||
|
lines.push(`Architecture: ${process.arch}`);
|
||||||
|
lines.push(`OS type: ${os.type()}`);
|
||||||
|
lines.push(`OS release: ${os.release()}`);
|
||||||
|
lines.push(`OS version: ${typeof os.version === 'function' ? os.version() : 'n/a'}`);
|
||||||
|
lines.push(`Hostname: ${os.hostname()}`);
|
||||||
|
lines.push(`System uptime: ${formatDuration(os.uptime())}`);
|
||||||
|
lines.push('');
|
||||||
|
|
||||||
|
lines.push('## Container');
|
||||||
|
lines.push(`Running in Docker: ${docker.inDocker ? 'yes' : 'no'}`);
|
||||||
|
if (docker.evidence.length > 0) {
|
||||||
|
lines.push(`Evidence: ${docker.evidence.join('; ')}`);
|
||||||
|
}
|
||||||
|
if (process.env.FREDY_IMAGE_TAG) {
|
||||||
|
lines.push(`Image tag: ${process.env.FREDY_IMAGE_TAG}`);
|
||||||
|
}
|
||||||
|
lines.push('');
|
||||||
|
|
||||||
|
lines.push('## Hardware');
|
||||||
|
lines.push(`CPU count: ${cpus.length}`);
|
||||||
|
lines.push(`CPU model: ${cpus[0]?.model || 'unknown'}`);
|
||||||
|
lines.push(`Total memory: ${formatBytes(os.totalmem())}`);
|
||||||
|
lines.push(`Free memory: ${formatBytes(os.freemem())}`);
|
||||||
|
lines.push(`Process RSS: ${formatBytes(procMem.rss)}`);
|
||||||
|
lines.push(`Process heapUsed: ${formatBytes(procMem.heapUsed)}`);
|
||||||
|
lines.push(`Process heapTotal: ${formatBytes(procMem.heapTotal)}`);
|
||||||
|
lines.push('');
|
||||||
|
|
||||||
|
if (settings && typeof settings === 'object') {
|
||||||
|
lines.push('## Settings (sanitized)');
|
||||||
|
const safe = { ...settings };
|
||||||
|
if (safe.proxyUrl) safe.proxyUrl = sanitizeUrlLike(safe.proxyUrl);
|
||||||
|
delete safe.session_secret;
|
||||||
|
delete safe.sessionSecret;
|
||||||
|
for (const [key, value] of Object.entries(safe)) {
|
||||||
|
let printed;
|
||||||
|
if (value == null) {
|
||||||
|
printed = 'null';
|
||||||
|
} else if (typeof value === 'object') {
|
||||||
|
try {
|
||||||
|
printed = JSON.stringify(value);
|
||||||
|
} catch {
|
||||||
|
printed = String(value);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
printed = String(value);
|
||||||
|
}
|
||||||
|
lines.push(`${key}: ${printed}`);
|
||||||
|
}
|
||||||
|
lines.push('');
|
||||||
|
}
|
||||||
|
|
||||||
|
return lines.join('\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build the final debug bundle zip buffer (logs.txt + sys.txt). The caller is
|
||||||
|
* responsible for checking wasEverEnabled() before invoking this, we still produce
|
||||||
|
* a valid zip even when there are zero log rows (logs.txt will contain a placeholder)
|
||||||
|
* because the route layer handles the user-friendly 409 case.
|
||||||
|
*
|
||||||
|
* @param {object} [options]
|
||||||
|
* @param {object} [options.settings] Runtime settings to embed in sys.txt.
|
||||||
|
* @returns {Promise<Buffer>}
|
||||||
|
*/
|
||||||
|
export async function buildDebugBundleZip({ settings = null } = {}) {
|
||||||
|
const logsContent = renderLogsTxt() || 'No debug log entries are currently stored.\n';
|
||||||
|
const sysContent = await buildSystemInfo({ settings });
|
||||||
|
|
||||||
|
const AdmZip = await getAdmZip();
|
||||||
|
const zip = new AdmZip();
|
||||||
|
zip.addFile(LOGS_FILE_NAME, Buffer.from(logsContent, 'utf-8'));
|
||||||
|
zip.addFile(SYSTEM_INFO_FILE_NAME, Buffer.from(sysContent, 'utf-8'));
|
||||||
|
return zip.toBuffer();
|
||||||
|
}
|
||||||
346
lib/services/debug/debugLogStorage.js
Normal file
346
lib/services/debug/debugLogStorage.js
Normal file
@@ -0,0 +1,346 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (c) 2026 by Christian Kellner.
|
||||||
|
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||||
|
*/
|
||||||
|
|
||||||
|
import SqliteConnection from '../storage/SqliteConnection.js';
|
||||||
|
import { upsertSettings, getSettings } from '../storage/settingsStorage.js';
|
||||||
|
import logger from '../logger.js';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Hard cap on the total UTF-8 byte length of stored log MESSAGES (5 MiB).
|
||||||
|
*
|
||||||
|
* Note: this measures the payload bytes (message strings only); SQLite per-row
|
||||||
|
* overhead (id, ts, level, byte_size columns + page housekeeping) means the actual
|
||||||
|
* sqlite_master page count for debug_logs can be larger than this cap by a constant
|
||||||
|
* factor. The cap is intentionally about user-visible payload to keep the math
|
||||||
|
* predictable and to align with what ends up in logs.txt.
|
||||||
|
*
|
||||||
|
* The cap is enforced via a rolling buffer: when the live size exceeds it, the
|
||||||
|
* oldest rows are removed until the size falls below the limit again.
|
||||||
|
* @type {number}
|
||||||
|
*/
|
||||||
|
export const MAX_DEBUG_LOG_BYTES = 5 * 1024 * 1024;
|
||||||
|
|
||||||
|
/** Settings key persisting the active on/off flag. */
|
||||||
|
const SETTING_ENABLED = 'debug_logging_enabled';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Settings key persisting "this feature has been turned on at least once". Used to
|
||||||
|
* decide whether the download endpoint returns 409 (never enabled) or whether the
|
||||||
|
* "delete previous logs?" confirm dialog should be shown on (re)enable.
|
||||||
|
*/
|
||||||
|
const SETTING_EVER_ENABLED = 'debug_logging_ever_enabled';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Cached live byte size of all rows in debug_logs. Initialized lazily from the DB on
|
||||||
|
* the first call and kept in sync by append / clear / trim. Storing this in-memory
|
||||||
|
* avoids running SUM() on every single insert (logger writes can be very frequent).
|
||||||
|
* @type {number|null}
|
||||||
|
*/
|
||||||
|
let cachedSize = null;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Cached value of debug_logging_enabled. Reflects DB state; flipped by enable() /
|
||||||
|
* disable() so the logger hot-path does not have to hit the settings cache for every
|
||||||
|
* log line.
|
||||||
|
* @type {boolean|null}
|
||||||
|
*/
|
||||||
|
let cachedEnabled = null;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compute the UTF-8 byte length of a string. Falls back to character count for
|
||||||
|
* environments where Buffer is not available (vitest covers Node, so it always is).
|
||||||
|
* @param {string} str
|
||||||
|
* @returns {number}
|
||||||
|
*/
|
||||||
|
function byteLengthOf(str) {
|
||||||
|
if (typeof str !== 'string') return 0;
|
||||||
|
if (typeof Buffer !== 'undefined' && typeof Buffer.byteLength === 'function') {
|
||||||
|
return Buffer.byteLength(str, 'utf-8');
|
||||||
|
}
|
||||||
|
return str.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read the current total byte size from the DB and update the local cache.
|
||||||
|
* @returns {number}
|
||||||
|
*/
|
||||||
|
function refreshSizeFromDb() {
|
||||||
|
const rows = SqliteConnection.query('SELECT COALESCE(SUM(byte_size), 0) AS total FROM debug_logs');
|
||||||
|
cachedSize = Number(rows?.[0]?.total ?? 0);
|
||||||
|
return cachedSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Lazily ensure the cached enabled/size values are up to date. Called by every public
|
||||||
|
* method that needs to know either value, so external init is not required.
|
||||||
|
* @returns {Promise<void>}
|
||||||
|
*/
|
||||||
|
async function ensureCachesInitialized() {
|
||||||
|
if (cachedEnabled == null) {
|
||||||
|
const settings = await getSettings();
|
||||||
|
cachedEnabled = settings[SETTING_ENABLED] === true;
|
||||||
|
}
|
||||||
|
if (cachedSize == null) {
|
||||||
|
refreshSizeFromDb();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Cached prepared statements used by trimToFit(). Initialized on first use so we do
|
||||||
|
* not pay the prepare cost on every overflow event, and skipped entirely when the
|
||||||
|
* feature is never activated.
|
||||||
|
* @type {{select:any, del:any}|null}
|
||||||
|
*/
|
||||||
|
let trimStatements = null;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Drop the oldest rows from debug_logs until the cached size falls below
|
||||||
|
* MAX_DEBUG_LOG_BYTES. Implements the rolling buffer behavior chosen for the feature.
|
||||||
|
*
|
||||||
|
* The deletion is performed in batches of up to 100 oldest rows wrapped in a single
|
||||||
|
* transaction. The size cache is updated only after the transaction commits, so a
|
||||||
|
* mid-batch failure (rolled back by SQLite) cannot leave cachedSize out of sync with
|
||||||
|
* the on-disk reality. A defensive resync via SUM() is performed on transaction
|
||||||
|
* failure to recover from any unexpected drift.
|
||||||
|
*
|
||||||
|
* @returns {void}
|
||||||
|
*/
|
||||||
|
function trimToFit() {
|
||||||
|
if (cachedSize == null || cachedSize <= MAX_DEBUG_LOG_BYTES) return;
|
||||||
|
|
||||||
|
const db = SqliteConnection.getConnection();
|
||||||
|
if (trimStatements == null) {
|
||||||
|
trimStatements = {
|
||||||
|
select: db.prepare('SELECT id, byte_size FROM debug_logs ORDER BY id ASC LIMIT 100'),
|
||||||
|
del: db.prepare('DELETE FROM debug_logs WHERE id = @id'),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
while (cachedSize > MAX_DEBUG_LOG_BYTES) {
|
||||||
|
const oldest = trimStatements.select.all();
|
||||||
|
if (oldest.length === 0) {
|
||||||
|
// Table is empty but the cache still claims we are over the cap. That can only
|
||||||
|
// happen if cachedSize drifted (e.g. external DB modification, zero-byte
|
||||||
|
// messages that never contributed to SUM(byte_size), or a previous trim that
|
||||||
|
// partially succeeded). Resync from the source of truth and bail out.
|
||||||
|
refreshSizeFromDb();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Pick exactly enough oldest rows to bring the cache back under the cap. We do
|
||||||
|
// NOT delete the entire 100-row batch unconditionally, that would over-trim in
|
||||||
|
// edge cases where just one or two rows are enough.
|
||||||
|
const needToFree = cachedSize - MAX_DEBUG_LOG_BYTES;
|
||||||
|
let freed = 0;
|
||||||
|
const idsToDelete = [];
|
||||||
|
for (const row of oldest) {
|
||||||
|
idsToDelete.push(row.id);
|
||||||
|
freed += Number(row.byte_size) || 0;
|
||||||
|
if (freed >= needToFree) break;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const tx = db.transaction((ids) => {
|
||||||
|
for (const id of ids) {
|
||||||
|
trimStatements.del.run({ id });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
tx(idsToDelete);
|
||||||
|
// Only decrement after the transaction has committed; a mid-batch failure
|
||||||
|
// would roll the DELETEs back and leave cachedSize untouched.
|
||||||
|
cachedSize -= freed;
|
||||||
|
if (freed === 0) {
|
||||||
|
// We deleted rows but they all had byte_size <= 0, so cachedSize did not
|
||||||
|
// move. Without intervention the outer loop would spin again with the same
|
||||||
|
// condition. Resync from the DB and bail to prevent that.
|
||||||
|
refreshSizeFromDb();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// SQLite rolled the batch back; resync cachedSize from the DB to recover from
|
||||||
|
// any unexpected drift, then bail out so we do not spin forever on a persistent
|
||||||
|
// failure (e.g. database is locked or read-only).
|
||||||
|
refreshSizeFromDb();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (cachedSize < 0) cachedSize = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Whether debug logging is currently enabled. Synchronous and cheap so the logger
|
||||||
|
* hot-path can call it on every log line.
|
||||||
|
*
|
||||||
|
* @returns {boolean} True if logs should be persisted to the debug_logs table.
|
||||||
|
*/
|
||||||
|
export function isEnabled() {
|
||||||
|
return cachedEnabled === true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Append a single log entry to debug_logs (if enabled) and trim the rolling buffer if
|
||||||
|
* the new row pushes the live size above the cap.
|
||||||
|
*
|
||||||
|
* Safe to call even when logging is disabled, it becomes a no-op. Any storage error
|
||||||
|
* is swallowed so the logger never breaks the calling code; bookkeeping for cachedSize
|
||||||
|
* stays consistent because we update it only after a successful insert.
|
||||||
|
*
|
||||||
|
* @param {{ts:number, level:string, message:string}} entry
|
||||||
|
* @returns {void}
|
||||||
|
*/
|
||||||
|
export function appendLogEntry(entry) {
|
||||||
|
if (!isEnabled()) return;
|
||||||
|
if (!entry || typeof entry.message !== 'string') return;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const ts = Number.isFinite(entry.ts) ? entry.ts : Date.now();
|
||||||
|
const level = String(entry.level || 'info');
|
||||||
|
const message = entry.message;
|
||||||
|
const byte_size = byteLengthOf(message);
|
||||||
|
|
||||||
|
SqliteConnection.execute(
|
||||||
|
'INSERT INTO debug_logs (ts, level, message, byte_size) VALUES (@ts, @level, @message, @byte_size)',
|
||||||
|
{ ts, level, message, byte_size },
|
||||||
|
);
|
||||||
|
|
||||||
|
if (cachedSize == null) {
|
||||||
|
refreshSizeFromDb();
|
||||||
|
} else {
|
||||||
|
cachedSize += byte_size;
|
||||||
|
}
|
||||||
|
trimToFit();
|
||||||
|
} catch {
|
||||||
|
// Logging must never break the application. Swallow storage errors silently.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Remove every row from debug_logs and reset the cached size to zero. Used by both
|
||||||
|
* the "clear previous logs" path on (re)enable and by explicit clear actions.
|
||||||
|
*
|
||||||
|
* @returns {void}
|
||||||
|
*/
|
||||||
|
export function clearAllDebugLogs() {
|
||||||
|
SqliteConnection.execute('DELETE FROM debug_logs');
|
||||||
|
cachedSize = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return the cached live byte size of the debug_logs table contents.
|
||||||
|
* @returns {Promise<number>}
|
||||||
|
*/
|
||||||
|
export async function getCurrentSize() {
|
||||||
|
await ensureCachesInitialized();
|
||||||
|
return cachedSize ?? 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return the configured maximum size for the debug_logs table.
|
||||||
|
* @returns {number}
|
||||||
|
*/
|
||||||
|
export function getMaxSize() {
|
||||||
|
return MAX_DEBUG_LOG_BYTES;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check whether the debug_logs table contains at least one row.
|
||||||
|
* @returns {boolean}
|
||||||
|
*/
|
||||||
|
export function hasAnyLogs() {
|
||||||
|
const row = SqliteConnection.query('SELECT 1 AS one FROM debug_logs LIMIT 1');
|
||||||
|
return Array.isArray(row) && row.length > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Has debug logging ever been enabled in this installation? Used by the download
|
||||||
|
* endpoint to distinguish "no logs yet" (empty table) from "feature never used"
|
||||||
|
* (which returns 409 to surface a friendlier UI error).
|
||||||
|
*
|
||||||
|
* @returns {Promise<boolean>}
|
||||||
|
*/
|
||||||
|
export async function wasEverEnabled() {
|
||||||
|
const settings = await getSettings();
|
||||||
|
return settings[SETTING_EVER_ENABLED] === true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Turn debug logging on. Persists both the active flag and the "ever enabled" flag,
|
||||||
|
* optionally clearing previous logs when the caller passes clearPrevious=true (this
|
||||||
|
* is the path taken when the UI confirm dialog "Delete previous logs?" is accepted).
|
||||||
|
*
|
||||||
|
* @param {object} [options]
|
||||||
|
* @param {boolean} [options.clearPrevious=false]
|
||||||
|
* @returns {Promise<void>}
|
||||||
|
*/
|
||||||
|
export async function enableDebugLogging({ clearPrevious = false } = {}) {
|
||||||
|
if (clearPrevious) {
|
||||||
|
clearAllDebugLogs();
|
||||||
|
}
|
||||||
|
upsertSettings({ [SETTING_ENABLED]: true, [SETTING_EVER_ENABLED]: true });
|
||||||
|
cachedEnabled = true;
|
||||||
|
if (cachedSize == null) {
|
||||||
|
refreshSizeFromDb();
|
||||||
|
}
|
||||||
|
// Attach the logger sink only while recording is on so the logger hot path pays
|
||||||
|
// no per-call cost (Date.now + stringifyArgs) when nobody enabled the feature.
|
||||||
|
logger.setDebugLogSink(appendLogEntry);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Turn debug logging off. Previous logs are kept on disk so the user can still
|
||||||
|
* download them; they are only cleared when the user re-enables and chooses "delete
|
||||||
|
* previous logs".
|
||||||
|
*
|
||||||
|
* @returns {Promise<void>}
|
||||||
|
*/
|
||||||
|
export async function disableDebugLogging() {
|
||||||
|
upsertSettings({ [SETTING_ENABLED]: false });
|
||||||
|
cachedEnabled = false;
|
||||||
|
// Detach the sink so the logger hot path returns immediately on its `if (sink)`
|
||||||
|
// check instead of paying the no-op cost on every log line.
|
||||||
|
logger.setDebugLogSink(null);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return all stored log entries ordered chronologically. Used by the bundle builder
|
||||||
|
* when assembling logs.txt.
|
||||||
|
*
|
||||||
|
* @returns {{id:number, ts:number, level:string, message:string}[]}
|
||||||
|
*/
|
||||||
|
export function getAllDebugLogs() {
|
||||||
|
return SqliteConnection.query('SELECT id, ts, level, message FROM debug_logs ORDER BY id ASC');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reload the cached enabled flag from settings storage. Called from the logger at
|
||||||
|
* startup so the cache reflects the persisted state after a Fredy restart.
|
||||||
|
*
|
||||||
|
* @returns {Promise<boolean>} The active enabled flag.
|
||||||
|
*/
|
||||||
|
export async function reloadEnabledFromSettings() {
|
||||||
|
const settings = await getSettings();
|
||||||
|
cachedEnabled = settings[SETTING_ENABLED] === true;
|
||||||
|
// (Un)wire the sink to match the persisted state. Note: startup work that runs
|
||||||
|
// before index.js calls this (CloakBrowser binary check, runMigrations) still
|
||||||
|
// logs to stdout only, since the sink is not attached yet at that point.
|
||||||
|
if (cachedEnabled) {
|
||||||
|
logger.setDebugLogSink(appendLogEntry);
|
||||||
|
} else {
|
||||||
|
logger.setDebugLogSink(null);
|
||||||
|
}
|
||||||
|
return cachedEnabled;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test-only helper to drop in-memory caches between unit tests. Resets every piece
|
||||||
|
* of module-scoped mutable state so a test that swaps the underlying DB does not
|
||||||
|
* inherit stale prepared statements from a previous run.
|
||||||
|
* @returns {void}
|
||||||
|
*/
|
||||||
|
export function _resetForTests() {
|
||||||
|
cachedSize = null;
|
||||||
|
cachedEnabled = null;
|
||||||
|
trimStatements = null;
|
||||||
|
}
|
||||||
@@ -4,7 +4,7 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
import { launch } from 'cloakbrowser/puppeteer';
|
import { launch } from 'cloakbrowser/puppeteer';
|
||||||
import { debug, botDetected } from './utils.js';
|
import { botDetected, debug } from './utils.js';
|
||||||
import { getPreLaunchConfig } from './botPrevention.js';
|
import { getPreLaunchConfig } from './botPrevention.js';
|
||||||
import logger from '../logger.js';
|
import logger from '../logger.js';
|
||||||
import { trackPoi } from '../tracking/Tracker.js';
|
import { trackPoi } from '../tracking/Tracker.js';
|
||||||
@@ -50,7 +50,7 @@ export async function launchBrowser(url, options) {
|
|||||||
preCfg.windowSizeArg,
|
preCfg.windowSizeArg,
|
||||||
];
|
];
|
||||||
|
|
||||||
const browser = await launch({
|
return await launch({
|
||||||
headless: options?.puppeteerHeadless ?? true,
|
headless: options?.puppeteerHeadless ?? true,
|
||||||
humanize: true,
|
humanize: true,
|
||||||
args,
|
args,
|
||||||
@@ -59,8 +59,6 @@ export async function launchBrowser(url, options) {
|
|||||||
...(options?.proxyUrl ? { proxy: options.proxyUrl } : {}),
|
...(options?.proxyUrl ? { proxy: options.proxyUrl } : {}),
|
||||||
...(preCfg.timezone ? { timezone: preCfg.timezone } : {}),
|
...(preCfg.timezone ? { timezone: preCfg.timezone } : {}),
|
||||||
});
|
});
|
||||||
|
|
||||||
return browser;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -103,6 +103,13 @@ const EQUIPMENT_MAP = {
|
|||||||
lodgerflat: 'lodgerflat',
|
lodgerflat: 'lodgerflat',
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// The web UI uses "swapflat", but the mobile API only understands "swap_flat".
|
||||||
|
// An unknown value is not ignored: the API silently returns 0 results for the
|
||||||
|
// whole search. Other values (e.g. "projectlisting") are identical on both APIs.
|
||||||
|
const EXCLUSION_CRITERIA_MAP = {
|
||||||
|
swapflat: 'swap_flat',
|
||||||
|
};
|
||||||
|
|
||||||
const REAL_ESTATE_TYPE = {
|
const REAL_ESTATE_TYPE = {
|
||||||
'haus-mieten': 'houserent',
|
'haus-mieten': 'houserent',
|
||||||
'wohnung-mieten': 'apartmentrent',
|
'wohnung-mieten': 'apartmentrent',
|
||||||
@@ -141,6 +148,43 @@ const WEB_PATH_TO_APARTMENT_EQUIPMENT_MAP = {
|
|||||||
'barrierefreie-wohnung-mieten': { equipment: ['handicappedaccessible'] },
|
'barrierefreie-wohnung-mieten': { equipment: ['handicappedaccessible'] },
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// SEO-optimized rental paths used by the ImmoScout web UI when the user
|
||||||
|
// configures a maximum warmrent. Example: "wohnung-bis-800-euro-warm" means
|
||||||
|
// "apartment for rent up to 800 EUR warmrent". The web UI generates these
|
||||||
|
// paths instead of explicit `price` / `pricetype` query parameters.
|
||||||
|
// Note: only the warmrent variant uses an SEO slug; max coldrent searches
|
||||||
|
// use the regular "wohnung-mieten" path with explicit `price` and
|
||||||
|
// `pricetype=rentpermonth` query params, which the existing translator
|
||||||
|
// already handles.
|
||||||
|
const SEO_RENT_TYPE_TO_REAL_ESTATE_TYPE = {
|
||||||
|
wohnung: 'apartmentrent',
|
||||||
|
haus: 'houserent',
|
||||||
|
};
|
||||||
|
const SEO_MAX_WARMRENT_PATH_PATTERN = /^(?<type>wohnung|haus)-bis-(?<price>\d+)-euro-warm$/;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parses SEO-optimized ImmoScout web paths that encode a maximum warmrent, such
|
||||||
|
* as "wohnung-bis-800-euro-warm". Returns the corresponding mobile API real
|
||||||
|
* estate type and the implicit price/pricetype parameters, or null if the path
|
||||||
|
* does not match the known SEO max-warmrent pattern.
|
||||||
|
*
|
||||||
|
* @param {string} realTypeKey The last segment of the URL path.
|
||||||
|
* @returns {{ realType: string, additionalParams: Record<string, string> } | null}
|
||||||
|
*/
|
||||||
|
function parseSeoMaxWarmrentPath(realTypeKey) {
|
||||||
|
const match = realTypeKey.match(SEO_MAX_WARMRENT_PATH_PATTERN);
|
||||||
|
if (!match) return null;
|
||||||
|
|
||||||
|
const { type, price } = match.groups;
|
||||||
|
return {
|
||||||
|
realType: SEO_RENT_TYPE_TO_REAL_ESTATE_TYPE[type],
|
||||||
|
additionalParams: {
|
||||||
|
price: `-${price}`,
|
||||||
|
pricetype: 'calculatedtotalrent',
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export function convertWebToMobile(webUrl) {
|
export function convertWebToMobile(webUrl) {
|
||||||
let url;
|
let url;
|
||||||
try {
|
try {
|
||||||
@@ -164,7 +208,14 @@ export function convertWebToMobile(webUrl) {
|
|||||||
additionalParamsFromWebPath = WEB_PATH_TO_APARTMENT_EQUIPMENT_MAP[realTypeKey];
|
additionalParamsFromWebPath = WEB_PATH_TO_APARTMENT_EQUIPMENT_MAP[realTypeKey];
|
||||||
realType = REAL_ESTATE_TYPE['wohnung-mieten'];
|
realType = REAL_ESTATE_TYPE['wohnung-mieten'];
|
||||||
} else {
|
} else {
|
||||||
throw new Error(`Real estate type not found: ${realTypeKey}`);
|
// Test for SEO max-warmrent path, e.g. "wohnung-bis-800-euro-warm"
|
||||||
|
const seoMaxWarmrent = parseSeoMaxWarmrentPath(realTypeKey);
|
||||||
|
if (seoMaxWarmrent) {
|
||||||
|
realType = seoMaxWarmrent.realType;
|
||||||
|
additionalParamsFromWebPath = seoMaxWarmrent.additionalParams;
|
||||||
|
} else {
|
||||||
|
throw new Error(`Real estate type not found: ${realTypeKey}`);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -207,6 +258,9 @@ export function convertWebToMobile(webUrl) {
|
|||||||
...(currentEquipmentParams ?? []),
|
...(currentEquipmentParams ?? []),
|
||||||
...items.map((item) => EQUIPMENT_MAP[item.toLowerCase()]).filter(Boolean),
|
...items.map((item) => EQUIPMENT_MAP[item.toLowerCase()]).filter(Boolean),
|
||||||
];
|
];
|
||||||
|
} else if (key === 'exclusioncriteria') {
|
||||||
|
const items = [].concat(val).flatMap((v) => `${v}`.split(','));
|
||||||
|
mobileParams[PARAM_NAME_MAP[key]] = items.map((item) => EXCLUSION_CRITERIA_MAP[item.toLowerCase()] ?? item);
|
||||||
} else {
|
} else {
|
||||||
mobileParams[PARAM_NAME_MAP[key]] = val;
|
mobileParams[PARAM_NAME_MAP[key]] = val;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import * as similarityCache from '../similarity-check/similarityCache.js';
|
|||||||
import { isRunning, markFinished, markRunning } from './run-state.js';
|
import { isRunning, markFinished, markRunning } from './run-state.js';
|
||||||
import { sendToUsers } from '../sse/sse-broker.js';
|
import { sendToUsers } from '../sse/sse-broker.js';
|
||||||
import * as puppeteerExtractor from '../extractor/puppeteerExtractor.js';
|
import * as puppeteerExtractor from '../extractor/puppeteerExtractor.js';
|
||||||
|
import { getSettings } from '../storage/settingsStorage.js';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Initializes the job execution service.
|
* Initializes the job execution service.
|
||||||
@@ -103,15 +104,11 @@ export function initJobExecutionService({ providers, settings, intervalMs }) {
|
|||||||
logger.debug('Working hours set. Skipping as outside of working hours.');
|
logger.debug('Working hours set. Skipping as outside of working hours.');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
settings.lastRun = now;
|
const jobs = jobStorage.getJobs().filter((job) => {
|
||||||
const jobs = jobStorage
|
if (!context) return true; // startup/cron → all
|
||||||
.getJobs()
|
if (context.isAdmin) return true; // admin → all
|
||||||
.filter((job) => job.enabled)
|
return context.userId ? job.userId === context.userId : false; // user → own
|
||||||
.filter((job) => {
|
});
|
||||||
if (!context) return true; // startup/cron → all
|
|
||||||
if (context.isAdmin) return true; // admin → all
|
|
||||||
return context.userId ? job.userId === context.userId : false; // user → own
|
|
||||||
});
|
|
||||||
|
|
||||||
for (const job of jobs) {
|
for (const job of jobs) {
|
||||||
await executeJob(job);
|
await executeJob(job);
|
||||||
@@ -152,6 +149,13 @@ export function initJobExecutionService({ providers, settings, intervalMs }) {
|
|||||||
}
|
}
|
||||||
const acquired = markRunning(job.id);
|
const acquired = markRunning(job.id);
|
||||||
if (!acquired) return;
|
if (!acquired) return;
|
||||||
|
// Persist the trigger time so the dashboard "last search" KPI can be
|
||||||
|
// derived per accessible user without an in-memory cache.
|
||||||
|
try {
|
||||||
|
jobStorage.updateJobLastRunAt(job.id, Date.now());
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn('Failed to persist last_run_at for job', job.id, err);
|
||||||
|
}
|
||||||
// notify listeners (SSE) that the job started
|
// notify listeners (SSE) that the job started
|
||||||
try {
|
try {
|
||||||
bus.emit('jobs:status', { jobId: job.id, running: true });
|
bus.emit('jobs:status', { jobId: job.id, running: true });
|
||||||
@@ -160,6 +164,14 @@ export function initJobExecutionService({ providers, settings, intervalMs }) {
|
|||||||
}
|
}
|
||||||
let browser;
|
let browser;
|
||||||
try {
|
try {
|
||||||
|
// Read the proxy live (not from the startup snapshot) so changing it in the
|
||||||
|
// UI takes effect on the next run without a backend restart. An empty value
|
||||||
|
// disables the proxy. Routing the headless browser through a (German
|
||||||
|
// residential) proxy avoids datacenter-IP based bot detection on the
|
||||||
|
// Puppeteer-based providers (immowelt, immonet, kleinanzeigen, ...).
|
||||||
|
const liveSettings = await getSettings();
|
||||||
|
const proxyUrl = typeof liveSettings?.proxyUrl === 'string' ? liveSettings.proxyUrl.trim() : '';
|
||||||
|
|
||||||
const jobProviders = job.provider.filter(
|
const jobProviders = job.provider.filter(
|
||||||
(p) => providers.find((loaded) => loaded.metaInformation.id === p.id) != null,
|
(p) => providers.find((loaded) => loaded.metaInformation.id === p.id) != null,
|
||||||
);
|
);
|
||||||
@@ -168,14 +180,14 @@ export function initJobExecutionService({ providers, settings, intervalMs }) {
|
|||||||
const matchedProvider = providers.find((loaded) => loaded.metaInformation.id === prov.id);
|
const matchedProvider = providers.find((loaded) => loaded.metaInformation.id === prov.id);
|
||||||
matchedProvider.init({ ...prov, userId: job.userId }, job.blacklist);
|
matchedProvider.init({ ...prov, userId: job.userId }, job.blacklist);
|
||||||
|
|
||||||
if (browser && !browser.isConnected()) {
|
if (browser && !browser.connected) {
|
||||||
logger.debug('Browser is disconnected, nullifying to launch a new one.');
|
logger.debug('Browser is disconnected, nullifying to launch a new one.');
|
||||||
await puppeteerExtractor.closeBrowser(browser);
|
await puppeteerExtractor.closeBrowser(browser);
|
||||||
browser = null;
|
browser = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!browser && matchedProvider.config.getListings == null) {
|
if (!browser && matchedProvider.config.getListings == null) {
|
||||||
browser = await puppeteerExtractor.launchBrowser(matchedProvider.config.url, {});
|
browser = await puppeteerExtractor.launchBrowser(matchedProvider.config.url, proxyUrl ? { proxyUrl } : {});
|
||||||
}
|
}
|
||||||
|
|
||||||
await new FredyPipelineExecutioner(matchedProvider.config, job, prov.id, similarityCache, browser).execute();
|
await new FredyPipelineExecutioner(matchedProvider.config, job, prov.id, similarityCache, browser).execute();
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ import logger from '../../services/logger.js';
|
|||||||
* Concurrency: network-bound checks are executed with a configurable concurrency limit.
|
* Concurrency: network-bound checks are executed with a configurable concurrency limit.
|
||||||
*
|
*
|
||||||
* @param {object} [opts]
|
* @param {object} [opts]
|
||||||
* @param {number} [opts.concurrency=8] Max number of parallel activeTester calls.
|
* @param {number} [opts.concurrency=4] Max number of parallel activeTester calls.
|
||||||
* @returns {Promise<void>}
|
* @returns {Promise<void>}
|
||||||
*/
|
*/
|
||||||
export default async function runActiveChecker(opts = {}) {
|
export default async function runActiveChecker(opts = {}) {
|
||||||
|
|||||||
@@ -17,16 +17,16 @@ const userAgents = [
|
|||||||
];
|
];
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check if a listing is still active with up to 5 attempts and exponential backoff.
|
* Check if a listing is still active with up to `maxAttempts` attempts and exponential backoff.
|
||||||
* Backoff waits are randomized and capped.
|
* Backoff waits are randomized and capped.
|
||||||
*
|
*
|
||||||
* Rules:
|
* Rules:
|
||||||
* - HTTP 200 => return 1 (if checkForText is provided and found, returns 0)
|
* - HTTP 200 => return 1 (if checkForText is provided and found, returns 0)
|
||||||
* - HTTP 401/403 => return -1 (most certainly detected as a bot)
|
* - HTTP 401/403 => return -1 (most certainly detected as a bot)
|
||||||
* - HTTP 404 => return 0
|
* - HTTP 404/410 => return 0
|
||||||
* - Other statuses or network errors => retry until attempts are exhausted
|
* - Other statuses or network errors => retry until attempts are exhausted
|
||||||
*
|
*
|
||||||
* @returns {Promise<Integer>} 1 if active, 0 if not active and -1 if detected as bot
|
* @returns {Promise<number>} 1 if active, 0 if not active and -1 if detected as bot
|
||||||
*/
|
*/
|
||||||
export default async function checkIfListingIsActive(link, checkForText = null) {
|
export default async function checkIfListingIsActive(link, checkForText = null) {
|
||||||
await sleep(randomBetween(50, 100));
|
await sleep(randomBetween(50, 100));
|
||||||
|
|||||||
@@ -14,6 +14,20 @@ const COLORS = {
|
|||||||
const env = process.env.NODE_ENV || 'development';
|
const env = process.env.NODE_ENV || 'development';
|
||||||
const useColor = process.stdout.isTTY || process.stderr.isTTY;
|
const useColor = process.stdout.isTTY || process.stderr.isTTY;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Optional sink that forwards formatted log entries to the opt-in "Debug Logging"
|
||||||
|
* DB storage. Wired and unwired by debugLogStorage as the feature is toggled, so
|
||||||
|
* when nobody enabled the feature this stays null and the logger hot path skips
|
||||||
|
* the Date.now + stringifyArgs work entirely.
|
||||||
|
*
|
||||||
|
* We deliberately do NOT import debugLogStorage here, because that would create a
|
||||||
|
* cycle (debugLogStorage → SqliteConnection → utils → logger → debugLogStorage).
|
||||||
|
* Inversion of control via setDebugLogSink() keeps the dependency one-way.
|
||||||
|
*
|
||||||
|
* @type {((entry:{ts:number, level:string, message:string}) => void)|null}
|
||||||
|
*/
|
||||||
|
let debugLogSink = null;
|
||||||
|
|
||||||
function ts() {
|
function ts() {
|
||||||
const d = new Date();
|
const d = new Date();
|
||||||
const yyyy = d.getFullYear();
|
const yyyy = d.getFullYear();
|
||||||
@@ -31,10 +45,50 @@ function lvl(level) {
|
|||||||
return `${COLORS[level] || ''}${upper}${COLORS.reset}`;
|
return `${COLORS[level] || ''}${upper}${COLORS.reset}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build a colour-free plain text representation of variadic console args. Errors
|
||||||
|
* are unwrapped to their stack/message, objects are JSON-serialized. Used when
|
||||||
|
* forwarding to the DB sink so the stored text is portable across terminals.
|
||||||
|
*
|
||||||
|
* @param {any[]} args
|
||||||
|
* @returns {string}
|
||||||
|
*/
|
||||||
|
function stringifyArgs(args) {
|
||||||
|
return args
|
||||||
|
.map((a) => {
|
||||||
|
if (a == null) return String(a);
|
||||||
|
if (a instanceof Error) return a.stack || a.message;
|
||||||
|
if (typeof a === 'object') {
|
||||||
|
try {
|
||||||
|
return JSON.stringify(a);
|
||||||
|
} catch {
|
||||||
|
return String(a);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return String(a);
|
||||||
|
})
|
||||||
|
.join(' ');
|
||||||
|
}
|
||||||
|
|
||||||
/* eslint-disable no-console */
|
/* eslint-disable no-console */
|
||||||
function log(level, ...args) {
|
function log(level, ...args) {
|
||||||
|
// Forward to the DB sink first (regardless of console suppression rules) so the
|
||||||
|
// recorded debug bundle truly contains every level, including debug entries that
|
||||||
|
// would otherwise be silenced in production.
|
||||||
|
if (debugLogSink) {
|
||||||
|
try {
|
||||||
|
debugLogSink({
|
||||||
|
ts: Date.now(),
|
||||||
|
level,
|
||||||
|
message: `${stringifyArgs(args)}`,
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
// never break the caller because of logging
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (level === 'debug' && env !== 'development') {
|
if (level === 'debug' && env !== 'development') {
|
||||||
return; // Skip debug logs in non-development environments
|
return; // Skip debug logs in non-development environments (console only)
|
||||||
}
|
}
|
||||||
|
|
||||||
const prefix = `[${ts()}] ${lvl(level)}:`;
|
const prefix = `[${ts()}] ${lvl(level)}:`;
|
||||||
@@ -56,9 +110,28 @@ function log(level, ...args) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Register a sink function that receives every log entry the logger sees, regardless
|
||||||
|
* of console suppression rules. debugLogStorage attaches its sink only while the
|
||||||
|
* feature is enabled and detaches it on disable, so the logger's hot path can use
|
||||||
|
* the null check as a cheap on/off gate and skip stringification when off.
|
||||||
|
*
|
||||||
|
* Pass null to remove the sink (used both by the storage module on disable and by
|
||||||
|
* tests to reset state between cases).
|
||||||
|
*
|
||||||
|
* @param {((entry:{ts:number, level:string, message:string}) => void)|null} sink
|
||||||
|
* @returns {void}
|
||||||
|
*/
|
||||||
|
function setDebugLogSink(sink) {
|
||||||
|
debugLogSink = typeof sink === 'function' ? sink : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export { setDebugLogSink };
|
||||||
|
|
||||||
export default {
|
export default {
|
||||||
debug: (...a) => log('debug', ...a),
|
debug: (...a) => log('debug', ...a),
|
||||||
info: (...a) => log('info', ...a),
|
info: (...a) => log('info', ...a),
|
||||||
warn: (...a) => log('warn', ...a),
|
warn: (...a) => log('warn', ...a),
|
||||||
error: (...a) => log('error', ...a),
|
error: (...a) => log('error', ...a),
|
||||||
|
setDebugLogSink,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -40,7 +40,8 @@ class SqliteConnection {
|
|||||||
}
|
}
|
||||||
/**
|
/**
|
||||||
* Returns a singleton instance of better-sqlite3 Database.
|
* Returns a singleton instance of better-sqlite3 Database.
|
||||||
* Respects env var SQLITE_DB_PATH and defaults to db/listings.db.
|
* Uses the configured `sqlitepath` (from conf/config.json) as the directory,
|
||||||
|
* defaulting to `/db` (relative to the project root) when unset.
|
||||||
*/
|
*/
|
||||||
static getConnection() {
|
static getConnection() {
|
||||||
if (this.#db) return this.#db;
|
if (this.#db) return this.#db;
|
||||||
|
|||||||
@@ -97,6 +97,7 @@ export const getJob = (jobId) => {
|
|||||||
j.notification_adapter AS notificationAdapter,
|
j.notification_adapter AS notificationAdapter,
|
||||||
j.spatial_filter AS spatialFilter,
|
j.spatial_filter AS spatialFilter,
|
||||||
j.spec_filter AS specFilter,
|
j.spec_filter AS specFilter,
|
||||||
|
j.last_run_at AS lastRunAt,
|
||||||
(SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id AND l.is_active = 1 AND l.manually_deleted = 0) AS numberOfFoundListings
|
(SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id AND l.is_active = 1 AND l.manually_deleted = 0) AS numberOfFoundListings
|
||||||
FROM jobs j
|
FROM jobs j
|
||||||
WHERE j.id = @id
|
WHERE j.id = @id
|
||||||
@@ -116,6 +117,24 @@ export const getJob = (jobId) => {
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Record the timestamp at which a job was last triggered.
|
||||||
|
*
|
||||||
|
* Called from the job execution service when a job starts running. The value
|
||||||
|
* is persisted so that the dashboard "last search" KPI survives restarts and
|
||||||
|
* can be computed per accessible user.
|
||||||
|
*
|
||||||
|
* @param {string} jobId - Job primary key.
|
||||||
|
* @param {number} timestamp - Epoch milliseconds.
|
||||||
|
* @returns {void}
|
||||||
|
*/
|
||||||
|
export const updateJobLastRunAt = (jobId, timestamp) => {
|
||||||
|
SqliteConnection.execute(`UPDATE jobs SET last_run_at = @timestamp WHERE id = @id`, {
|
||||||
|
id: jobId,
|
||||||
|
timestamp,
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Update job enabled status.
|
* Update job enabled status.
|
||||||
* @param {{jobId: string, status: boolean}} params - Parameters.
|
* @param {{jobId: string, status: boolean}} params - Parameters.
|
||||||
@@ -150,9 +169,17 @@ export const removeJobsByUserId = (userId) => {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Get all jobs.
|
* Get all jobs.
|
||||||
|
*
|
||||||
|
* By default only enabled jobs are returned, since most callers (scheduler,
|
||||||
|
* geocoding cron, tracker, dashboard) operate on active jobs only. The UI,
|
||||||
|
* however, must also be able to load disabled jobs (e.g. to edit them or view
|
||||||
|
* their listings), so it passes `includeDisabled: true`.
|
||||||
|
*
|
||||||
|
* @param {Object} [params]
|
||||||
|
* @param {boolean} [params.includeDisabled=false] - When true, disabled jobs are included.
|
||||||
* @returns {Job[]} List of jobs ordered by name (NULLs last).
|
* @returns {Job[]} List of jobs ordered by name (NULLs last).
|
||||||
*/
|
*/
|
||||||
export const getJobs = () => {
|
export const getJobs = ({ includeDisabled = false } = {}) => {
|
||||||
const rows = SqliteConnection.query(
|
const rows = SqliteConnection.query(
|
||||||
`SELECT j.id,
|
`SELECT j.id,
|
||||||
j.user_id AS userId,
|
j.user_id AS userId,
|
||||||
@@ -164,9 +191,10 @@ export const getJobs = () => {
|
|||||||
j.notification_adapter AS notificationAdapter,
|
j.notification_adapter AS notificationAdapter,
|
||||||
j.spatial_filter AS spatialFilter,
|
j.spatial_filter AS spatialFilter,
|
||||||
j.spec_filter AS specFilter,
|
j.spec_filter AS specFilter,
|
||||||
|
j.last_run_at AS lastRunAt,
|
||||||
(SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id AND l.is_active = 1 AND l.manually_deleted = 0) AS numberOfFoundListings
|
(SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id AND l.is_active = 1 AND l.manually_deleted = 0) AS numberOfFoundListings
|
||||||
FROM jobs j
|
FROM jobs j
|
||||||
WHERE j.enabled = 1
|
${includeDisabled ? '' : 'WHERE j.enabled = 1'}
|
||||||
ORDER BY j.name IS NULL, j.name`,
|
ORDER BY j.name IS NULL, j.name`,
|
||||||
);
|
);
|
||||||
return rows.map((row) => ({
|
return rows.map((row) => ({
|
||||||
@@ -269,6 +297,7 @@ export const queryJobs = ({
|
|||||||
j.notification_adapter AS notificationAdapter,
|
j.notification_adapter AS notificationAdapter,
|
||||||
j.spatial_filter AS spatialFilter,
|
j.spatial_filter AS spatialFilter,
|
||||||
j.spec_filter AS specFilter,
|
j.spec_filter AS specFilter,
|
||||||
|
j.last_run_at AS lastRunAt,
|
||||||
(SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id AND l.is_active = 1 AND l.manually_deleted = 0) AS numberOfFoundListings
|
(SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id AND l.is_active = 1 AND l.manually_deleted = 0) AS numberOfFoundListings
|
||||||
FROM jobs j
|
FROM jobs j
|
||||||
${whereSql}
|
${whereSql}
|
||||||
|
|||||||
@@ -3,10 +3,27 @@
|
|||||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { nullOrEmpty } from '../../utils.js';
|
import { nullOrEmpty, fromJson } from '../../utils.js';
|
||||||
import SqliteConnection from './SqliteConnection.js';
|
import SqliteConnection from './SqliteConnection.js';
|
||||||
import { nanoid } from 'nanoid';
|
import { nanoid } from 'nanoid';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse the JSON `status` column of a listing row in place.
|
||||||
|
*
|
||||||
|
* The DB stores status as a JSON payload `{ status, setAt }` (or NULL).
|
||||||
|
* Consumers expect an object/null, so we normalize before returning.
|
||||||
|
*
|
||||||
|
* @param {Object|null|undefined} row - A raw row from the listings table.
|
||||||
|
* @returns {Object|null|undefined} The same row with `status` parsed.
|
||||||
|
*/
|
||||||
|
const parseListingStatus = (row) => {
|
||||||
|
if (row == null) return row;
|
||||||
|
if (typeof row.status === 'string') {
|
||||||
|
row.status = fromJson(row.status, null);
|
||||||
|
}
|
||||||
|
return row;
|
||||||
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Return a list of known listing hashes for a given job and provider.
|
* Return a list of known listing hashes for a given job and provider.
|
||||||
* Useful to de-duplicate before inserting new listings.
|
* Useful to de-duplicate before inserting new listings.
|
||||||
@@ -43,18 +60,14 @@ export const getListingsKpisForJobIds = (jobIds = []) => {
|
|||||||
|
|
||||||
const placeholders = jobIds.map(() => '?').join(',');
|
const placeholders = jobIds.map(() => '?').join(',');
|
||||||
const rows = SqliteConnection.query(
|
const rows = SqliteConnection.query(
|
||||||
`SELECT
|
`SELECT is_active, price
|
||||||
SUM(CASE WHEN is_active = 1 THEN 1 ELSE 0 END) OVER() AS active_count,
|
FROM listings
|
||||||
price
|
WHERE job_id IN (${placeholders})
|
||||||
FROM listings
|
AND manually_deleted = 0`,
|
||||||
WHERE job_id IN (${placeholders})
|
|
||||||
AND manually_deleted = 0
|
|
||||||
GROUP BY
|
|
||||||
id`,
|
|
||||||
jobIds,
|
jobIds,
|
||||||
);
|
);
|
||||||
|
|
||||||
const activeCount = rows[0]?.active_count ?? 0;
|
const activeCount = rows.filter((r) => r.is_active === 1).length;
|
||||||
|
|
||||||
const prices = rows
|
const prices = rows
|
||||||
.map((r) => r.price)
|
.map((r) => r.price)
|
||||||
@@ -214,6 +227,8 @@ export const storeListings = (jobId, providerId, listings) => {
|
|||||||
longitude: item.longitude || null,
|
longitude: item.longitude || null,
|
||||||
};
|
};
|
||||||
stmt.run(params);
|
stmt.run(params);
|
||||||
|
// Propagate the DB primary key back so downstream pipeline steps use the correct id
|
||||||
|
item.id = params.id;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -242,12 +257,14 @@ export const storeListings = (jobId, providerId, listings) => {
|
|||||||
* @param {object} [params.jobNameFilter]
|
* @param {object} [params.jobNameFilter]
|
||||||
* @param {object} [params.providerFilter]
|
* @param {object} [params.providerFilter]
|
||||||
* @param {object} [params.watchListFilter]
|
* @param {object} [params.watchListFilter]
|
||||||
|
* @param {('applied'|'rejected'|'accepted'|'none')} [params.statusFilter] - Filter by listing status. 'none' matches NULL.
|
||||||
* @param {string|null} [params.sortField=null] - One of: 'created_at','price','size','provider','title'.
|
* @param {string|null} [params.sortField=null] - One of: 'created_at','price','size','provider','title'.
|
||||||
* @param {('asc'|'desc')} [params.sortDir='asc']
|
* @param {('asc'|'desc')} [params.sortDir='asc']
|
||||||
* @param {number} [params.createdAfter] - Only include listings created at or after this unix timestamp (ms).
|
* @param {number} [params.createdAfter] - Only include listings created at or after this unix timestamp (ms).
|
||||||
* @param {number} [params.createdBefore] - Only include listings created at or before this unix timestamp (ms).
|
* @param {number} [params.createdBefore] - Only include listings created at or before this unix timestamp (ms).
|
||||||
* @param {string} [params.userId] - Current user id used to scope listings (ignored for admins).
|
* @param {string} [params.userId] - Current user id used to scope listings (ignored for admins).
|
||||||
* @param {boolean} [params.isAdmin=false] - When true, returns all listings.
|
* @param {boolean} [params.isAdmin=false] - When true, returns all listings.
|
||||||
|
* @param {boolean} [params.hiddenOnly=false] - When true, returns only soft-deleted (manually_deleted = 1) listings.
|
||||||
* @returns {{ totalNumber:number, page:number, result:Object[] }}
|
* @returns {{ totalNumber:number, page:number, result:Object[] }}
|
||||||
*/
|
*/
|
||||||
export const queryListings = ({
|
export const queryListings = ({
|
||||||
@@ -258,6 +275,7 @@ export const queryListings = ({
|
|||||||
jobIdFilter,
|
jobIdFilter,
|
||||||
providerFilter,
|
providerFilter,
|
||||||
watchListFilter,
|
watchListFilter,
|
||||||
|
statusFilter,
|
||||||
freeTextFilter,
|
freeTextFilter,
|
||||||
sortField = null,
|
sortField = null,
|
||||||
sortDir = 'asc',
|
sortDir = 'asc',
|
||||||
@@ -267,6 +285,7 @@ export const queryListings = ({
|
|||||||
maxPrice = null,
|
maxPrice = null,
|
||||||
userId = null,
|
userId = null,
|
||||||
isAdmin = false,
|
isAdmin = false,
|
||||||
|
hiddenOnly = false,
|
||||||
} = {}) => {
|
} = {}) => {
|
||||||
// sanitize inputs
|
// sanitize inputs
|
||||||
const safePageSize = Number.isFinite(pageSize) && pageSize > 0 ? Math.min(1000, Math.floor(pageSize)) : 50;
|
const safePageSize = Number.isFinite(pageSize) && pageSize > 0 ? Math.min(1000, Math.floor(pageSize)) : 50;
|
||||||
@@ -287,13 +306,15 @@ export const queryListings = ({
|
|||||||
}
|
}
|
||||||
if (freeTextFilter && String(freeTextFilter).trim().length > 0) {
|
if (freeTextFilter && String(freeTextFilter).trim().length > 0) {
|
||||||
params.filter = `%${String(freeTextFilter).trim()}%`;
|
params.filter = `%${String(freeTextFilter).trim()}%`;
|
||||||
whereParts.push(`(title LIKE @filter OR address LIKE @filter OR provider LIKE @filter OR link LIKE @filter)`);
|
whereParts.push(
|
||||||
|
`(l.title LIKE @filter OR l.address LIKE @filter OR l.provider LIKE @filter OR l.link LIKE @filter)`,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
// activityFilter: when true -> only active listings (is_active = 1), false -> only inactive
|
// activityFilter: when true -> only active listings (is_active = 1), false -> only inactive
|
||||||
if (activityFilter === true) {
|
if (activityFilter === true) {
|
||||||
whereParts.push('(is_active = 1)');
|
whereParts.push('(l.is_active = 1)');
|
||||||
} else if (activityFilter === false) {
|
} else if (activityFilter === false) {
|
||||||
whereParts.push('(is_active = 0)');
|
whereParts.push('(l.is_active = 0)');
|
||||||
}
|
}
|
||||||
// Prefer filtering by job id when provided (unambiguous and robust)
|
// Prefer filtering by job id when provided (unambiguous and robust)
|
||||||
if (jobIdFilter && String(jobIdFilter).trim().length > 0) {
|
if (jobIdFilter && String(jobIdFilter).trim().length > 0) {
|
||||||
@@ -307,7 +328,7 @@ export const queryListings = ({
|
|||||||
// providerFilter: when provided as string (assumed provider name), filter listings where provider equals that name (exact match)
|
// providerFilter: when provided as string (assumed provider name), filter listings where provider equals that name (exact match)
|
||||||
if (providerFilter && String(providerFilter).trim().length > 0) {
|
if (providerFilter && String(providerFilter).trim().length > 0) {
|
||||||
params.providerName = String(providerFilter).trim();
|
params.providerName = String(providerFilter).trim();
|
||||||
whereParts.push('(provider = @providerName)');
|
whereParts.push('(l.provider = @providerName)');
|
||||||
}
|
}
|
||||||
// watchListFilter: when true -> only watched listings, false -> only unwatched
|
// watchListFilter: when true -> only watched listings, false -> only unwatched
|
||||||
if (watchListFilter === true) {
|
if (watchListFilter === true) {
|
||||||
@@ -315,14 +336,26 @@ export const queryListings = ({
|
|||||||
} else if (watchListFilter === false) {
|
} else if (watchListFilter === false) {
|
||||||
whereParts.push('(wl.id IS NULL)');
|
whereParts.push('(wl.id IS NULL)');
|
||||||
}
|
}
|
||||||
|
// statusFilter: 'applied'|'rejected'|'accepted' -> equality on JSON status field; 'none' -> NULL.
|
||||||
|
// The status column is a JSON payload `{ status, setAt }`, so we extract the inner
|
||||||
|
// status string for comparison instead of matching the raw text.
|
||||||
|
if (statusFilter === 'none') {
|
||||||
|
whereParts.push('(l.status IS NULL)');
|
||||||
|
} else if (
|
||||||
|
typeof statusFilter === 'string' &&
|
||||||
|
['applied', 'rejected', 'accepted'].includes(statusFilter.toLowerCase())
|
||||||
|
) {
|
||||||
|
params.statusValue = statusFilter.toLowerCase();
|
||||||
|
whereParts.push(`(json_extract(l.status, '$.status') = @statusValue)`);
|
||||||
|
}
|
||||||
// Time range filters (unix timestamps in milliseconds)
|
// Time range filters (unix timestamps in milliseconds)
|
||||||
if (Number.isFinite(createdAfter) && createdAfter > 0) {
|
if (Number.isFinite(createdAfter) && createdAfter > 0) {
|
||||||
params.createdAfter = createdAfter;
|
params.createdAfter = createdAfter;
|
||||||
whereParts.push('(created_at >= @createdAfter)');
|
whereParts.push('(l.created_at >= @createdAfter)');
|
||||||
}
|
}
|
||||||
if (Number.isFinite(createdBefore) && createdBefore > 0) {
|
if (Number.isFinite(createdBefore) && createdBefore > 0) {
|
||||||
params.createdBefore = createdBefore;
|
params.createdBefore = createdBefore;
|
||||||
whereParts.push('(created_at <= @createdBefore)');
|
whereParts.push('(l.created_at <= @createdBefore)');
|
||||||
}
|
}
|
||||||
// Price range filters
|
// Price range filters
|
||||||
if (Number.isFinite(minPrice) && minPrice >= 0) {
|
if (Number.isFinite(minPrice) && minPrice >= 0) {
|
||||||
@@ -334,35 +367,25 @@ export const queryListings = ({
|
|||||||
whereParts.push('(l.price <= @maxPrice)');
|
whereParts.push('(l.price <= @maxPrice)');
|
||||||
}
|
}
|
||||||
|
|
||||||
// Build whereSql (filtering by manually_deleted = 0)
|
// Build whereSql: in normal mode hide soft-deleted; in hiddenOnly mode show only soft-deleted.
|
||||||
whereParts.push('(l.manually_deleted = 0)');
|
whereParts.push(hiddenOnly ? '(l.manually_deleted = 1)' : '(l.manually_deleted = 0)');
|
||||||
|
|
||||||
const whereSql = whereParts.length ? `WHERE ${whereParts.join(' AND ')}` : '';
|
const whereSqlWithAlias = whereParts.length ? `WHERE ${whereParts.join(' AND ')}` : '';
|
||||||
const whereSqlWithAlias = whereSql
|
|
||||||
.replace(/\btitle\b/g, 'l.title')
|
|
||||||
.replace(/\bdescription\b/g, 'l.description')
|
|
||||||
.replace(/\baddress\b/g, 'l.address')
|
|
||||||
.replace(/\bprovider\b/g, 'l.provider')
|
|
||||||
.replace(/\blink\b/g, 'l.link')
|
|
||||||
.replace(/\bis_active\b/g, 'l.is_active')
|
|
||||||
.replace(/\bj\.user_id\b/g, 'j.user_id')
|
|
||||||
.replace(/\bj\.name\b/g, 'j.name')
|
|
||||||
.replace(/\bwl\.id\b/g, 'wl.id');
|
|
||||||
|
|
||||||
// whitelist sortable fields to avoid SQL injection
|
// whitelist sortable fields to avoid SQL injection; map to fully-qualified expressions
|
||||||
const sortable = new Set(['created_at', 'price', 'size', 'provider', 'title', 'job_name', 'is_active', 'isWatched']);
|
const sortableMap = {
|
||||||
const safeSortField = sortField && sortable.has(sortField) ? sortField : null;
|
created_at: 'l.created_at',
|
||||||
|
price: 'l.price',
|
||||||
|
size: 'l.size',
|
||||||
|
provider: 'l.provider',
|
||||||
|
title: 'l.title',
|
||||||
|
job_name: 'j.name',
|
||||||
|
is_active: 'l.is_active',
|
||||||
|
isWatched: 'CASE WHEN wl.id IS NOT NULL THEN 1 ELSE 0 END',
|
||||||
|
};
|
||||||
|
const safeSortExpr = sortField && sortableMap[sortField] ? sortableMap[sortField] : null;
|
||||||
const safeSortDir = String(sortDir).toLowerCase() === 'desc' ? 'DESC' : 'ASC';
|
const safeSortDir = String(sortDir).toLowerCase() === 'desc' ? 'DESC' : 'ASC';
|
||||||
const orderSql = safeSortField ? `ORDER BY ${safeSortField} ${safeSortDir}` : 'ORDER BY created_at DESC';
|
const orderSqlWithAlias = safeSortExpr ? `ORDER BY ${safeSortExpr} ${safeSortDir}` : 'ORDER BY l.created_at DESC';
|
||||||
const orderSqlWithAlias = orderSql
|
|
||||||
.replace(/\bcreated_at\b/g, 'l.created_at')
|
|
||||||
.replace(/\bprice\b/g, 'l.price')
|
|
||||||
.replace(/\bsize\b/g, 'l.size')
|
|
||||||
.replace(/\bprovider\b/g, 'l.provider')
|
|
||||||
.replace(/\btitle\b/g, 'l.title')
|
|
||||||
.replace(/\bjob_name\b/g, 'j.name')
|
|
||||||
// Sort by computed watch flag when requested
|
|
||||||
.replace(/\bisWatched\b/g, 'CASE WHEN wl.id IS NOT NULL THEN 1 ELSE 0 END');
|
|
||||||
|
|
||||||
// count total with same WHERE
|
// count total with same WHERE
|
||||||
const countRow = SqliteConnection.query(
|
const countRow = SqliteConnection.query(
|
||||||
@@ -389,7 +412,7 @@ export const queryListings = ({
|
|||||||
params,
|
params,
|
||||||
);
|
);
|
||||||
|
|
||||||
return { totalNumber, page: safePage, result: rows };
|
return { totalNumber, page: safePage, result: rows.map(parseListingStatus) };
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -417,9 +440,10 @@ export const deleteListingsByJobId = (jobId, hardDelete = false) => {
|
|||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Delete listings by a list of listing IDs.
|
* Delete listings by a list of listing IDs (the nanoid primary key stored in the `id` column).
|
||||||
|
* Used by API routes that receive row IDs from the client.
|
||||||
*
|
*
|
||||||
* @param {string[]} ids - Array of listing IDs to delete.
|
* @param {string[]} ids - Array of DB row IDs to delete.
|
||||||
* @param {boolean} [hardDelete=false] - Whether to hard delete from DB or just mark as deleted.
|
* @param {boolean} [hardDelete=false] - Whether to hard delete from DB or just mark as deleted.
|
||||||
* @returns {any} The result from SqliteConnection.execute.
|
* @returns {any} The result from SqliteConnection.execute.
|
||||||
*/
|
*/
|
||||||
@@ -441,6 +465,23 @@ export const deleteListingsById = (ids, hardDelete = false) => {
|
|||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Restore previously soft-deleted listings by clearing their `manually_deleted` flag.
|
||||||
|
*
|
||||||
|
* @param {string[]} ids - Array of DB row IDs to restore.
|
||||||
|
* @returns {any} The result from SqliteConnection.execute.
|
||||||
|
*/
|
||||||
|
export const restoreListingsById = (ids) => {
|
||||||
|
if (!Array.isArray(ids) || ids.length === 0) return;
|
||||||
|
const placeholders = ids.map(() => '?').join(',');
|
||||||
|
return SqliteConnection.execute(
|
||||||
|
`UPDATE listings
|
||||||
|
SET manually_deleted = 0
|
||||||
|
WHERE id IN (${placeholders})`,
|
||||||
|
ids,
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Return all listings that are active, have an address, and do not yet have geocoordinates.
|
* Return all listings that are active, have an address, and do not yet have geocoordinates.
|
||||||
*
|
*
|
||||||
@@ -482,7 +523,7 @@ export const updateListingGeocoordinates = (id, latitude, longitude) => {
|
|||||||
* @param {string} [params.jobId]
|
* @param {string} [params.jobId]
|
||||||
* @param {string} [params.userId]
|
* @param {string} [params.userId]
|
||||||
* @param {boolean} [params.isAdmin=false]
|
* @param {boolean} [params.isAdmin=false]
|
||||||
* @returns {{listings: Object[], maxPrice: number}} Object containing listings and maxPrice.
|
* @returns {{listings: Object[]}} Object containing listings.
|
||||||
*/
|
*/
|
||||||
export const getListingsForMap = ({ jobId, userId = null, isAdmin = false } = {}) => {
|
export const getListingsForMap = ({ jobId, userId = null, isAdmin = false } = {}) => {
|
||||||
const baseWhereParts = [
|
const baseWhereParts = [
|
||||||
@@ -623,7 +664,7 @@ export const getListingById = (id, userId = null, isAdmin = false) => {
|
|||||||
if (!isAdmin) {
|
if (!isAdmin) {
|
||||||
whereScoping = `AND (j.user_id = @userId OR EXISTS (SELECT 1 FROM json_each(j.shared_with_user) AS sw WHERE sw.value = @userId))`;
|
whereScoping = `AND (j.user_id = @userId OR EXISTS (SELECT 1 FROM json_each(j.shared_with_user) AS sw WHERE sw.value = @userId))`;
|
||||||
}
|
}
|
||||||
return (
|
return parseListingStatus(
|
||||||
SqliteConnection.query(
|
SqliteConnection.query(
|
||||||
`SELECT l.*, j.name AS job_name, CASE WHEN wl.id IS NOT NULL THEN 1 ELSE 0 END AS isWatched
|
`SELECT l.*, j.name AS job_name, CASE WHEN wl.id IS NOT NULL THEN 1 ELSE 0 END AS isWatched
|
||||||
FROM listings l
|
FROM listings l
|
||||||
@@ -631,10 +672,57 @@ export const getListingById = (id, userId = null, isAdmin = false) => {
|
|||||||
LEFT JOIN watch_list wl ON wl.listing_id = l.id AND wl.user_id = @userId
|
LEFT JOIN watch_list wl ON wl.listing_id = l.id AND wl.user_id = @userId
|
||||||
WHERE l.id = @id AND l.manually_deleted = 0 ${whereScoping}`,
|
WHERE l.id = @id AND l.manually_deleted = 0 ${whereScoping}`,
|
||||||
params,
|
params,
|
||||||
)[0] || null
|
)[0] || null,
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Set or clear the notes attached to a single listing.
|
||||||
|
*
|
||||||
|
* Empty strings are normalized to NULL so the DB doesn't keep meaningless
|
||||||
|
* whitespace and queries can filter "has notes" with a simple IS NOT NULL.
|
||||||
|
*
|
||||||
|
* @param {string} id - The listing ID.
|
||||||
|
* @param {string|null} notes - The note text to store, or null/empty to clear.
|
||||||
|
* @returns {number} Number of rows affected (0 if listing not found).
|
||||||
|
*/
|
||||||
|
export const setListingNotes = (id, notes) => {
|
||||||
|
if (!id) return 0;
|
||||||
|
const trimmed = typeof notes === 'string' ? notes.trim() : null;
|
||||||
|
const value = trimmed && trimmed.length > 0 ? trimmed : null;
|
||||||
|
const res = SqliteConnection.execute(`UPDATE listings SET notes = @notes WHERE id = @id`, {
|
||||||
|
id,
|
||||||
|
notes: value,
|
||||||
|
});
|
||||||
|
return res?.changes ?? 0;
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Set or clear the status of a single listing.
|
||||||
|
*
|
||||||
|
* The status column stores a JSON payload `{ status, setAt }` so consumers
|
||||||
|
* can show both the user's decision and when it was made. Passing `null`
|
||||||
|
* clears the column.
|
||||||
|
*
|
||||||
|
* @param {string} id - The listing ID.
|
||||||
|
* @param {('applied'|'rejected'|'accepted'|null)} status - New status, or null to clear.
|
||||||
|
* @returns {number} Number of rows affected (0 if listing not found).
|
||||||
|
*/
|
||||||
|
export const setListingStatus = (id, status) => {
|
||||||
|
if (!id) return 0;
|
||||||
|
const allowed = ['applied', 'rejected', 'accepted'];
|
||||||
|
const normalized = status == null ? null : String(status).toLowerCase();
|
||||||
|
if (normalized != null && !allowed.includes(normalized)) {
|
||||||
|
throw new Error(`Invalid listing status: ${status}`);
|
||||||
|
}
|
||||||
|
const payload = normalized == null ? null : JSON.stringify({ status: normalized, setAt: Date.now() });
|
||||||
|
const res = SqliteConnection.execute(`UPDATE listings SET status = @status WHERE id = @id`, {
|
||||||
|
id,
|
||||||
|
status: payload,
|
||||||
|
});
|
||||||
|
return res?.changes ?? 0;
|
||||||
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Resets geocoordinates and distance for all listings related to a user.
|
* Resets geocoordinates and distance for all listings related to a user.
|
||||||
*
|
*
|
||||||
|
|||||||
11
lib/services/storage/migrations/sql/18.add-listing-status.js
Normal file
11
lib/services/storage/migrations/sql/18.add-listing-status.js
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (c) 2026 by Christian Kellner.
|
||||||
|
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||||
|
*/
|
||||||
|
|
||||||
|
export function up(db) {
|
||||||
|
db.exec(`
|
||||||
|
ALTER TABLE listings ADD COLUMN status JSON;
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_listings_status ON listings (json_extract(status, '$.status'));
|
||||||
|
`);
|
||||||
|
}
|
||||||
10
lib/services/storage/migrations/sql/19.add-listing-notes.js
Normal file
10
lib/services/storage/migrations/sql/19.add-listing-notes.js
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (c) 2026 by Christian Kellner.
|
||||||
|
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||||
|
*/
|
||||||
|
|
||||||
|
export function up(db) {
|
||||||
|
db.exec(`
|
||||||
|
ALTER TABLE listings ADD COLUMN notes TEXT;
|
||||||
|
`);
|
||||||
|
}
|
||||||
32
lib/services/storage/migrations/sql/20.add-debug-logs.js
Normal file
32
lib/services/storage/migrations/sql/20.add-debug-logs.js
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (c) 2026 by Christian Kellner.
|
||||||
|
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Migration: create the debug_logs table used by the opt-in "Debug Logging" feature.
|
||||||
|
*
|
||||||
|
* Each row is a single log line (timestamp + level + message) captured by the in-app
|
||||||
|
* logger while debug logging is enabled. We store the UTF-8 byte size of the message
|
||||||
|
* alongside the row so the debugLogStorage can maintain a rolling 5 MB cap without
|
||||||
|
* having to run length() / SUM() on every insert.
|
||||||
|
*
|
||||||
|
* The "debug_logging_enabled" and "debug_logging_ever_enabled" flags are persisted in
|
||||||
|
* the existing settings table (no schema change needed there) and are managed by
|
||||||
|
* debugLogStorage.js at runtime.
|
||||||
|
*/
|
||||||
|
export function up(db) {
|
||||||
|
// id is INTEGER PRIMARY KEY AUTOINCREMENT, which is an alias for SQLite's rowid and
|
||||||
|
// is implicitly indexed. No additional index needed; selecting / deleting by id and
|
||||||
|
// ordering by id ASC (rolling buffer) both use the existing rowid index.
|
||||||
|
db.exec(`
|
||||||
|
CREATE TABLE IF NOT EXISTS debug_logs
|
||||||
|
(
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
ts INTEGER NOT NULL,
|
||||||
|
level TEXT NOT NULL,
|
||||||
|
message TEXT NOT NULL,
|
||||||
|
byte_size INTEGER NOT NULL
|
||||||
|
);
|
||||||
|
`);
|
||||||
|
}
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (c) 2026 by Christian Kellner.
|
||||||
|
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Migration: add `last_run_at` to the `jobs` table.
|
||||||
|
*
|
||||||
|
* Stores the epoch-ms timestamp at which a job was last triggered. Used by the
|
||||||
|
* dashboard "last search" KPI so the value survives restarts and reflects the
|
||||||
|
* actual jobs the requesting user can see (own, shared, or all for admins),
|
||||||
|
* replacing the previous in-memory `settings.lastRun` value.
|
||||||
|
*
|
||||||
|
* NULL means the job has not yet been triggered since this column was added.
|
||||||
|
*/
|
||||||
|
export function up(db) {
|
||||||
|
db.exec(`
|
||||||
|
ALTER TABLE jobs ADD COLUMN last_run_at INTEGER
|
||||||
|
`);
|
||||||
|
}
|
||||||
@@ -123,8 +123,11 @@ export function upsertSettings(settingsMapOrEntry, userId = null) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// keep cache in sync (only for global settings)
|
// Invalidate cache synchronously so the next getSettings() call rebuilds it.
|
||||||
|
// refreshSettingsCache() is async (reads config.json), so we cannot await it
|
||||||
|
// here without making upsertSettings async everywhere. Nulling is safe because
|
||||||
|
// getSettings() will call refreshSettingsCache() on the next invocation.
|
||||||
if (userId == null) {
|
if (userId == null) {
|
||||||
refreshSettingsCache();
|
cachedSettingsConfig = null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -47,6 +47,25 @@ export const deleteWatch = (listingId, userId) => {
|
|||||||
return { deleted: Boolean(res?.changes) };
|
return { deleted: Boolean(res?.changes) };
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ensure a watch entry exists. Does not toggle; safe to call when row may already exist.
|
||||||
|
* Used by the status endpoint to auto-watch a listing when a status is set.
|
||||||
|
* @param {string} listingId
|
||||||
|
* @param {string} userId
|
||||||
|
* @returns {{watched:boolean}}
|
||||||
|
*/
|
||||||
|
export const ensureWatch = (listingId, userId) => {
|
||||||
|
if (!listingId || !userId) return { watched: false };
|
||||||
|
const { created } = createWatch(listingId, userId);
|
||||||
|
if (created) return { watched: true };
|
||||||
|
const exists =
|
||||||
|
SqliteConnection.query(
|
||||||
|
`SELECT 1 AS ok FROM watch_list WHERE listing_id = @listing_id AND user_id = @user_id LIMIT 1`,
|
||||||
|
{ listing_id: listingId, user_id: userId },
|
||||||
|
).length > 0;
|
||||||
|
return { watched: exists };
|
||||||
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Toggle a watch entry. If exists -> delete, otherwise create.
|
* Toggle a watch entry. If exists -> delete, otherwise create.
|
||||||
* @param {string} listingId
|
* @param {string} listingId
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import { getSettings } from '../storage/settingsStorage.js';
|
|||||||
const deviceId = getUniqueId() || 'N/A';
|
const deviceId = getUniqueId() || 'N/A';
|
||||||
const version = await getPackageVersion();
|
const version = await getPackageVersion();
|
||||||
const FREDY_TRACKING_URL = 'https://fredy.orange-coding.net/tracking';
|
const FREDY_TRACKING_URL = 'https://fredy.orange-coding.net/tracking';
|
||||||
|
const TRACKING_CATEGORY = 'fredy';
|
||||||
const isDocker = process.env.IS_DOCKER != null;
|
const isDocker = process.env.IS_DOCKER != null;
|
||||||
|
|
||||||
const staticTrackingData = {
|
const staticTrackingData = {
|
||||||
@@ -95,6 +96,7 @@ async function enrichTrackingObject(trackingObject) {
|
|||||||
const settings = await getSettings();
|
const settings = await getSettings();
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
category: TRACKING_CATEGORY,
|
||||||
...trackingObject,
|
...trackingObject,
|
||||||
...staticTrackingData,
|
...staticTrackingData,
|
||||||
isDemo: settings.demoMode,
|
isDemo: settings.demoMode,
|
||||||
|
|||||||
@@ -18,6 +18,7 @@
|
|||||||
* @property {SpatialFilter | null} [spatialFilter] Optional spatial filter configuration as GeoJSON FeatureCollection.
|
* @property {SpatialFilter | null} [spatialFilter] Optional spatial filter configuration as GeoJSON FeatureCollection.
|
||||||
* @property {SpecFilter | null} [specFilter] Optional listing specifications.
|
* @property {SpecFilter | null} [specFilter] Optional listing specifications.
|
||||||
* @property {number} [numberOfFoundListings] Count of active listings for this job.
|
* @property {number} [numberOfFoundListings] Count of active listings for this job.
|
||||||
|
* @property {number | null} [lastRunAt] Epoch ms at which the job was last triggered, or null if never triggered.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export {};
|
export {};
|
||||||
|
|||||||
@@ -5,12 +5,13 @@
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Extract the first number from a string like "1.234 €" or "70 m²".
|
* Extract the first number from a string like "1.234 €" or "70 m²".
|
||||||
* Removes dots/commas before parsing. Returns null on invalid input.
|
* Removes dots/commas before parsing. Returns null when the input is
|
||||||
|
* null/undefined or cannot be parsed into a number.
|
||||||
* @param {string|undefined|null} str
|
* @param {string|undefined|null} str
|
||||||
* @returns {number|null}
|
* @returns {number|null}
|
||||||
*/
|
*/
|
||||||
export const extractNumber = (str) => {
|
export const extractNumber = (str) => {
|
||||||
if (str == null) return 0;
|
if (str == null) return null;
|
||||||
if (typeof str === 'number') return str;
|
if (typeof str === 'number') return str;
|
||||||
const cleaned = str.replace(/\./g, '').replace(',', '.');
|
const cleaned = str.replace(/\./g, '').replace(',', '.');
|
||||||
const num = parseFloat(cleaned);
|
const num = parseFloat(cleaned);
|
||||||
|
|||||||
57
package.json
57
package.json
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"name": "fredy",
|
"name": "fredy",
|
||||||
"version": "22.0.6",
|
"version": "22.9.1",
|
||||||
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
|
"description": "Fredy - [F]ind [R]eal [E]state [D]amn Eas[y] - Fredy keeps searching for new apartments, houses, and flats in Germany on platforms like ImmoScout24, Immowelt, Immonet, eBay Kleinanzeigen, and WG-Gesucht and instantly delivers the results to you via Slack, Telegram, Email, Discord or ntfy, so you can focus on the more important things in life ;)",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"prepare": "husky",
|
"prepare": "husky",
|
||||||
"start:backend": "x-var NODE_ENV=production node index.js",
|
"start:backend": "x-var NODE_ENV=production node index.js",
|
||||||
@@ -42,6 +42,7 @@
|
|||||||
"house",
|
"house",
|
||||||
"rent",
|
"rent",
|
||||||
"immoscout",
|
"immoscout",
|
||||||
|
"kleinanzeigen",
|
||||||
"scraper",
|
"scraper",
|
||||||
"immonet",
|
"immonet",
|
||||||
"immowelt",
|
"immowelt",
|
||||||
@@ -62,9 +63,9 @@
|
|||||||
"Firefox ESR"
|
"Firefox ESR"
|
||||||
],
|
],
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@douyinfe/semi-icons": "^2.97.0",
|
"@douyinfe/semi-icons": "^2.100.0",
|
||||||
"@douyinfe/semi-ui": "2.97.0",
|
"@douyinfe/semi-ui": "2.100.0",
|
||||||
"@douyinfe/semi-ui-19": "^2.97.0",
|
"@douyinfe/semi-ui-19": "^2.100.0",
|
||||||
"@fastify/cookie": "^11.0.2",
|
"@fastify/cookie": "^11.0.2",
|
||||||
"@fastify/helmet": "^13.0.2",
|
"@fastify/helmet": "^13.0.2",
|
||||||
"@fastify/session": "^11.1.1",
|
"@fastify/session": "^11.1.1",
|
||||||
@@ -73,12 +74,12 @@
|
|||||||
"@modelcontextprotocol/sdk": "^1.29.0",
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
||||||
"@sendgrid/mail": "8.1.6",
|
"@sendgrid/mail": "8.1.6",
|
||||||
"@turf/boolean-point-in-polygon": "^7.3.5",
|
"@turf/boolean-point-in-polygon": "^7.3.5",
|
||||||
"@vitejs/plugin-react": "6.0.1",
|
"@vitejs/plugin-react": "6.0.2",
|
||||||
"adm-zip": "^0.5.17",
|
"adm-zip": "^0.5.17",
|
||||||
"better-sqlite3": "^12.10.0",
|
"better-sqlite3": "^12.10.1",
|
||||||
"chart.js": "^4.5.1",
|
"chart.js": "^4.5.1",
|
||||||
"cheerio": "^1.2.0",
|
"cheerio": "^1.2.0",
|
||||||
"cloakbrowser": "^0.3.28",
|
"cloakbrowser": "^0.3.31",
|
||||||
"fastify": "^5.8.5",
|
"fastify": "^5.8.5",
|
||||||
"handlebars": "4.7.9",
|
"handlebars": "4.7.9",
|
||||||
"maplibre-gl": "^5.24.0",
|
"maplibre-gl": "^5.24.0",
|
||||||
@@ -86,41 +87,41 @@
|
|||||||
"node-cron": "^4.2.1",
|
"node-cron": "^4.2.1",
|
||||||
"node-fetch": "3.3.2",
|
"node-fetch": "3.3.2",
|
||||||
"node-mailjet": "6.0.11",
|
"node-mailjet": "6.0.11",
|
||||||
"nodemailer": "^8.0.7",
|
"nodemailer": "^8.0.11",
|
||||||
"p-throttle": "^8.1.0",
|
"p-throttle": "^8.1.0",
|
||||||
"package-up": "^5.0.0",
|
"package-up": "^5.0.0",
|
||||||
"puppeteer-core": "^24.43.1",
|
"puppeteer-core": "^25.1.0",
|
||||||
"query-string": "9.3.1",
|
"query-string": "9.4.0",
|
||||||
"react": "19.2.6",
|
"react": "19.2.7",
|
||||||
"react-chartjs-2": "^5.3.1",
|
"react-chartjs-2": "^5.3.1",
|
||||||
"react-dom": "19.2.6",
|
"react-dom": "19.2.7",
|
||||||
"react-range-slider-input": "^3.3.5",
|
"react-range-slider-input": "^3.3.5",
|
||||||
"react-router": "7.15.0",
|
"react-router": "7.17.0",
|
||||||
"react-router-dom": "7.15.0",
|
"react-router-dom": "7.17.0",
|
||||||
"resend": "^6.12.3",
|
"resend": "^6.12.4",
|
||||||
"semver": "^7.8.0",
|
"semver": "^7.8.4",
|
||||||
"slack": "11.0.2",
|
"slack": "11.0.2",
|
||||||
"vite": "8.0.12",
|
"vite": "8.0.16",
|
||||||
"x-var": "^3.0.1",
|
"x-var": "^3.0.1",
|
||||||
"zustand": "^5.0.13"
|
"zustand": "^5.0.14"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@babel/core": "7.29.0",
|
"@babel/core": "7.29.7",
|
||||||
"@babel/eslint-parser": "7.28.6",
|
"@babel/eslint-parser": "7.29.7",
|
||||||
"@babel/preset-env": "7.29.5",
|
"@babel/preset-env": "7.29.7",
|
||||||
"@babel/preset-react": "7.28.5",
|
"@babel/preset-react": "7.29.7",
|
||||||
"@eslint/js": "^10.0.1",
|
"@eslint/js": "^10.0.1",
|
||||||
"chalk": "^5.6.2",
|
"chalk": "^5.6.2",
|
||||||
"eslint": "10.3.0",
|
"eslint": "10.5.0",
|
||||||
"eslint-config-prettier": "10.1.8",
|
"eslint-config-prettier": "10.1.8",
|
||||||
"eslint-plugin-react": "7.37.5",
|
"eslint-plugin-react": "7.37.5",
|
||||||
"globals": "^17.6.0",
|
"globals": "^17.6.0",
|
||||||
"history": "5.3.0",
|
"history": "5.3.0",
|
||||||
"husky": "9.1.7",
|
"husky": "9.1.7",
|
||||||
"less": "4.6.4",
|
"less": "4.6.6",
|
||||||
"lint-staged": "17.0.4",
|
"lint-staged": "17.0.7",
|
||||||
"nodemon": "^3.1.14",
|
"nodemon": "^3.1.14",
|
||||||
"prettier": "3.8.3",
|
"prettier": "3.8.4",
|
||||||
"vitest": "^4.1.6"
|
"vitest": "^4.1.8"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,8 +17,12 @@ export const getGeocoordinatesByAddress = (any) => {
|
|||||||
return null;
|
return null;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let userSettings = null;
|
||||||
|
export function setUserSettings(settings) {
|
||||||
|
userSettings = settings;
|
||||||
|
}
|
||||||
export function getUserSettings(userId) {
|
export function getUserSettings(userId) {
|
||||||
return null;
|
return userSettings;
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function getSettings() {
|
export async function getSettings() {
|
||||||
@@ -32,4 +36,7 @@ export const deletedIds = [];
|
|||||||
export const deleteListingsById = (ids) => {
|
export const deleteListingsById = (ids) => {
|
||||||
deletedIds.push(...ids);
|
deletedIds.push(...ids);
|
||||||
};
|
};
|
||||||
|
export const deleteListingsByHash = (hashes) => {
|
||||||
|
deletedIds.push(...hashes);
|
||||||
|
};
|
||||||
/* eslint-enable no-unused-vars */
|
/* eslint-enable no-unused-vars */
|
||||||
|
|||||||
415
test/notification/telegram.test.js
Normal file
415
test/notification/telegram.test.js
Normal file
@@ -0,0 +1,415 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (c) 2026 by Christian Kellner.
|
||||||
|
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||||
|
|
||||||
|
// Mock external deps BEFORE importing the module under test.
|
||||||
|
vi.mock('node-fetch', () => ({ default: vi.fn() }));
|
||||||
|
vi.mock('../../lib/services/storage/jobStorage.js', () => ({
|
||||||
|
getJob: (jobKey) => ({ id: jobKey, name: jobKey }),
|
||||||
|
}));
|
||||||
|
vi.mock('../../lib/services/markdown.js', () => ({
|
||||||
|
markdown2Html: () => '',
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Helpers to build mock fetch responses.
|
||||||
|
function jsonOk(body = { ok: true }) {
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
text: async () => JSON.stringify(body),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function jsonErr(status, body) {
|
||||||
|
return {
|
||||||
|
ok: false,
|
||||||
|
status,
|
||||||
|
text: async () => JSON.stringify(body),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function imageOk(bytes = new Uint8Array([0xff, 0xd8, 0xff])) {
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
headers: {
|
||||||
|
get: (h) => {
|
||||||
|
const k = h.toLowerCase();
|
||||||
|
if (k === 'content-type') return 'image/jpeg';
|
||||||
|
if (k === 'content-length') return String(bytes.byteLength);
|
||||||
|
return null;
|
||||||
|
},
|
||||||
|
},
|
||||||
|
arrayBuffer: async () => bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Globals are mocked too so buildPhotoFormData (which uses global fetch) can be
|
||||||
|
// intercepted by the same single mock.
|
||||||
|
let mockNodeFetch;
|
||||||
|
let mockGlobalFetch;
|
||||||
|
let send;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
// Reset modules to get a fresh import with our mocks applied.
|
||||||
|
vi.resetModules();
|
||||||
|
const nodeFetchMod = await import('node-fetch');
|
||||||
|
mockNodeFetch = nodeFetchMod.default;
|
||||||
|
mockNodeFetch.mockReset();
|
||||||
|
|
||||||
|
mockGlobalFetch = vi.fn();
|
||||||
|
vi.stubGlobal('fetch', mockGlobalFetch);
|
||||||
|
|
||||||
|
({ send } = await import('../../lib/notification/adapter/telegram.js'));
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.unstubAllGlobals();
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
|
||||||
|
const baseConfig = {
|
||||||
|
id: 'telegram',
|
||||||
|
fields: { token: 'TKN', chatId: '999' },
|
||||||
|
};
|
||||||
|
|
||||||
|
describe('telegram send() - HTTP URL path (default for .jpg / .png)', () => {
|
||||||
|
it('POSTs JSON to sendPhoto for a .jpg image URL', async () => {
|
||||||
|
mockNodeFetch.mockResolvedValueOnce(jsonOk());
|
||||||
|
|
||||||
|
await send({
|
||||||
|
serviceName: 'immowelt',
|
||||||
|
newListings: [
|
||||||
|
{
|
||||||
|
id: 'a',
|
||||||
|
title: 'Listing',
|
||||||
|
link: 'https://example.com/a',
|
||||||
|
address: 'Addr',
|
||||||
|
price: '500€',
|
||||||
|
size: '50m²',
|
||||||
|
image: 'https://mms.immowelt.de/x/y/z/w/abc.jpg?ci_seal=hash&w=525&h=394',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
notificationConfig: [baseConfig],
|
||||||
|
jobKey: 'Berlin',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(mockNodeFetch).toHaveBeenCalledTimes(1);
|
||||||
|
const [url, opts] = mockNodeFetch.mock.calls[0];
|
||||||
|
expect(url).toBe('https://api.telegram.org/botTKN/sendPhoto');
|
||||||
|
expect(opts.method).toBe('post');
|
||||||
|
expect(opts.headers?.['Content-Type']).toBe('application/json');
|
||||||
|
const body = JSON.parse(opts.body);
|
||||||
|
expect(body.chat_id).toBe('999');
|
||||||
|
expect(body.photo).toBe('https://mms.immowelt.de/x/y/z/w/abc.jpg?ci_seal=hash&w=525&h=394');
|
||||||
|
expect(body.parse_mode).toBe('HTML');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does NOT pre-fetch the image when using HTTP URL path', async () => {
|
||||||
|
mockNodeFetch.mockResolvedValueOnce(jsonOk());
|
||||||
|
|
||||||
|
await send({
|
||||||
|
serviceName: 'immowelt',
|
||||||
|
newListings: [
|
||||||
|
{
|
||||||
|
id: 'a',
|
||||||
|
title: 't',
|
||||||
|
link: 'l',
|
||||||
|
address: 'a',
|
||||||
|
price: '',
|
||||||
|
size: '',
|
||||||
|
image: 'https://example.com/x.jpg',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
notificationConfig: [baseConfig],
|
||||||
|
jobKey: 'Berlin',
|
||||||
|
});
|
||||||
|
|
||||||
|
// global fetch (used by buildPhotoFormData) must not be called
|
||||||
|
expect(mockGlobalFetch).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('falls back to sendMessage when sendPhoto fails', async () => {
|
||||||
|
mockNodeFetch
|
||||||
|
.mockResolvedValueOnce(jsonErr(400, { ok: false, description: 'boom' }))
|
||||||
|
.mockResolvedValueOnce(jsonOk());
|
||||||
|
|
||||||
|
await send({
|
||||||
|
serviceName: 'immowelt',
|
||||||
|
newListings: [
|
||||||
|
{
|
||||||
|
id: 'a',
|
||||||
|
title: 't',
|
||||||
|
link: 'l',
|
||||||
|
address: 'a',
|
||||||
|
price: '',
|
||||||
|
size: '',
|
||||||
|
image: 'https://example.com/x.jpg',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
notificationConfig: [baseConfig],
|
||||||
|
jobKey: 'Berlin',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(mockNodeFetch).toHaveBeenCalledTimes(2);
|
||||||
|
expect(mockNodeFetch.mock.calls[0][0]).toBe('https://api.telegram.org/botTKN/sendPhoto');
|
||||||
|
expect(mockNodeFetch.mock.calls[1][0]).toBe('https://api.telegram.org/botTKN/sendMessage');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('telegram send() - multipart path (.webp URLs)', () => {
|
||||||
|
it('pre-fetches the image then POSTs FormData to sendPhoto for a .webp URL', async () => {
|
||||||
|
// 1st: GET image via global fetch
|
||||||
|
mockGlobalFetch.mockResolvedValueOnce(imageOk());
|
||||||
|
// 2nd: POST sendPhoto via node-fetch
|
||||||
|
mockNodeFetch.mockResolvedValueOnce(jsonOk());
|
||||||
|
|
||||||
|
await send({
|
||||||
|
serviceName: 'immowelt',
|
||||||
|
newListings: [
|
||||||
|
{
|
||||||
|
id: 'a',
|
||||||
|
title: 'Listing',
|
||||||
|
link: 'https://example.com/a',
|
||||||
|
address: 'Addr',
|
||||||
|
price: '500€',
|
||||||
|
size: '50m²',
|
||||||
|
image: 'https://mms.immowelt.de/1/1/6/5/abc.webp?ci_seal=hash&w=525&h=394',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
notificationConfig: [baseConfig],
|
||||||
|
jobKey: 'Berlin',
|
||||||
|
});
|
||||||
|
|
||||||
|
// image was fetched
|
||||||
|
expect(mockGlobalFetch).toHaveBeenCalledTimes(1);
|
||||||
|
expect(mockGlobalFetch.mock.calls[0][0]).toBe('https://mms.immowelt.de/1/1/6/5/abc.webp?ci_seal=hash&w=525&h=394');
|
||||||
|
|
||||||
|
// sendPhoto called via node-fetch with FormData
|
||||||
|
expect(mockNodeFetch).toHaveBeenCalledTimes(1);
|
||||||
|
const [url, opts] = mockNodeFetch.mock.calls[0];
|
||||||
|
expect(url).toBe('https://api.telegram.org/botTKN/sendPhoto');
|
||||||
|
expect(opts.method).toBe('post');
|
||||||
|
expect(opts.body).toBeInstanceOf(FormData);
|
||||||
|
// No explicit Content-Type header - fetch sets multipart boundary itself
|
||||||
|
expect(opts.headers).toBeUndefined();
|
||||||
|
expect(opts.body.get('chat_id')).toBe('999');
|
||||||
|
expect(opts.body.get('parse_mode')).toBe('HTML');
|
||||||
|
const photo = opts.body.get('photo');
|
||||||
|
expect(photo).toBeTruthy();
|
||||||
|
expect(photo.size).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('falls back to sendMessage when the image pre-fetch fails for a .webp URL', async () => {
|
||||||
|
// image fetch fails (404 from CDN)
|
||||||
|
mockGlobalFetch.mockResolvedValueOnce({
|
||||||
|
ok: false,
|
||||||
|
status: 404,
|
||||||
|
headers: { get: () => null },
|
||||||
|
arrayBuffer: async () => new ArrayBuffer(0),
|
||||||
|
});
|
||||||
|
// then sendMessage succeeds via node-fetch
|
||||||
|
mockNodeFetch.mockResolvedValueOnce(jsonOk());
|
||||||
|
|
||||||
|
await send({
|
||||||
|
serviceName: 'immowelt',
|
||||||
|
newListings: [
|
||||||
|
{
|
||||||
|
id: 'a',
|
||||||
|
title: 't',
|
||||||
|
link: 'l',
|
||||||
|
address: 'a',
|
||||||
|
price: '',
|
||||||
|
size: '',
|
||||||
|
image: 'https://example.com/gone.webp',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
notificationConfig: [baseConfig],
|
||||||
|
jobKey: 'Berlin',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(mockNodeFetch).toHaveBeenCalledTimes(1);
|
||||||
|
expect(mockNodeFetch.mock.calls[0][0]).toBe('https://api.telegram.org/botTKN/sendMessage');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('falls back to sendMessage when multipart sendPhoto returns a Telegram error', async () => {
|
||||||
|
mockGlobalFetch.mockResolvedValueOnce(imageOk());
|
||||||
|
mockNodeFetch
|
||||||
|
.mockResolvedValueOnce(jsonErr(400, { description: 'broke' })) // multipart sendPhoto
|
||||||
|
.mockResolvedValueOnce(jsonOk()); // sendMessage fallback
|
||||||
|
|
||||||
|
await send({
|
||||||
|
serviceName: 'immowelt',
|
||||||
|
newListings: [
|
||||||
|
{
|
||||||
|
id: 'a',
|
||||||
|
title: 't',
|
||||||
|
link: 'l',
|
||||||
|
address: 'a',
|
||||||
|
price: '',
|
||||||
|
size: '',
|
||||||
|
image: 'https://example.com/x.webp',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
notificationConfig: [baseConfig],
|
||||||
|
jobKey: 'Berlin',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(mockNodeFetch).toHaveBeenCalledTimes(2);
|
||||||
|
expect(mockNodeFetch.mock.calls[1][0]).toBe('https://api.telegram.org/botTKN/sendMessage');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('telegram send() - mixed batch (regression-safety)', () => {
|
||||||
|
it('handles a batch with both .jpg and .webp - jpg uses URL, webp uses multipart', async () => {
|
||||||
|
// .webp image fetch
|
||||||
|
mockGlobalFetch.mockResolvedValueOnce(imageOk());
|
||||||
|
// both sendPhoto calls succeed
|
||||||
|
mockNodeFetch
|
||||||
|
.mockResolvedValueOnce(jsonOk()) // could be either listing first
|
||||||
|
.mockResolvedValueOnce(jsonOk());
|
||||||
|
|
||||||
|
await send({
|
||||||
|
serviceName: 'immowelt',
|
||||||
|
newListings: [
|
||||||
|
{
|
||||||
|
id: 'jpg-listing',
|
||||||
|
title: 'a',
|
||||||
|
link: 'l',
|
||||||
|
address: 'a',
|
||||||
|
price: '',
|
||||||
|
size: '',
|
||||||
|
image: 'https://example.com/a.jpg',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'webp-listing',
|
||||||
|
title: 'b',
|
||||||
|
link: 'l',
|
||||||
|
address: 'a',
|
||||||
|
price: '',
|
||||||
|
size: '',
|
||||||
|
image: 'https://example.com/b.webp',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
notificationConfig: [baseConfig],
|
||||||
|
jobKey: 'Berlin',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(mockGlobalFetch).toHaveBeenCalledTimes(1); // only webp pre-fetches
|
||||||
|
expect(mockNodeFetch).toHaveBeenCalledTimes(2);
|
||||||
|
|
||||||
|
// Verify one call had FormData and one had JSON body
|
||||||
|
const bodies = mockNodeFetch.mock.calls.map((c) => c[1].body);
|
||||||
|
const hasFormData = bodies.some((b) => b instanceof FormData);
|
||||||
|
const hasJson = bodies.some((b) => typeof b === 'string' && b.startsWith('{'));
|
||||||
|
expect(hasFormData).toBe(true);
|
||||||
|
expect(hasJson).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('uses sendMessage (not sendPhoto) when image is null', async () => {
|
||||||
|
mockNodeFetch.mockResolvedValueOnce(jsonOk());
|
||||||
|
|
||||||
|
await send({
|
||||||
|
serviceName: 'immowelt',
|
||||||
|
newListings: [
|
||||||
|
{
|
||||||
|
id: 'a',
|
||||||
|
title: 't',
|
||||||
|
link: 'l',
|
||||||
|
address: 'a',
|
||||||
|
price: '',
|
||||||
|
size: '',
|
||||||
|
image: null,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
notificationConfig: [baseConfig],
|
||||||
|
jobKey: 'Berlin',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(mockNodeFetch).toHaveBeenCalledTimes(1);
|
||||||
|
expect(mockNodeFetch.mock.calls[0][0]).toBe('https://api.telegram.org/botTKN/sendMessage');
|
||||||
|
expect(mockGlobalFetch).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('telegram send() - multiple chat IDs', () => {
|
||||||
|
const listing = {
|
||||||
|
id: '1',
|
||||||
|
title: 'Flat',
|
||||||
|
link: 'https://ex.com',
|
||||||
|
address: 'Berlin',
|
||||||
|
price: '800',
|
||||||
|
size: '50',
|
||||||
|
image: 'https://ex.com/img.jpg',
|
||||||
|
};
|
||||||
|
|
||||||
|
it('sends to every chat ID in a comma-separated list', async () => {
|
||||||
|
mockNodeFetch.mockResolvedValue(jsonOk());
|
||||||
|
|
||||||
|
await send({
|
||||||
|
serviceName: 'immoscout',
|
||||||
|
newListings: [listing],
|
||||||
|
notificationConfig: [{ id: 'telegram', fields: { token: 'TKN', chatId: '111, 222' } }],
|
||||||
|
jobKey: 'Berlin',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(mockNodeFetch).toHaveBeenCalledTimes(2);
|
||||||
|
const bodies = mockNodeFetch.mock.calls.map((c) => JSON.parse(c[1].body));
|
||||||
|
expect(bodies.map((b) => b.chat_id)).toEqual(expect.arrayContaining(['111', '222']));
|
||||||
|
});
|
||||||
|
|
||||||
|
it('trims whitespace around each chat ID', async () => {
|
||||||
|
mockNodeFetch.mockResolvedValue(jsonOk());
|
||||||
|
|
||||||
|
await send({
|
||||||
|
serviceName: 'immoscout',
|
||||||
|
newListings: [listing],
|
||||||
|
notificationConfig: [{ id: 'telegram', fields: { token: 'TKN', chatId: ' 333 , 444 ' } }],
|
||||||
|
jobKey: 'Berlin',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(mockNodeFetch).toHaveBeenCalledTimes(2);
|
||||||
|
const bodies = mockNodeFetch.mock.calls.map((c) => JSON.parse(c[1].body));
|
||||||
|
expect(bodies.map((b) => b.chat_id)).toEqual(expect.arrayContaining(['333', '444']));
|
||||||
|
});
|
||||||
|
|
||||||
|
it('sends each listing to each chat ID (N listings × M chats)', async () => {
|
||||||
|
mockNodeFetch.mockResolvedValue(jsonOk());
|
||||||
|
|
||||||
|
await send({
|
||||||
|
serviceName: 'immoscout',
|
||||||
|
newListings: [listing, { ...listing, id: '2' }],
|
||||||
|
notificationConfig: [{ id: 'telegram', fields: { token: 'TKN', chatId: '555, 666' } }],
|
||||||
|
jobKey: 'Berlin',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(mockNodeFetch).toHaveBeenCalledTimes(4);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('telegram send() - config validation', () => {
|
||||||
|
it('throws when telegram adapter config is missing', () => {
|
||||||
|
expect(() =>
|
||||||
|
send({
|
||||||
|
serviceName: 's',
|
||||||
|
newListings: [],
|
||||||
|
notificationConfig: [],
|
||||||
|
jobKey: 'k',
|
||||||
|
}),
|
||||||
|
).toThrow(/configuration missing/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('throws when token or chatId is missing', () => {
|
||||||
|
expect(() =>
|
||||||
|
send({
|
||||||
|
serviceName: 's',
|
||||||
|
newListings: [],
|
||||||
|
notificationConfig: [{ id: 'telegram', fields: { token: '' } }],
|
||||||
|
jobKey: 'k',
|
||||||
|
}),
|
||||||
|
).toThrow(/token.*chatId/);
|
||||||
|
});
|
||||||
|
});
|
||||||
287
test/notification/telegramPhotoUploader.test.js
Normal file
287
test/notification/telegramPhotoUploader.test.js
Normal file
@@ -0,0 +1,287 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (c) 2026 by Christian Kellner.
|
||||||
|
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||||
|
import { shouldUseMultipart, buildPhotoFormData } from '../../lib/notification/adapter/telegramPhotoUploader.js';
|
||||||
|
|
||||||
|
describe('shouldUseMultipart', () => {
|
||||||
|
it('returns true for .webp URL with query string', () => {
|
||||||
|
expect(shouldUseMultipart('https://mms.immowelt.de/1/1/6/5/abc.webp?ci_seal=hash&w=525&h=394')).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns true for .webp URL without query string', () => {
|
||||||
|
expect(shouldUseMultipart('https://example.com/photo.webp')).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns true for uppercase .WEBP extension', () => {
|
||||||
|
expect(shouldUseMultipart('https://example.com/IMG.WEBP?x=1')).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns false for .jpg URL with query string', () => {
|
||||||
|
expect(shouldUseMultipart('https://mms.immowelt.de/a/b/c/d/xyz.jpg?ci_seal=hash&w=525&h=394')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns false for .jpeg URL', () => {
|
||||||
|
expect(shouldUseMultipart('https://example.com/photo.jpeg')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns false for .png URL with query string', () => {
|
||||||
|
expect(shouldUseMultipart('https://example.com/photo.png?w=100')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns false for .gif URL', () => {
|
||||||
|
expect(shouldUseMultipart('https://example.com/photo.gif')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns false for null', () => {
|
||||||
|
expect(shouldUseMultipart(null)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns false for undefined', () => {
|
||||||
|
expect(shouldUseMultipart(undefined)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns false for empty string', () => {
|
||||||
|
expect(shouldUseMultipart('')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns false for malformed URL', () => {
|
||||||
|
expect(shouldUseMultipart('not a url')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns false for URL where webp is in the query but not the path', () => {
|
||||||
|
expect(shouldUseMultipart('https://example.com/photo.jpg?format=webp')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns false for URL with no extension at all', () => {
|
||||||
|
expect(shouldUseMultipart('https://example.com/photo')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns false for non-https schemes', () => {
|
||||||
|
// file/data/ftp URLs should not be relevant; safer to skip multipart
|
||||||
|
expect(shouldUseMultipart('http://example.com/photo.webp')).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('buildPhotoFormData', () => {
|
||||||
|
let mockFetch;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
mockFetch = vi.fn();
|
||||||
|
vi.stubGlobal('fetch', mockFetch);
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.unstubAllGlobals();
|
||||||
|
});
|
||||||
|
|
||||||
|
function makeImageResponse({ contentType = 'image/jpeg', bytes = new Uint8Array([0xff, 0xd8, 0xff]) } = {}) {
|
||||||
|
const buf = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
headers: {
|
||||||
|
get: (h) =>
|
||||||
|
h.toLowerCase() === 'content-type'
|
||||||
|
? contentType
|
||||||
|
: h.toLowerCase() === 'content-length'
|
||||||
|
? String(bytes.byteLength)
|
||||||
|
: null,
|
||||||
|
},
|
||||||
|
arrayBuffer: async () => buf,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
it('fetches image with Accept header that excludes webp so the CDN transcodes to JPEG', async () => {
|
||||||
|
mockFetch.mockResolvedValueOnce(makeImageResponse());
|
||||||
|
|
||||||
|
await buildPhotoFormData({
|
||||||
|
chatId: '123',
|
||||||
|
imageUrl: 'https://example.com/photo.webp',
|
||||||
|
caption: 'hi',
|
||||||
|
parseMode: 'HTML',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(mockFetch).toHaveBeenCalledTimes(1);
|
||||||
|
const [url, opts] = mockFetch.mock.calls[0];
|
||||||
|
expect(url).toBe('https://example.com/photo.webp');
|
||||||
|
expect(opts?.headers?.Accept || opts?.headers?.accept).toMatch(/image\/jpeg/);
|
||||||
|
expect(opts?.headers?.Accept || opts?.headers?.accept).not.toMatch(/image\/webp/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns FormData containing chat_id, caption, parse_mode, and photo fields', async () => {
|
||||||
|
mockFetch.mockResolvedValueOnce(makeImageResponse());
|
||||||
|
|
||||||
|
const fd = await buildPhotoFormData({
|
||||||
|
chatId: '12345',
|
||||||
|
imageUrl: 'https://example.com/abc.webp',
|
||||||
|
caption: 'My caption',
|
||||||
|
parseMode: 'HTML',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(fd).toBeInstanceOf(FormData);
|
||||||
|
expect(fd.get('chat_id')).toBe('12345');
|
||||||
|
expect(fd.get('caption')).toBe('My caption');
|
||||||
|
expect(fd.get('parse_mode')).toBe('HTML');
|
||||||
|
const photo = fd.get('photo');
|
||||||
|
expect(photo).toBeTruthy();
|
||||||
|
// File-like (Blob); has a name and a size
|
||||||
|
expect(typeof photo.name).toBe('string');
|
||||||
|
expect(photo.size).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('uses a .jpg filename (Telegram uses URL/filename extension for type detection)', async () => {
|
||||||
|
mockFetch.mockResolvedValueOnce(makeImageResponse());
|
||||||
|
|
||||||
|
const fd = await buildPhotoFormData({
|
||||||
|
chatId: '1',
|
||||||
|
imageUrl: 'https://example.com/source.webp',
|
||||||
|
caption: 'c',
|
||||||
|
parseMode: 'HTML',
|
||||||
|
});
|
||||||
|
|
||||||
|
const photo = fd.get('photo');
|
||||||
|
expect(photo.name).toMatch(/\.jpg$/i);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('includes message_thread_id when provided', async () => {
|
||||||
|
mockFetch.mockResolvedValueOnce(makeImageResponse());
|
||||||
|
|
||||||
|
const fd = await buildPhotoFormData({
|
||||||
|
chatId: '1',
|
||||||
|
imageUrl: 'https://example.com/source.webp',
|
||||||
|
caption: 'c',
|
||||||
|
parseMode: 'HTML',
|
||||||
|
messageThreadId: 42,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(fd.get('message_thread_id')).toBe('42');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('omits message_thread_id when not provided', async () => {
|
||||||
|
mockFetch.mockResolvedValueOnce(makeImageResponse());
|
||||||
|
|
||||||
|
const fd = await buildPhotoFormData({
|
||||||
|
chatId: '1',
|
||||||
|
imageUrl: 'https://example.com/source.webp',
|
||||||
|
caption: 'c',
|
||||||
|
parseMode: 'HTML',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(fd.get('message_thread_id')).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('omits parse_mode when not provided (plain text mode)', async () => {
|
||||||
|
mockFetch.mockResolvedValueOnce(makeImageResponse());
|
||||||
|
|
||||||
|
const fd = await buildPhotoFormData({
|
||||||
|
chatId: '1',
|
||||||
|
imageUrl: 'https://example.com/source.webp',
|
||||||
|
caption: 'c',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(fd.get('parse_mode')).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('throws when the image fetch returns non-200', async () => {
|
||||||
|
mockFetch.mockResolvedValueOnce({
|
||||||
|
ok: false,
|
||||||
|
status: 404,
|
||||||
|
headers: { get: () => null },
|
||||||
|
arrayBuffer: async () => new ArrayBuffer(0),
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
buildPhotoFormData({
|
||||||
|
chatId: '1',
|
||||||
|
imageUrl: 'https://example.com/gone.webp',
|
||||||
|
caption: 'c',
|
||||||
|
parseMode: 'HTML',
|
||||||
|
}),
|
||||||
|
).rejects.toThrow(/404/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('throws when the image fetch throws (network error)', async () => {
|
||||||
|
mockFetch.mockRejectedValueOnce(new Error('ECONNREFUSED'));
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
buildPhotoFormData({
|
||||||
|
chatId: '1',
|
||||||
|
imageUrl: 'https://example.com/x.webp',
|
||||||
|
caption: 'c',
|
||||||
|
parseMode: 'HTML',
|
||||||
|
}),
|
||||||
|
).rejects.toThrow(/ECONNREFUSED/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('throws when the image exceeds 10 MB (Telegram multipart limit)', async () => {
|
||||||
|
// 11 MB
|
||||||
|
const big = new Uint8Array(11 * 1024 * 1024);
|
||||||
|
mockFetch.mockResolvedValueOnce(makeImageResponse({ bytes: big }));
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
buildPhotoFormData({
|
||||||
|
chatId: '1',
|
||||||
|
imageUrl: 'https://example.com/huge.webp',
|
||||||
|
caption: 'c',
|
||||||
|
parseMode: 'HTML',
|
||||||
|
}),
|
||||||
|
).rejects.toThrow(/size|large|10/i);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects early when content-length header advertises > 10 MB (avoids download)', async () => {
|
||||||
|
mockFetch.mockResolvedValueOnce({
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
headers: {
|
||||||
|
get: (h) => {
|
||||||
|
const k = h.toLowerCase();
|
||||||
|
if (k === 'content-type') return 'image/jpeg';
|
||||||
|
if (k === 'content-length') return String(50 * 1024 * 1024);
|
||||||
|
return null;
|
||||||
|
},
|
||||||
|
},
|
||||||
|
arrayBuffer: async () => {
|
||||||
|
throw new Error('should not be called - size check should reject first');
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
buildPhotoFormData({
|
||||||
|
chatId: '1',
|
||||||
|
imageUrl: 'https://example.com/huge.webp',
|
||||||
|
caption: 'c',
|
||||||
|
parseMode: 'HTML',
|
||||||
|
}),
|
||||||
|
).rejects.toThrow(/size|large|10/i);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts exactly 10 MB images (boundary)', async () => {
|
||||||
|
const bytes = new Uint8Array(10 * 1024 * 1024);
|
||||||
|
mockFetch.mockResolvedValueOnce(makeImageResponse({ bytes }));
|
||||||
|
|
||||||
|
const fd = await buildPhotoFormData({
|
||||||
|
chatId: '1',
|
||||||
|
imageUrl: 'https://example.com/exact.webp',
|
||||||
|
caption: 'c',
|
||||||
|
parseMode: 'HTML',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(fd.get('photo').size).toBe(10 * 1024 * 1024);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('coerces non-string chatId (number) to string in form data', async () => {
|
||||||
|
mockFetch.mockResolvedValueOnce(makeImageResponse());
|
||||||
|
|
||||||
|
const fd = await buildPhotoFormData({
|
||||||
|
chatId: 999,
|
||||||
|
imageUrl: 'https://example.com/x.webp',
|
||||||
|
caption: 'c',
|
||||||
|
parseMode: 'HTML',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(fd.get('chat_id')).toBe('999');
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -38,6 +38,20 @@ async function tryReadFile(filepath) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function withRealEstateType(data, realEstateType) {
|
||||||
|
if (!realEstateType?.length || !Array.isArray(data?.resultListItems)) {
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
const cloned = typeof structuredClone === 'function' ? structuredClone(data) : JSON.parse(JSON.stringify(data));
|
||||||
|
for (const item of cloned.resultListItems) {
|
||||||
|
if (item?.type === 'EXPOSE_RESULT' && item?.item) {
|
||||||
|
item.item.realEstateType = realEstateType;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return cloned;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns fixture HTML for the given URL by mapping hostname → provider name,
|
* Returns fixture HTML for the given URL by mapping hostname → provider name,
|
||||||
* then distinguishing list vs detail pages by comparing the URL path against
|
* then distinguishing list vs detail pages by comparing the URL path against
|
||||||
@@ -83,7 +97,10 @@ export function buildFetchMock() {
|
|||||||
const raw = await tryReadFile(path.join(FIXTURES_DIR, 'immoscout_list.json'));
|
const raw = await tryReadFile(path.join(FIXTURES_DIR, 'immoscout_list.json'));
|
||||||
listData = raw ? JSON.parse(raw) : { resultListItems: [] };
|
listData = raw ? JSON.parse(raw) : { resultListItems: [] };
|
||||||
}
|
}
|
||||||
return { ok: true, status: 200, json: () => Promise.resolve(listData) };
|
|
||||||
|
const requestedType = new URL(urlStr).searchParams.get('realestatetype');
|
||||||
|
const responseData = withRealEstateType(listData, requestedType);
|
||||||
|
return { ok: true, status: 200, json: () => Promise.resolve(responseData) };
|
||||||
}
|
}
|
||||||
|
|
||||||
if (urlStr.includes('api.mobile.immobilienscout24.de/expose/')) {
|
if (urlStr.includes('api.mobile.immobilienscout24.de/expose/')) {
|
||||||
|
|||||||
@@ -3,9 +3,10 @@
|
|||||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { expect } from 'vitest';
|
import { afterEach, expect } from 'vitest';
|
||||||
import { mockFredy } from './utils.js';
|
import { mockFredy } from './utils.js';
|
||||||
import * as mockStore from './mocks/mockStore.js';
|
import * as mockStore from './mocks/mockStore.js';
|
||||||
|
import { get as getLastNotification } from './mocks/mockNotification.js';
|
||||||
|
|
||||||
describe('Issue reproduction: listings filtered by similarity or area should be marked as manually deleted', () => {
|
describe('Issue reproduction: listings filtered by similarity or area should be marked as manually deleted', () => {
|
||||||
it('should call deleteListingsById when listings are filtered by similarity', async () => {
|
it('should call deleteListingsById when listings are filtered by similarity', async () => {
|
||||||
@@ -113,3 +114,223 @@ describe('Issue reproduction: listings filtered by similarity or area should be
|
|||||||
expect(mockStore.deletedIds).toContain('2');
|
expect(mockStore.deletedIds).toContain('2');
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('Blacklist is re-applied after detail enrichment', () => {
|
||||||
|
afterEach(() => {
|
||||||
|
mockStore.setUserSettings(null);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('filters out a listing whose blacklisted term only appears in the enriched description', async () => {
|
||||||
|
const Fredy = await mockFredy();
|
||||||
|
const providerId = 'test-provider';
|
||||||
|
|
||||||
|
mockStore.setUserSettings({
|
||||||
|
provider_details: [providerId],
|
||||||
|
blacklist_filter_on_provider_details: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
const mockSimilarityCache = {
|
||||||
|
checkAndAddEntry: () => false,
|
||||||
|
};
|
||||||
|
|
||||||
|
const blacklist = ['allkauf'];
|
||||||
|
|
||||||
|
// The search results page returns a clean snippet (no blacklisted term).
|
||||||
|
// fetchDetails simulates loading the full detail page and discovers the
|
||||||
|
// blacklisted term hidden deep in the description.
|
||||||
|
const providerConfig = {
|
||||||
|
url: 'http://example.com',
|
||||||
|
getListings: () =>
|
||||||
|
Promise.resolve([
|
||||||
|
{
|
||||||
|
id: 'kept',
|
||||||
|
title: 'Nice house',
|
||||||
|
address: 'Some street',
|
||||||
|
price: '500000',
|
||||||
|
link: 'http://example.com/kept',
|
||||||
|
description: 'Cozy home with garden',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'blacklisted',
|
||||||
|
title: 'Eleganz trifft Raumkomfort',
|
||||||
|
address: 'Other street',
|
||||||
|
price: '600000',
|
||||||
|
link: 'http://example.com/blacklisted',
|
||||||
|
description: 'Eleganz trifft Raumkomfort',
|
||||||
|
},
|
||||||
|
]),
|
||||||
|
normalize: (l) => l,
|
||||||
|
filter: (l) => {
|
||||||
|
const text = `${l.title ?? ''} ${l.description ?? ''}`.toLowerCase();
|
||||||
|
return !blacklist.some((term) => text.includes(term));
|
||||||
|
},
|
||||||
|
fetchDetails: (listing) => {
|
||||||
|
if (listing.id === 'blacklisted') {
|
||||||
|
return Promise.resolve({
|
||||||
|
...listing,
|
||||||
|
description: 'Mit allkauf Haus wird dein Traum vom Eigenheim wahr.',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return Promise.resolve(listing);
|
||||||
|
},
|
||||||
|
crawlFields: {
|
||||||
|
id: 'id',
|
||||||
|
title: 'title',
|
||||||
|
address: 'address',
|
||||||
|
price: 'price',
|
||||||
|
link: 'link',
|
||||||
|
description: 'description',
|
||||||
|
},
|
||||||
|
requiredFieldNames: ['id', 'title', 'address', 'price', 'link', 'description'],
|
||||||
|
};
|
||||||
|
|
||||||
|
const mockedJob = {
|
||||||
|
id: 'blacklist-test-job',
|
||||||
|
notificationAdapter: null,
|
||||||
|
specFilter: null,
|
||||||
|
spatialFilter: null,
|
||||||
|
};
|
||||||
|
|
||||||
|
const fredy = new Fredy(providerConfig, mockedJob, providerId, mockSimilarityCache, undefined);
|
||||||
|
|
||||||
|
const result = await fredy.execute();
|
||||||
|
|
||||||
|
expect(result).toBeInstanceOf(Array);
|
||||||
|
const ids = result.map((l) => l.id);
|
||||||
|
expect(ids).toContain('kept');
|
||||||
|
expect(ids).not.toContain('blacklisted');
|
||||||
|
|
||||||
|
const notification = getLastNotification();
|
||||||
|
const notifiedIds = (notification?.payload ?? []).map((p) => p.id);
|
||||||
|
expect(notifiedIds).not.toContain('blacklisted');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('short-circuits the pipeline when all listings get blacklisted after enrichment', async () => {
|
||||||
|
const Fredy = await mockFredy();
|
||||||
|
const providerId = 'all-blacklisted-provider';
|
||||||
|
|
||||||
|
mockStore.setUserSettings({
|
||||||
|
provider_details: [providerId],
|
||||||
|
blacklist_filter_on_provider_details: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
const mockSimilarityCache = {
|
||||||
|
checkAndAddEntry: () => false,
|
||||||
|
};
|
||||||
|
|
||||||
|
const blacklist = ['allkauf'];
|
||||||
|
|
||||||
|
const providerConfig = {
|
||||||
|
url: 'http://example.com',
|
||||||
|
getListings: () =>
|
||||||
|
Promise.resolve([
|
||||||
|
{
|
||||||
|
id: 'only',
|
||||||
|
title: 'Eleganz trifft Raumkomfort',
|
||||||
|
address: 'Some street',
|
||||||
|
price: '700000',
|
||||||
|
link: 'http://example.com/only',
|
||||||
|
description: 'Eleganz trifft Raumkomfort',
|
||||||
|
},
|
||||||
|
]),
|
||||||
|
normalize: (l) => l,
|
||||||
|
filter: (l) => {
|
||||||
|
const text = `${l.title ?? ''} ${l.description ?? ''}`.toLowerCase();
|
||||||
|
return !blacklist.some((term) => text.includes(term));
|
||||||
|
},
|
||||||
|
fetchDetails: (listing) =>
|
||||||
|
Promise.resolve({
|
||||||
|
...listing,
|
||||||
|
description: 'Mit allkauf Haus wird dein Traum vom Eigenheim wahr.',
|
||||||
|
}),
|
||||||
|
crawlFields: {
|
||||||
|
id: 'id',
|
||||||
|
title: 'title',
|
||||||
|
address: 'address',
|
||||||
|
price: 'price',
|
||||||
|
link: 'link',
|
||||||
|
description: 'description',
|
||||||
|
},
|
||||||
|
requiredFieldNames: ['id', 'title', 'address', 'price', 'link', 'description'],
|
||||||
|
};
|
||||||
|
|
||||||
|
const mockedJob = {
|
||||||
|
id: 'all-blacklisted-job',
|
||||||
|
notificationAdapter: null,
|
||||||
|
specFilter: null,
|
||||||
|
spatialFilter: null,
|
||||||
|
};
|
||||||
|
|
||||||
|
const fredy = new Fredy(providerConfig, mockedJob, providerId, mockSimilarityCache, undefined);
|
||||||
|
|
||||||
|
// Should resolve to undefined (NoNewListingsWarning is caught in _handleError).
|
||||||
|
const result = await fredy.execute();
|
||||||
|
expect(result).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does NOT re-filter when blacklist_filter_on_provider_details is disabled', async () => {
|
||||||
|
const Fredy = await mockFredy();
|
||||||
|
const providerId = 'opt-out-provider';
|
||||||
|
|
||||||
|
// provider_details enabled (so fetchDetails runs) but blacklist re-filter NOT enabled.
|
||||||
|
mockStore.setUserSettings({
|
||||||
|
provider_details: [providerId],
|
||||||
|
blacklist_filter_on_provider_details: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
const mockSimilarityCache = {
|
||||||
|
checkAndAddEntry: () => false,
|
||||||
|
};
|
||||||
|
|
||||||
|
const blacklist = ['allkauf'];
|
||||||
|
|
||||||
|
const providerConfig = {
|
||||||
|
url: 'http://example.com',
|
||||||
|
getListings: () =>
|
||||||
|
Promise.resolve([
|
||||||
|
{
|
||||||
|
id: 'leaks-through',
|
||||||
|
title: 'Eleganz trifft Raumkomfort',
|
||||||
|
address: 'Other street',
|
||||||
|
price: '600000',
|
||||||
|
link: 'http://example.com/leaks-through',
|
||||||
|
description: 'Eleganz trifft Raumkomfort',
|
||||||
|
},
|
||||||
|
]),
|
||||||
|
normalize: (l) => l,
|
||||||
|
filter: (l) => {
|
||||||
|
const text = `${l.title ?? ''} ${l.description ?? ''}`.toLowerCase();
|
||||||
|
return !blacklist.some((term) => text.includes(term));
|
||||||
|
},
|
||||||
|
fetchDetails: (listing) =>
|
||||||
|
Promise.resolve({
|
||||||
|
...listing,
|
||||||
|
description: 'Mit allkauf Haus wird dein Traum vom Eigenheim wahr.',
|
||||||
|
}),
|
||||||
|
crawlFields: {
|
||||||
|
id: 'id',
|
||||||
|
title: 'title',
|
||||||
|
address: 'address',
|
||||||
|
price: 'price',
|
||||||
|
link: 'link',
|
||||||
|
description: 'description',
|
||||||
|
},
|
||||||
|
requiredFieldNames: ['id', 'title', 'address', 'price', 'link', 'description'],
|
||||||
|
};
|
||||||
|
|
||||||
|
const mockedJob = {
|
||||||
|
id: 'opt-out-job',
|
||||||
|
notificationAdapter: null,
|
||||||
|
specFilter: null,
|
||||||
|
spatialFilter: null,
|
||||||
|
};
|
||||||
|
|
||||||
|
const fredy = new Fredy(providerConfig, mockedJob, providerId, mockSimilarityCache, undefined);
|
||||||
|
|
||||||
|
const result = await fredy.execute();
|
||||||
|
|
||||||
|
// Listing leaks through because user has not opted in to the stricter check.
|
||||||
|
expect(result).toBeInstanceOf(Array);
|
||||||
|
expect(result.map((l) => l.id)).toContain('leaks-through');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -57,13 +57,17 @@ describe('#sparkasse testsuite()', () => {
|
|||||||
expect(notify.id).toBeTypeOf('string');
|
expect(notify.id).toBeTypeOf('string');
|
||||||
expect(notify.price).toBeTypeOf('string');
|
expect(notify.price).toBeTypeOf('string');
|
||||||
expect(notify.price).toContain('€');
|
expect(notify.price).toContain('€');
|
||||||
expect(notify.size).toBeTypeOf('string');
|
// Size can legitimately be absent for a card whose layout shifts the
|
||||||
expect(notify.size).toContain('m²');
|
// value out of the expected slot; when present it must be a formatted
|
||||||
|
// "… m²" string.
|
||||||
|
if (notify.size != null) {
|
||||||
|
expect(notify.size).toBeTypeOf('string');
|
||||||
|
expect(notify.size).toContain('m²');
|
||||||
|
}
|
||||||
expect(notify.title).toBeTypeOf('string');
|
expect(notify.title).toBeTypeOf('string');
|
||||||
expect(notify.link).toBeTypeOf('string');
|
expect(notify.link).toBeTypeOf('string');
|
||||||
expect(notify.address).toBeTypeOf('string');
|
expect(notify.address).toBeTypeOf('string');
|
||||||
/** check the values if possible **/
|
/** check the values if possible **/
|
||||||
expect(notify.size).toBeTypeOf('string');
|
|
||||||
expect(notify.title).not.toBe('');
|
expect(notify.title).not.toBe('');
|
||||||
expect(notify.address).not.toBe('');
|
expect(notify.address).not.toBe('');
|
||||||
});
|
});
|
||||||
|
|||||||
129
test/services/debug/debugBundleService.test.js
Normal file
129
test/services/debug/debugBundleService.test.js
Normal file
@@ -0,0 +1,129 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (c) 2026 by Christian Kellner.
|
||||||
|
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||||
|
import path from 'node:path';
|
||||||
|
|
||||||
|
describe('services/debug/debugBundleService.js', () => {
|
||||||
|
let svc;
|
||||||
|
let storedLogs;
|
||||||
|
let addedZipEntries;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
storedLogs = [];
|
||||||
|
addedZipEntries = [];
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Minimal AdmZip stand-in that records the in-memory entry names + payloads so we
|
||||||
|
* can assert what made it into the bundle without spinning up real zip parsing.
|
||||||
|
*/
|
||||||
|
class MockAdmZip {
|
||||||
|
constructor() {
|
||||||
|
this.entries = [];
|
||||||
|
}
|
||||||
|
addFile(name, buf) {
|
||||||
|
this.entries.push({ entryName: name, data: buf });
|
||||||
|
addedZipEntries.push({ entryName: name, content: buf.toString('utf-8') });
|
||||||
|
}
|
||||||
|
toBuffer() {
|
||||||
|
return Buffer.from(JSON.stringify(this.entries.map((e) => e.entryName)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
globalThis.__TEST_ADM_ZIP__ = MockAdmZip;
|
||||||
|
|
||||||
|
const ROOT = path.resolve('.');
|
||||||
|
const storagePath = path.join(ROOT, 'lib', 'services', 'debug', 'debugLogStorage.js');
|
||||||
|
const utilsPath = path.join(ROOT, 'lib', 'utils.js');
|
||||||
|
|
||||||
|
const storageMock = {
|
||||||
|
getAllDebugLogs: () => storedLogs,
|
||||||
|
};
|
||||||
|
const utilsMock = { getPackageVersion: async () => '22.5.0' };
|
||||||
|
|
||||||
|
vi.resetModules();
|
||||||
|
vi.doMock(storagePath, () => storageMock);
|
||||||
|
vi.doMock(utilsPath, () => utilsMock);
|
||||||
|
|
||||||
|
svc = await import(path.join(ROOT, 'lib', 'services', 'debug', 'debugBundleService.js'));
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
delete globalThis.__TEST_ADM_ZIP__;
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('renderLogsTxt', () => {
|
||||||
|
it('returns an empty string when there are no rows', () => {
|
||||||
|
expect(svc.renderLogsTxt()).toBe('');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('formats each row as [date] LEVEL: message and keeps order', () => {
|
||||||
|
storedLogs.push({ id: 1, ts: 1717855200000, level: 'info', message: 'first line' });
|
||||||
|
storedLogs.push({ id: 2, ts: 1717855201000, level: 'warn', message: 'second line' });
|
||||||
|
|
||||||
|
const out = svc.renderLogsTxt();
|
||||||
|
|
||||||
|
expect(out).toMatch(/\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\] INFO: first line/);
|
||||||
|
expect(out).toMatch(/\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\] WARN: second line/);
|
||||||
|
expect(out.indexOf('first line')).toBeLessThan(out.indexOf('second line'));
|
||||||
|
expect(out.endsWith('\n')).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('buildSystemInfo', () => {
|
||||||
|
it('contains Fredy version, Node version and OS platform', async () => {
|
||||||
|
const sys = await svc.buildSystemInfo({ settings: null });
|
||||||
|
expect(sys).toMatch(/Fredy version:\s+22\.5\.0/);
|
||||||
|
expect(sys).toContain(`Node.js version: ${process.version}`);
|
||||||
|
expect(sys).toContain(`Platform: ${process.platform}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('redacts proxy URL credentials', async () => {
|
||||||
|
const sys = await svc.buildSystemInfo({
|
||||||
|
settings: { proxyUrl: 'http://secret:hunter2@proxy.example:8080', port: 9998 },
|
||||||
|
});
|
||||||
|
expect(sys).not.toContain('hunter2');
|
||||||
|
expect(sys).not.toContain('secret');
|
||||||
|
expect(sys).toContain('proxy.example');
|
||||||
|
expect(sys).toContain('port: 9998');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('strips session secrets from sanitized settings output', async () => {
|
||||||
|
const sys = await svc.buildSystemInfo({
|
||||||
|
settings: { session_secret: 'top-secret', sessionSecret: 'other-secret', port: 9998 },
|
||||||
|
});
|
||||||
|
expect(sys).not.toContain('top-secret');
|
||||||
|
expect(sys).not.toContain('other-secret');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('buildDebugBundleFileName', () => {
|
||||||
|
it('matches YYYY-MM-DD-FredyDebug-<version>.zip', async () => {
|
||||||
|
const name = await svc.buildDebugBundleFileName();
|
||||||
|
expect(name).toMatch(/^\d{4}-\d{2}-\d{2}-FredyDebug-22\.5\.0\.zip$/);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('buildDebugBundleZip', () => {
|
||||||
|
it('always emits both logs.txt and sys.txt entries', async () => {
|
||||||
|
storedLogs.push({ id: 1, ts: 1717855200000, level: 'info', message: 'recorded line' });
|
||||||
|
await svc.buildDebugBundleZip({ settings: { port: 9998 } });
|
||||||
|
|
||||||
|
const names = addedZipEntries.map((e) => e.entryName).sort();
|
||||||
|
expect(names).toEqual(['logs.txt', 'sys.txt']);
|
||||||
|
|
||||||
|
const logs = addedZipEntries.find((e) => e.entryName === 'logs.txt');
|
||||||
|
const sys = addedZipEntries.find((e) => e.entryName === 'sys.txt');
|
||||||
|
expect(logs.content).toContain('recorded line');
|
||||||
|
expect(sys.content).toMatch(/Fredy version:\s+22\.5\.0/);
|
||||||
|
expect(sys.content).toContain('port: 9998');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('includes a placeholder message when no logs are stored', async () => {
|
||||||
|
await svc.buildDebugBundleZip({ settings: null });
|
||||||
|
const logs = addedZipEntries.find((e) => e.entryName === 'logs.txt');
|
||||||
|
expect(logs.content).toMatch(/no debug log entries/i);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
278
test/services/debug/debugLogStorage.test.js
Normal file
278
test/services/debug/debugLogStorage.test.js
Normal file
@@ -0,0 +1,278 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (c) 2026 by Christian Kellner.
|
||||||
|
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||||
|
import path from 'node:path';
|
||||||
|
import Database from 'better-sqlite3';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wire up an in-memory better-sqlite3 instance plus a stubbed settings module so the
|
||||||
|
* storage module under test can exercise real SQL while the rest of the dependency
|
||||||
|
* graph stays inert.
|
||||||
|
*/
|
||||||
|
async function bootstrap() {
|
||||||
|
const db = new Database(':memory:');
|
||||||
|
db.exec(`
|
||||||
|
CREATE TABLE debug_logs (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
ts INTEGER NOT NULL,
|
||||||
|
level TEXT NOT NULL,
|
||||||
|
message TEXT NOT NULL,
|
||||||
|
byte_size INTEGER NOT NULL
|
||||||
|
);
|
||||||
|
`);
|
||||||
|
|
||||||
|
const settings = { debug_logging_enabled: false, debug_logging_ever_enabled: false };
|
||||||
|
|
||||||
|
const ROOT = path.resolve('.');
|
||||||
|
const sqlitePath = path.join(ROOT, 'lib', 'services', 'storage', 'SqliteConnection.js');
|
||||||
|
const settingsPath = path.join(ROOT, 'lib', 'services', 'storage', 'settingsStorage.js');
|
||||||
|
|
||||||
|
const sqliteMock = {
|
||||||
|
default: {
|
||||||
|
getConnection: () => db,
|
||||||
|
execute: (sql, params = {}) => db.prepare(sql).run(params),
|
||||||
|
query: (sql, params = {}) => db.prepare(sql).all(params),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const settingsMock = {
|
||||||
|
getSettings: async () => ({ ...settings }),
|
||||||
|
upsertSettings: (entries) => {
|
||||||
|
const map = Array.isArray(entries) ? Object.fromEntries(entries) : entries;
|
||||||
|
for (const [k, v] of Object.entries(map)) {
|
||||||
|
settings[k] = v;
|
||||||
|
}
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
vi.resetModules();
|
||||||
|
vi.doMock(sqlitePath, () => sqliteMock);
|
||||||
|
vi.doMock(settingsPath, () => settingsMock);
|
||||||
|
|
||||||
|
const storage = await import(path.join(ROOT, 'lib', 'services', 'debug', 'debugLogStorage.js'));
|
||||||
|
storage._resetForTests();
|
||||||
|
return { storage, db, settings };
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('services/debug/debugLogStorage.js', () => {
|
||||||
|
let ctx;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
ctx = await bootstrap();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
try {
|
||||||
|
ctx.db.close();
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('isEnabled is false before enableDebugLogging is called', async () => {
|
||||||
|
expect(ctx.storage.isEnabled()).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('enableDebugLogging flips the cached flag and persists ever-enabled', async () => {
|
||||||
|
await ctx.storage.enableDebugLogging();
|
||||||
|
expect(ctx.storage.isEnabled()).toBe(true);
|
||||||
|
expect(ctx.settings.debug_logging_enabled).toBe(true);
|
||||||
|
expect(ctx.settings.debug_logging_ever_enabled).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('reloadEnabledFromSettings picks up persisted state after restart', async () => {
|
||||||
|
ctx.settings.debug_logging_enabled = true;
|
||||||
|
const enabled = await ctx.storage.reloadEnabledFromSettings();
|
||||||
|
expect(enabled).toBe(true);
|
||||||
|
expect(ctx.storage.isEnabled()).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('appendLogEntry writes only while enabled', async () => {
|
||||||
|
ctx.storage.appendLogEntry({ ts: 1, level: 'info', message: 'before-enable' });
|
||||||
|
expect(ctx.db.prepare('SELECT COUNT(*) AS c FROM debug_logs').get().c).toBe(0);
|
||||||
|
|
||||||
|
await ctx.storage.enableDebugLogging();
|
||||||
|
ctx.storage.appendLogEntry({ ts: 2, level: 'warn', message: 'after-enable' });
|
||||||
|
expect(ctx.db.prepare('SELECT COUNT(*) AS c FROM debug_logs').get().c).toBe(1);
|
||||||
|
|
||||||
|
const row = ctx.db.prepare('SELECT level, message, byte_size FROM debug_logs').get();
|
||||||
|
expect(row.level).toBe('warn');
|
||||||
|
expect(row.message).toBe('after-enable');
|
||||||
|
expect(row.byte_size).toBe(Buffer.byteLength('after-enable', 'utf-8'));
|
||||||
|
});
|
||||||
|
|
||||||
|
it('disableDebugLogging stops writes but keeps existing rows', async () => {
|
||||||
|
await ctx.storage.enableDebugLogging();
|
||||||
|
ctx.storage.appendLogEntry({ ts: 1, level: 'info', message: 'keep-me' });
|
||||||
|
await ctx.storage.disableDebugLogging();
|
||||||
|
ctx.storage.appendLogEntry({ ts: 2, level: 'info', message: 'never-written' });
|
||||||
|
|
||||||
|
const rows = ctx.db.prepare('SELECT message FROM debug_logs ORDER BY id').all();
|
||||||
|
expect(rows).toHaveLength(1);
|
||||||
|
expect(rows[0].message).toBe('keep-me');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('enableDebugLogging clears previous logs only when asked', async () => {
|
||||||
|
await ctx.storage.enableDebugLogging();
|
||||||
|
ctx.storage.appendLogEntry({ ts: 1, level: 'info', message: 'pre-existing' });
|
||||||
|
await ctx.storage.disableDebugLogging();
|
||||||
|
|
||||||
|
await ctx.storage.enableDebugLogging({ clearPrevious: false });
|
||||||
|
expect(ctx.db.prepare('SELECT COUNT(*) AS c FROM debug_logs').get().c).toBe(1);
|
||||||
|
|
||||||
|
await ctx.storage.disableDebugLogging();
|
||||||
|
await ctx.storage.enableDebugLogging({ clearPrevious: true });
|
||||||
|
expect(ctx.db.prepare('SELECT COUNT(*) AS c FROM debug_logs').get().c).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('hasAnyLogs / wasEverEnabled report correctly', async () => {
|
||||||
|
expect(ctx.storage.hasAnyLogs()).toBe(false);
|
||||||
|
expect(await ctx.storage.wasEverEnabled()).toBe(false);
|
||||||
|
|
||||||
|
await ctx.storage.enableDebugLogging();
|
||||||
|
ctx.storage.appendLogEntry({ ts: 1, level: 'info', message: 'hi' });
|
||||||
|
expect(ctx.storage.hasAnyLogs()).toBe(true);
|
||||||
|
expect(await ctx.storage.wasEverEnabled()).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('getCurrentSize reflects the on-disk byte total', async () => {
|
||||||
|
await ctx.storage.enableDebugLogging();
|
||||||
|
ctx.storage.appendLogEntry({ ts: 1, level: 'info', message: 'hello' }); // 5 bytes
|
||||||
|
ctx.storage.appendLogEntry({ ts: 2, level: 'info', message: 'world!' }); // 6 bytes
|
||||||
|
expect(await ctx.storage.getCurrentSize()).toBe(11);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rolling buffer drops oldest rows once the cap is exceeded', async () => {
|
||||||
|
await ctx.storage.enableDebugLogging();
|
||||||
|
const cap = ctx.storage.getMaxSize();
|
||||||
|
|
||||||
|
// Insert one row whose payload exceeds the entire cap. trimToFit must drop the
|
||||||
|
// oldest row(s) until the live size falls back under the cap. With a single
|
||||||
|
// oversized row, the only outcome is "table empty".
|
||||||
|
const giantText = 'X'.repeat(cap + 1024);
|
||||||
|
ctx.storage.appendLogEntry({ ts: 10, level: 'info', message: giantText });
|
||||||
|
|
||||||
|
const remaining = await ctx.storage.getCurrentSize();
|
||||||
|
expect(remaining).toBeLessThanOrEqual(cap);
|
||||||
|
expect(remaining).toBe(0);
|
||||||
|
expect(ctx.db.prepare('SELECT COUNT(*) AS c FROM debug_logs').get().c).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rolling buffer keeps newer rows when only the oldest need to go', async () => {
|
||||||
|
await ctx.storage.enableDebugLogging();
|
||||||
|
const cap = ctx.storage.getMaxSize();
|
||||||
|
|
||||||
|
// Push the size just over the cap with one big row, then a smaller "newer" row
|
||||||
|
// that should survive the trim because it is not at the head of the queue.
|
||||||
|
const bigText = 'A'.repeat(cap - 10); // ~5 MiB - 10 bytes
|
||||||
|
ctx.storage.appendLogEntry({ ts: 1, level: 'info', message: bigText });
|
||||||
|
|
||||||
|
// At this point we are just under the cap. Pushing one more row will tip us over.
|
||||||
|
ctx.storage.appendLogEntry({ ts: 2, level: 'warn', message: 'tip-over message which keeps us above cap' });
|
||||||
|
|
||||||
|
const remainingRows = ctx.db.prepare('SELECT message FROM debug_logs ORDER BY id ASC').all();
|
||||||
|
// The oldest (big) row must be gone; the newer one survives.
|
||||||
|
expect(remainingRows).toHaveLength(1);
|
||||||
|
expect(remainingRows[0].message).toContain('tip-over');
|
||||||
|
|
||||||
|
const remainingSize = await ctx.storage.getCurrentSize();
|
||||||
|
expect(remainingSize).toBeLessThanOrEqual(cap);
|
||||||
|
// And the cache must match what SQLite reports, verifies no drift after trim.
|
||||||
|
const dbSize = ctx.db.prepare('SELECT COALESCE(SUM(byte_size),0) AS s FROM debug_logs').get().s;
|
||||||
|
expect(remainingSize).toBe(dbSize);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('cachedSize stays consistent across enable → append → disable → re-enable(clear) cycles', async () => {
|
||||||
|
await ctx.storage.enableDebugLogging();
|
||||||
|
ctx.storage.appendLogEntry({ ts: 1, level: 'info', message: 'one' });
|
||||||
|
ctx.storage.appendLogEntry({ ts: 2, level: 'info', message: 'two' });
|
||||||
|
const sizeAfterFirst = await ctx.storage.getCurrentSize();
|
||||||
|
|
||||||
|
await ctx.storage.disableDebugLogging();
|
||||||
|
expect(await ctx.storage.getCurrentSize()).toBe(sizeAfterFirst);
|
||||||
|
|
||||||
|
await ctx.storage.enableDebugLogging({ clearPrevious: true });
|
||||||
|
expect(await ctx.storage.getCurrentSize()).toBe(0);
|
||||||
|
|
||||||
|
ctx.storage.appendLogEntry({ ts: 3, level: 'info', message: 'fresh' });
|
||||||
|
const dbSize = ctx.db.prepare('SELECT COALESCE(SUM(byte_size),0) AS s FROM debug_logs').get().s;
|
||||||
|
expect(await ctx.storage.getCurrentSize()).toBe(dbSize);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('clearAllDebugLogs empties the table and resets cached size', async () => {
|
||||||
|
await ctx.storage.enableDebugLogging();
|
||||||
|
ctx.storage.appendLogEntry({ ts: 1, level: 'info', message: 'foo' });
|
||||||
|
ctx.storage.appendLogEntry({ ts: 2, level: 'info', message: 'bar' });
|
||||||
|
expect(await ctx.storage.getCurrentSize()).toBeGreaterThan(0);
|
||||||
|
|
||||||
|
ctx.storage.clearAllDebugLogs();
|
||||||
|
|
||||||
|
expect(ctx.db.prepare('SELECT COUNT(*) AS c FROM debug_logs').get().c).toBe(0);
|
||||||
|
expect(await ctx.storage.getCurrentSize()).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('getAllDebugLogs returns rows ordered chronologically', async () => {
|
||||||
|
await ctx.storage.enableDebugLogging();
|
||||||
|
ctx.storage.appendLogEntry({ ts: 1, level: 'info', message: 'first' });
|
||||||
|
ctx.storage.appendLogEntry({ ts: 2, level: 'warn', message: 'second' });
|
||||||
|
ctx.storage.appendLogEntry({ ts: 3, level: 'error', message: 'third' });
|
||||||
|
|
||||||
|
const rows = ctx.storage.getAllDebugLogs();
|
||||||
|
expect(rows.map((r) => r.message)).toEqual(['first', 'second', 'third']);
|
||||||
|
expect(rows.map((r) => r.level)).toEqual(['info', 'warn', 'error']);
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('logger sink wiring', () => {
|
||||||
|
let logger;
|
||||||
|
let consoleSpies;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
// Storage imports the same logger module; vi.resetModules() ensured both share
|
||||||
|
// the same fresh instance for this test. Spies silence console output so the
|
||||||
|
// vitest report stays clean while we exercise real logger.info() calls.
|
||||||
|
logger = (await import(path.resolve('lib/services/logger.js'))).default;
|
||||||
|
consoleSpies = {
|
||||||
|
debug: vi.spyOn(console, 'debug').mockImplementation(() => {}),
|
||||||
|
info: vi.spyOn(console, 'info').mockImplementation(() => {}),
|
||||||
|
warn: vi.spyOn(console, 'warn').mockImplementation(() => {}),
|
||||||
|
error: vi.spyOn(console, 'error').mockImplementation(() => {}),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
// Detach sink between tests to prevent cross-test pollution from the shared
|
||||||
|
// logger module instance.
|
||||||
|
logger.setDebugLogSink(null);
|
||||||
|
for (const spy of Object.values(consoleSpies)) spy.mockRestore();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('routes logger calls into debug_logs once enabled', async () => {
|
||||||
|
await ctx.storage.enableDebugLogging();
|
||||||
|
logger.info('captured-via-logger');
|
||||||
|
const rows = ctx.db.prepare('SELECT level, message FROM debug_logs').all();
|
||||||
|
expect(rows).toHaveLength(1);
|
||||||
|
expect(rows[0].level).toBe('info');
|
||||||
|
expect(rows[0].message).toContain('captured-via-logger');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('detaches the sink on disable so logger calls no longer hit the DB', async () => {
|
||||||
|
await ctx.storage.enableDebugLogging();
|
||||||
|
await ctx.storage.disableDebugLogging();
|
||||||
|
logger.info('not-captured');
|
||||||
|
expect(ctx.db.prepare('SELECT COUNT(*) AS c FROM debug_logs').get().c).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('restores the sink on reloadEnabledFromSettings when persisted state is on', async () => {
|
||||||
|
ctx.settings.debug_logging_enabled = true;
|
||||||
|
await ctx.storage.reloadEnabledFromSettings();
|
||||||
|
logger.warn('captured-after-restart');
|
||||||
|
const rows = ctx.db.prepare('SELECT level, message FROM debug_logs').all();
|
||||||
|
expect(rows).toHaveLength(1);
|
||||||
|
expect(rows[0].level).toBe('warn');
|
||||||
|
expect(rows[0].message).toContain('captured-after-restart');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
250
test/services/debug/debugRouter.test.js
Normal file
250
test/services/debug/debugRouter.test.js
Normal file
@@ -0,0 +1,250 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (c) 2026 by Christian Kellner.
|
||||||
|
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||||
|
import path from 'node:path';
|
||||||
|
import Fastify from 'fastify';
|
||||||
|
|
||||||
|
describe('api/routes/debugRouter.js', () => {
|
||||||
|
let app;
|
||||||
|
let state;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
state = {
|
||||||
|
enabled: false,
|
||||||
|
hasLogs: false,
|
||||||
|
everEnabled: false,
|
||||||
|
size: 0,
|
||||||
|
max: 5 * 1024 * 1024,
|
||||||
|
};
|
||||||
|
|
||||||
|
const ROOT = path.resolve('.');
|
||||||
|
const storagePath = path.join(ROOT, 'lib', 'services', 'debug', 'debugLogStorage.js');
|
||||||
|
const bundlePath = path.join(ROOT, 'lib', 'services', 'debug', 'debugBundleService.js');
|
||||||
|
const settingsStoragePath = path.join(ROOT, 'lib', 'services', 'storage', 'settingsStorage.js');
|
||||||
|
|
||||||
|
const storageMock = {
|
||||||
|
isEnabled: () => state.enabled,
|
||||||
|
enableDebugLogging: async ({ clearPrevious = false } = {}) => {
|
||||||
|
state.enabled = true;
|
||||||
|
state.everEnabled = true;
|
||||||
|
if (clearPrevious) {
|
||||||
|
state.hasLogs = false;
|
||||||
|
state.size = 0;
|
||||||
|
}
|
||||||
|
},
|
||||||
|
disableDebugLogging: async () => {
|
||||||
|
state.enabled = false;
|
||||||
|
},
|
||||||
|
getCurrentSize: async () => state.size,
|
||||||
|
getMaxSize: () => state.max,
|
||||||
|
hasAnyLogs: () => state.hasLogs,
|
||||||
|
wasEverEnabled: async () => state.everEnabled,
|
||||||
|
clearAllDebugLogs: () => {
|
||||||
|
state.hasLogs = false;
|
||||||
|
state.size = 0;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const bundleMock = {
|
||||||
|
buildDebugBundleFileName: async () => '2026-06-08-FredyDebug-22.5.0.zip',
|
||||||
|
buildDebugBundleZip: async () => Buffer.from('FAKEZIP'),
|
||||||
|
};
|
||||||
|
|
||||||
|
const settingsMock = {
|
||||||
|
getSettings: async () => ({ port: 9998 }),
|
||||||
|
};
|
||||||
|
|
||||||
|
vi.resetModules();
|
||||||
|
vi.doMock(storagePath, () => storageMock);
|
||||||
|
vi.doMock(bundlePath, () => bundleMock);
|
||||||
|
vi.doMock(settingsStoragePath, () => settingsMock);
|
||||||
|
|
||||||
|
const mod = await import(path.join(ROOT, 'lib', 'api', 'routes', 'debugRouter.js'));
|
||||||
|
const plugin = mod.default;
|
||||||
|
app = Fastify({ logger: false });
|
||||||
|
await app.register(plugin, { prefix: '/api/admin/debug' });
|
||||||
|
await app.register(
|
||||||
|
async (sub) => {
|
||||||
|
mod.registerDebugPublicProbe(sub);
|
||||||
|
},
|
||||||
|
{ prefix: '/api/debug' },
|
||||||
|
);
|
||||||
|
await app.ready();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
if (app) await app.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('GET /status returns the current snapshot', async () => {
|
||||||
|
const res = await app.inject({ method: 'GET', url: '/api/admin/debug/status' });
|
||||||
|
expect(res.statusCode).toBe(200);
|
||||||
|
expect(res.json()).toEqual({
|
||||||
|
enabled: false,
|
||||||
|
size: 0,
|
||||||
|
max: state.max,
|
||||||
|
hasLogs: false,
|
||||||
|
everEnabled: false,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('POST /enable flips the feature on and returns updated status', async () => {
|
||||||
|
const res = await app.inject({
|
||||||
|
method: 'POST',
|
||||||
|
url: '/api/admin/debug/enable',
|
||||||
|
payload: {},
|
||||||
|
});
|
||||||
|
expect(res.statusCode).toBe(200);
|
||||||
|
const json = res.json();
|
||||||
|
expect(json.enabled).toBe(true);
|
||||||
|
expect(json.everEnabled).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('POST /enable with clearPrevious=true wipes existing logs first', async () => {
|
||||||
|
state.hasLogs = true;
|
||||||
|
state.size = 1234;
|
||||||
|
state.everEnabled = true;
|
||||||
|
|
||||||
|
const res = await app.inject({
|
||||||
|
method: 'POST',
|
||||||
|
url: '/api/admin/debug/enable',
|
||||||
|
payload: { clearPrevious: true },
|
||||||
|
});
|
||||||
|
expect(res.statusCode).toBe(200);
|
||||||
|
const json = res.json();
|
||||||
|
expect(json.enabled).toBe(true);
|
||||||
|
expect(json.hasLogs).toBe(false);
|
||||||
|
expect(json.size).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('POST /disable turns the feature off without losing existing logs', async () => {
|
||||||
|
state.enabled = true;
|
||||||
|
state.hasLogs = true;
|
||||||
|
state.everEnabled = true;
|
||||||
|
state.size = 99;
|
||||||
|
|
||||||
|
const res = await app.inject({ method: 'POST', url: '/api/admin/debug/disable' });
|
||||||
|
expect(res.statusCode).toBe(200);
|
||||||
|
const json = res.json();
|
||||||
|
expect(json.enabled).toBe(false);
|
||||||
|
expect(json.hasLogs).toBe(true);
|
||||||
|
expect(json.size).toBe(99);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('GET /download returns 409 when the feature was never enabled', async () => {
|
||||||
|
const res = await app.inject({ method: 'GET', url: '/api/admin/debug/download' });
|
||||||
|
expect(res.statusCode).toBe(409);
|
||||||
|
expect(res.json().error).toMatch(/never produced any data/i);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('GET /download returns 409 when ever-enabled but no logs are stored', async () => {
|
||||||
|
state.everEnabled = true;
|
||||||
|
state.hasLogs = false;
|
||||||
|
const res = await app.inject({ method: 'GET', url: '/api/admin/debug/download' });
|
||||||
|
expect(res.statusCode).toBe(409);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('GET /download streams a zip with the expected headers when logs exist', async () => {
|
||||||
|
state.everEnabled = true;
|
||||||
|
state.hasLogs = true;
|
||||||
|
|
||||||
|
const res = await app.inject({ method: 'GET', url: '/api/admin/debug/download' });
|
||||||
|
expect(res.statusCode).toBe(200);
|
||||||
|
expect(res.headers['content-type']).toBe('application/zip');
|
||||||
|
expect(res.headers['content-disposition']).toContain('FredyDebug');
|
||||||
|
expect(res.rawPayload.toString('utf-8')).toBe('FAKEZIP');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('DELETE /logs wipes stored logs without touching the enabled flag', async () => {
|
||||||
|
state.enabled = true;
|
||||||
|
state.hasLogs = true;
|
||||||
|
state.everEnabled = true;
|
||||||
|
state.size = 1234;
|
||||||
|
|
||||||
|
const res = await app.inject({ method: 'DELETE', url: '/api/admin/debug/logs' });
|
||||||
|
expect(res.statusCode).toBe(200);
|
||||||
|
const json = res.json();
|
||||||
|
expect(json.enabled).toBe(true);
|
||||||
|
expect(json.hasLogs).toBe(false);
|
||||||
|
expect(json.size).toBe(0);
|
||||||
|
// everEnabled must stay true so the download button does not change semantics.
|
||||||
|
expect(json.everEnabled).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('GET /api/debug/active returns only the enabled boolean (no other settings)', async () => {
|
||||||
|
state.enabled = false;
|
||||||
|
let res = await app.inject({ method: 'GET', url: '/api/debug/active' });
|
||||||
|
expect(res.statusCode).toBe(200);
|
||||||
|
expect(res.json()).toEqual({ enabled: false });
|
||||||
|
|
||||||
|
state.enabled = true;
|
||||||
|
res = await app.inject({ method: 'GET', url: '/api/debug/active' });
|
||||||
|
expect(res.statusCode).toBe(200);
|
||||||
|
expect(res.json()).toEqual({ enabled: true });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('api/routes/debugRouter.js - admin-only enforcement', () => {
|
||||||
|
let app;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
const ROOT = path.resolve('.');
|
||||||
|
const storagePath = path.join(ROOT, 'lib', 'services', 'debug', 'debugLogStorage.js');
|
||||||
|
const bundlePath = path.join(ROOT, 'lib', 'services', 'debug', 'debugBundleService.js');
|
||||||
|
const settingsStoragePath = path.join(ROOT, 'lib', 'services', 'storage', 'settingsStorage.js');
|
||||||
|
|
||||||
|
vi.resetModules();
|
||||||
|
vi.doMock(storagePath, () => ({
|
||||||
|
isEnabled: () => false,
|
||||||
|
enableDebugLogging: async () => {},
|
||||||
|
disableDebugLogging: async () => {},
|
||||||
|
getCurrentSize: async () => 0,
|
||||||
|
getMaxSize: () => 5 * 1024 * 1024,
|
||||||
|
hasAnyLogs: () => false,
|
||||||
|
wasEverEnabled: async () => false,
|
||||||
|
clearAllDebugLogs: () => {},
|
||||||
|
}));
|
||||||
|
vi.doMock(bundlePath, () => ({
|
||||||
|
buildDebugBundleFileName: async () => 'x.zip',
|
||||||
|
buildDebugBundleZip: async () => Buffer.from(''),
|
||||||
|
}));
|
||||||
|
vi.doMock(settingsStoragePath, () => ({
|
||||||
|
getSettings: async () => ({}),
|
||||||
|
}));
|
||||||
|
|
||||||
|
const plugin = (await import(path.join(ROOT, 'lib', 'api', 'routes', 'debugRouter.js'))).default;
|
||||||
|
app = Fastify({ logger: false });
|
||||||
|
await app.register(
|
||||||
|
async (sub) => {
|
||||||
|
// Same wiring shape as lib/api/api.js: apply adminHook before the plugin.
|
||||||
|
sub.addHook('preHandler', async (request, reply) => {
|
||||||
|
reply.code(401).send();
|
||||||
|
});
|
||||||
|
sub.register(plugin, { prefix: '/api/admin/debug' });
|
||||||
|
},
|
||||||
|
{ prefix: '/' },
|
||||||
|
);
|
||||||
|
await app.ready();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
if (app) await app.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects non-admin callers with 401 on every endpoint', async () => {
|
||||||
|
for (const route of [
|
||||||
|
['GET', '/api/admin/debug/status'],
|
||||||
|
['POST', '/api/admin/debug/enable'],
|
||||||
|
['POST', '/api/admin/debug/disable'],
|
||||||
|
['GET', '/api/admin/debug/download'],
|
||||||
|
['DELETE', '/api/admin/debug/logs'],
|
||||||
|
]) {
|
||||||
|
const [method, url] = route;
|
||||||
|
const res = await app.inject({ method, url, payload: method === 'POST' ? {} : undefined });
|
||||||
|
expect(res.statusCode, `${method} ${url}`).toBe(401);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
37
test/services/extractor/puppeteerExtractor.test.js
Normal file
37
test/services/extractor/puppeteerExtractor.test.js
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (c) 2026 by Christian Kellner.
|
||||||
|
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { vi, describe, it, expect, beforeEach } from 'vitest';
|
||||||
|
|
||||||
|
// Mock the CloakBrowser launcher so no real Chromium binary is needed and we can
|
||||||
|
// assert which options get forwarded to it.
|
||||||
|
const { launchMock } = vi.hoisted(() => ({ launchMock: vi.fn() }));
|
||||||
|
|
||||||
|
vi.mock('cloakbrowser/puppeteer', () => ({
|
||||||
|
launch: launchMock,
|
||||||
|
}));
|
||||||
|
|
||||||
|
const { launchBrowser } = await import('../../../lib/services/extractor/puppeteerExtractor.js');
|
||||||
|
|
||||||
|
describe('launchBrowser proxy forwarding', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
launchMock.mockReset();
|
||||||
|
launchMock.mockResolvedValue({ close: async () => {} });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('forwards proxyUrl to CloakBrowser as the proxy option', async () => {
|
||||||
|
await launchBrowser('https://www.immowelt.de/', { proxyUrl: 'http://user:pass@host:8080' });
|
||||||
|
|
||||||
|
expect(launchMock).toHaveBeenCalledTimes(1);
|
||||||
|
expect(launchMock.mock.calls[0][0]).toMatchObject({ proxy: 'http://user:pass@host:8080' });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not set a proxy when no proxyUrl is given', async () => {
|
||||||
|
await launchBrowser('https://www.immowelt.de/', {});
|
||||||
|
|
||||||
|
expect(launchMock).toHaveBeenCalledTimes(1);
|
||||||
|
expect(launchMock.mock.calls[0][0].proxy).toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -4,11 +4,16 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
import { convertWebToMobile } from '../../../lib/services/immoscout/immoscout-web-translator.js';
|
import { convertWebToMobile } from '../../../lib/services/immoscout/immoscout-web-translator.js';
|
||||||
import { expect } from 'vitest';
|
import { expect, vi } from 'vitest';
|
||||||
import { readFile } from 'fs/promises';
|
import { readFile } from 'fs/promises';
|
||||||
|
import { buildFetchMock } from '../../offlineFixtures.js';
|
||||||
|
|
||||||
export const testData = JSON.parse(await readFile(new URL('./testdata.json', import.meta.url)));
|
export const testData = JSON.parse(await readFile(new URL('./testdata.json', import.meta.url)));
|
||||||
|
|
||||||
|
if (process.env.TEST_MODE === 'offline') {
|
||||||
|
vi.stubGlobal('fetch', buildFetchMock());
|
||||||
|
}
|
||||||
|
|
||||||
describe('#immoscout-mobile URL conversion', () => {
|
describe('#immoscout-mobile URL conversion', () => {
|
||||||
// Test shape URL conversion
|
// Test shape URL conversion
|
||||||
it('should convert a full web URL with shape to mobile URL', () => {
|
it('should convert a full web URL with shape to mobile URL', () => {
|
||||||
@@ -26,12 +31,35 @@ describe('#immoscout-mobile URL conversion', () => {
|
|||||||
const webUrl =
|
const webUrl =
|
||||||
'https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-mieten?heatingtypes=central,selfcontainedcentral&haspromotion=false&numberofrooms=2.0-5.0&livingspace=10.0-25.0&energyefficiencyclasses=a,b,c,d,e,f,g,h,a_plus&exclusioncriteria=projectlisting,swapflat&equipment=parking,cellar,builtinkitchen,lift,garden,guesttoilet,balcony&petsallowedtypes=no,yes,negotiable&price=10.0-100.0&constructionyear=1920-2026&apartmenttypes=halfbasement,penthouse,other,loft,groundfloor,terracedflat,raisedgroundfloor,roofstorey,apartment,maisonette&pricetype=calculatedtotalrent&floor=2-7&enteredFrom=result_list';
|
'https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-mieten?heatingtypes=central,selfcontainedcentral&haspromotion=false&numberofrooms=2.0-5.0&livingspace=10.0-25.0&energyefficiencyclasses=a,b,c,d,e,f,g,h,a_plus&exclusioncriteria=projectlisting,swapflat&equipment=parking,cellar,builtinkitchen,lift,garden,guesttoilet,balcony&petsallowedtypes=no,yes,negotiable&price=10.0-100.0&constructionyear=1920-2026&apartmenttypes=halfbasement,penthouse,other,loft,groundfloor,terracedflat,raisedgroundfloor,roofstorey,apartment,maisonette&pricetype=calculatedtotalrent&floor=2-7&enteredFrom=result_list';
|
||||||
const expectedMobileUrl =
|
const expectedMobileUrl =
|
||||||
'https://api.mobile.immobilienscout24.de/search/list?apartmenttypes=halfbasement,penthouse,other,loft,groundfloor,terracedflat,raisedgroundfloor,roofstorey,apartment,maisonette&constructionyear=1920-2026&energyefficiencyclasses=a,b,c,d,e,f,g,h,a_plus&equipment=parking,cellar,builtInKitchen,lift,garden,guestToilet,balcony&exclusioncriteria=projectlisting,swapflat&floor=2-7&geocodes=%2Fde%2Fberlin%2Fberlin&haspromotion=false&heatingtypes=central,selfcontainedcentral&livingspace=10.0-25.0&numberofrooms=2.0-5.0&petsallowedtypes=no,yes,negotiable&price=10.0-100.0&pricetype=calculatedtotalrent&realestatetype=apartmentrent&searchType=region';
|
'https://api.mobile.immobilienscout24.de/search/list?apartmenttypes=halfbasement,penthouse,other,loft,groundfloor,terracedflat,raisedgroundfloor,roofstorey,apartment,maisonette&constructionyear=1920-2026&energyefficiencyclasses=a,b,c,d,e,f,g,h,a_plus&equipment=parking,cellar,builtInKitchen,lift,garden,guestToilet,balcony&exclusioncriteria=projectlisting,swap_flat&floor=2-7&geocodes=%2Fde%2Fberlin%2Fberlin&haspromotion=false&heatingtypes=central,selfcontainedcentral&livingspace=10.0-25.0&numberofrooms=2.0-5.0&petsallowedtypes=no,yes,negotiable&price=10.0-100.0&pricetype=calculatedtotalrent&realestatetype=apartmentrent&searchType=region';
|
||||||
|
|
||||||
const actualMobileUrl = convertWebToMobile(webUrl);
|
const actualMobileUrl = convertWebToMobile(webUrl);
|
||||||
expect(actualMobileUrl).toBe(expectedMobileUrl);
|
expect(actualMobileUrl).toBe(expectedMobileUrl);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// The web UI encodes "no swap flats" as exclusioncriteria=swapflat, but the
|
||||||
|
// mobile API only understands swap_flat. Unknown values are not ignored by the
|
||||||
|
// API - the search silently returns 0 results, so the mapping is essential.
|
||||||
|
it('should map exclusioncriteria=swapflat to the mobile API value swap_flat', () => {
|
||||||
|
const webUrl =
|
||||||
|
'https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-mieten?exclusioncriteria=swapflat&price=-1500.0';
|
||||||
|
|
||||||
|
const converted = convertWebToMobile(webUrl);
|
||||||
|
const queryParams = new URL(converted).searchParams;
|
||||||
|
expect(queryParams.get('exclusioncriteria')).toBe('swap_flat');
|
||||||
|
});
|
||||||
|
|
||||||
|
// Values the mobile API shares with the web API (e.g. projectlisting) must
|
||||||
|
// pass through unchanged, in any combination with mapped values.
|
||||||
|
it('should keep other exclusioncriteria values untouched', () => {
|
||||||
|
const webUrl =
|
||||||
|
'https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-mieten?exclusioncriteria=projectlisting,swapflat';
|
||||||
|
|
||||||
|
const converted = convertWebToMobile(webUrl);
|
||||||
|
const queryParams = new URL(converted).searchParams;
|
||||||
|
expect(queryParams.get('exclusioncriteria')).toBe('projectlisting,swap_flat');
|
||||||
|
});
|
||||||
|
|
||||||
// Test URL conversion of web-only SEO path
|
// Test URL conversion of web-only SEO path
|
||||||
it('should convert a SEO web path to the correct query params', () => {
|
it('should convert a SEO web path to the correct query params', () => {
|
||||||
const webUrl = 'https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-mit-balkon-mieten?equipment=garden';
|
const webUrl = 'https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-mit-balkon-mieten?equipment=garden';
|
||||||
@@ -41,6 +69,60 @@ describe('#immoscout-mobile URL conversion', () => {
|
|||||||
expect(queryParams.get('equipment').split(',')).toEqual(expect.arrayContaining(['garden', 'balcony']));
|
expect(queryParams.get('equipment').split(',')).toEqual(expect.arrayContaining(['garden', 'balcony']));
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Test URL conversion of SEO web path for max warmrent. The ImmoScout web UI
|
||||||
|
// generates this special SEO slug instead of explicit price/pricetype params
|
||||||
|
// when the user configures a "Warmmiete" filter (real-world URL).
|
||||||
|
it('should convert a SEO apartment max warmrent path to rent + price + pricetype', () => {
|
||||||
|
const webUrl =
|
||||||
|
'https://www.immobilienscout24.de/Suche/de/nordrhein-westfalen/duesseldorf/wohnung-bis-800-euro-warm?livingspace=-800.0&enteredFrom=result_list';
|
||||||
|
|
||||||
|
const converted = convertWebToMobile(webUrl);
|
||||||
|
const queryParams = new URL(converted).searchParams;
|
||||||
|
expect(queryParams.get('realestatetype')).toBe('apartmentrent');
|
||||||
|
expect(queryParams.get('price')).toBe('-800');
|
||||||
|
expect(queryParams.get('pricetype')).toBe('calculatedtotalrent');
|
||||||
|
expect(queryParams.get('geocodes')).toBe('/de/nordrhein-westfalen/duesseldorf');
|
||||||
|
expect(queryParams.get('livingspace')).toBe('-800.0');
|
||||||
|
});
|
||||||
|
|
||||||
|
// Same SEO pattern for houses ("haus-bis-X-euro-warm" → houserent).
|
||||||
|
it('should convert a SEO house max warmrent path to rent + price + pricetype', () => {
|
||||||
|
const webUrl = 'https://www.immobilienscout24.de/Suche/de/berlin/berlin/haus-bis-1500-euro-warm';
|
||||||
|
|
||||||
|
const converted = convertWebToMobile(webUrl);
|
||||||
|
const queryParams = new URL(converted).searchParams;
|
||||||
|
expect(queryParams.get('realestatetype')).toBe('houserent');
|
||||||
|
expect(queryParams.get('price')).toBe('-1500');
|
||||||
|
expect(queryParams.get('pricetype')).toBe('calculatedtotalrent');
|
||||||
|
});
|
||||||
|
|
||||||
|
// Sanity check: max coldrent ("Kaltmiete") does NOT use an SEO slug. The web
|
||||||
|
// UI keeps the regular "wohnung-mieten" path and passes explicit
|
||||||
|
// price + pricetype query params, which the existing translator already
|
||||||
|
// handles (real-world URL).
|
||||||
|
it('should convert a max coldrent search via the regular wohnung-mieten path', () => {
|
||||||
|
const webUrl =
|
||||||
|
'https://www.immobilienscout24.de/Suche/de/nordrhein-westfalen/duesseldorf/wohnung-mieten?price=-800.0&livingspace=-800.0&pricetype=rentpermonth&enteredFrom=result_list';
|
||||||
|
|
||||||
|
const converted = convertWebToMobile(webUrl);
|
||||||
|
const queryParams = new URL(converted).searchParams;
|
||||||
|
expect(queryParams.get('realestatetype')).toBe('apartmentrent');
|
||||||
|
expect(queryParams.get('price')).toBe('-800.0');
|
||||||
|
expect(queryParams.get('pricetype')).toBe('rentpermonth');
|
||||||
|
expect(queryParams.get('geocodes')).toBe('/de/nordrhein-westfalen/duesseldorf');
|
||||||
|
});
|
||||||
|
|
||||||
|
// Explicit query params win over the SEO slug's implicit defaults.
|
||||||
|
it('should let explicit query params override SEO path price defaults', () => {
|
||||||
|
const webUrl = 'https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-bis-800-euro-warm?price=100-500';
|
||||||
|
|
||||||
|
const converted = convertWebToMobile(webUrl);
|
||||||
|
const queryParams = new URL(converted).searchParams;
|
||||||
|
expect(queryParams.get('realestatetype')).toBe('apartmentrent');
|
||||||
|
expect(queryParams.get('price')).toBe('100-500');
|
||||||
|
expect(queryParams.get('pricetype')).toBe('calculatedtotalrent');
|
||||||
|
});
|
||||||
|
|
||||||
// Test URL conversion with unsupported query parameters
|
// Test URL conversion with unsupported query parameters
|
||||||
it('should remove unsupported query parameters', () => {
|
it('should remove unsupported query parameters', () => {
|
||||||
const webUrl = 'https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-mieten?minimuminternetspeed=100000';
|
const webUrl = 'https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-mieten?minimuminternetspeed=100000';
|
||||||
|
|||||||
110
test/services/jobs/dashboardRouter.test.js
Normal file
110
test/services/jobs/dashboardRouter.test.js
Normal file
@@ -0,0 +1,110 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (c) 2026 by Christian Kellner.
|
||||||
|
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||||
|
import path from 'node:path';
|
||||||
|
import Fastify from 'fastify';
|
||||||
|
|
||||||
|
describe('api/routes/dashboardRouter.js', () => {
|
||||||
|
let app;
|
||||||
|
let state;
|
||||||
|
|
||||||
|
async function buildApp() {
|
||||||
|
const ROOT = path.resolve('.');
|
||||||
|
const jobStoragePath = path.join(ROOT, 'lib', 'services', 'storage', 'jobStorage.js');
|
||||||
|
const listingsStoragePath = path.join(ROOT, 'lib', 'services', 'storage', 'listingsStorage.js');
|
||||||
|
const settingsStoragePath = path.join(ROOT, 'lib', 'services', 'storage', 'settingsStorage.js');
|
||||||
|
const securityPath = path.join(ROOT, 'lib', 'api', 'security.js');
|
||||||
|
|
||||||
|
vi.resetModules();
|
||||||
|
vi.doMock(jobStoragePath, () => ({
|
||||||
|
getJobs: () => state.jobs.slice(),
|
||||||
|
}));
|
||||||
|
vi.doMock(listingsStoragePath, () => ({
|
||||||
|
getListingsKpisForJobIds: () => ({ numberOfActiveListings: 0, medianPriceOfListings: 0 }),
|
||||||
|
getProviderDistributionForJobIds: () => [],
|
||||||
|
}));
|
||||||
|
vi.doMock(settingsStoragePath, () => ({
|
||||||
|
getSettings: async () => ({ interval: 30 }),
|
||||||
|
}));
|
||||||
|
vi.doMock(securityPath, () => ({
|
||||||
|
isAdmin: () => state.admin,
|
||||||
|
}));
|
||||||
|
|
||||||
|
const mod = await import(path.join(ROOT, 'lib', 'api', 'routes', 'dashboardRouter.js'));
|
||||||
|
const plugin = mod.default;
|
||||||
|
const instance = Fastify({ logger: false });
|
||||||
|
instance.addHook('onRequest', async (request) => {
|
||||||
|
request.session = { currentUser: state.currentUser, createdAt: Date.now() };
|
||||||
|
});
|
||||||
|
await instance.register(plugin, { prefix: '/api/dashboard' });
|
||||||
|
await instance.ready();
|
||||||
|
return instance;
|
||||||
|
}
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
state = {
|
||||||
|
currentUser: 'u1',
|
||||||
|
admin: false,
|
||||||
|
jobs: [],
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
if (app) await app.close();
|
||||||
|
app = null;
|
||||||
|
});
|
||||||
|
|
||||||
|
it('derives lastRun from the most recent accessible job for a regular user', async () => {
|
||||||
|
state.jobs = [
|
||||||
|
{ id: 'a', userId: 'u1', shared_with_user: [], lastRunAt: 1000 },
|
||||||
|
{ id: 'b', userId: 'u1', shared_with_user: [], lastRunAt: 5000 },
|
||||||
|
{ id: 'c', userId: 'someone-else', shared_with_user: [], lastRunAt: 9999 },
|
||||||
|
];
|
||||||
|
app = await buildApp();
|
||||||
|
|
||||||
|
const res = await app.inject({ method: 'GET', url: '/api/dashboard/' });
|
||||||
|
expect(res.statusCode).toBe(200);
|
||||||
|
const body = res.json();
|
||||||
|
expect(body.general.lastRun).toBe(5000);
|
||||||
|
expect(body.general.nextRun).toBe(5000 + 30 * 60000);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('includes shared jobs in the lastRun calculation', async () => {
|
||||||
|
state.jobs = [
|
||||||
|
{ id: 'mine', userId: 'u1', shared_with_user: [], lastRunAt: 1000 },
|
||||||
|
{ id: 'shared', userId: 'someone-else', shared_with_user: ['u1'], lastRunAt: 4000 },
|
||||||
|
];
|
||||||
|
app = await buildApp();
|
||||||
|
|
||||||
|
const res = await app.inject({ method: 'GET', url: '/api/dashboard/' });
|
||||||
|
expect(res.json().general.lastRun).toBe(4000);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('admins see lastRun across all jobs', async () => {
|
||||||
|
state.admin = true;
|
||||||
|
state.jobs = [
|
||||||
|
{ id: 'a', userId: 'someone', shared_with_user: [], lastRunAt: 1000 },
|
||||||
|
{ id: 'b', userId: 'another', shared_with_user: [], lastRunAt: 7000 },
|
||||||
|
];
|
||||||
|
app = await buildApp();
|
||||||
|
|
||||||
|
const res = await app.inject({ method: 'GET', url: '/api/dashboard/' });
|
||||||
|
expect(res.json().general.lastRun).toBe(7000);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns null lastRun and 0 nextRun when no accessible job has ever run', async () => {
|
||||||
|
state.jobs = [
|
||||||
|
{ id: 'a', userId: 'u1', shared_with_user: [], lastRunAt: null },
|
||||||
|
{ id: 'b', userId: 'someone-else', shared_with_user: [], lastRunAt: 9999 },
|
||||||
|
];
|
||||||
|
app = await buildApp();
|
||||||
|
|
||||||
|
const res = await app.inject({ method: 'GET', url: '/api/dashboard/' });
|
||||||
|
const body = res.json();
|
||||||
|
expect(body.general.lastRun).toBeNull();
|
||||||
|
expect(body.general.nextRun).toBe(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -18,6 +18,7 @@ describe('services/jobs/jobExecutionService', () => {
|
|||||||
const busPath = root + '/lib/services/events/event-bus.js';
|
const busPath = root + '/lib/services/events/event-bus.js';
|
||||||
const jobStoragePath = root + '/lib/services/storage/jobStorage.js';
|
const jobStoragePath = root + '/lib/services/storage/jobStorage.js';
|
||||||
const userStoragePath = root + '/lib/services/storage/userStorage.js';
|
const userStoragePath = root + '/lib/services/storage/userStorage.js';
|
||||||
|
const settingsStoragePath = root + '/lib/services/storage/settingsStorage.js';
|
||||||
const brokerPath = root + '/lib/services/sse/sse-broker.js';
|
const brokerPath = root + '/lib/services/sse/sse-broker.js';
|
||||||
const utilsPath = root + '/lib/utils.js';
|
const utilsPath = root + '/lib/utils.js';
|
||||||
const loggerPath = root + '/lib/services/logger.js';
|
const loggerPath = root + '/lib/services/logger.js';
|
||||||
@@ -28,11 +29,15 @@ describe('services/jobs/jobExecutionService', () => {
|
|||||||
vi.doMock(jobStoragePath, () => ({
|
vi.doMock(jobStoragePath, () => ({
|
||||||
getJob: (id) => state.jobsById[id] || null,
|
getJob: (id) => state.jobsById[id] || null,
|
||||||
getJobs: () => state.jobsList.slice(),
|
getJobs: () => state.jobsList.slice(),
|
||||||
|
updateJobLastRunAt: (id, timestamp) => calls.lastRunUpdates.push({ id, timestamp }),
|
||||||
}));
|
}));
|
||||||
vi.doMock(userStoragePath, () => ({
|
vi.doMock(userStoragePath, () => ({
|
||||||
getUsers: () => state.users.slice(),
|
getUsers: () => state.users.slice(),
|
||||||
getUser: (id) => state.users.find((u) => u.id === id) || null,
|
getUser: (id) => state.users.find((u) => u.id === id) || null,
|
||||||
}));
|
}));
|
||||||
|
vi.doMock(settingsStoragePath, () => ({
|
||||||
|
getSettings: async () => ({}),
|
||||||
|
}));
|
||||||
vi.doMock(brokerPath, () => ({
|
vi.doMock(brokerPath, () => ({
|
||||||
sendToUsers: (...args) => calls.sent.push(args),
|
sendToUsers: (...args) => calls.sent.push(args),
|
||||||
}));
|
}));
|
||||||
@@ -61,7 +66,7 @@ describe('services/jobs/jobExecutionService', () => {
|
|||||||
|
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
bus = new EventEmitter();
|
bus = new EventEmitter();
|
||||||
calls = { sent: [], markRunning: [] };
|
calls = { sent: [], markRunning: [], lastRunUpdates: [] };
|
||||||
state = {
|
state = {
|
||||||
jobsById: {},
|
jobsById: {},
|
||||||
jobsList: [],
|
jobsList: [],
|
||||||
@@ -115,4 +120,23 @@ describe('services/jobs/jobExecutionService', () => {
|
|||||||
await new Promise((r) => setTimeout(r, 0));
|
await new Promise((r) => setTimeout(r, 0));
|
||||||
expect(new Set(calls.markRunning)).toEqual(new Set(['j1', 'j2']));
|
expect(new Set(calls.markRunning)).toEqual(new Set(['j1', 'j2']));
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('persists last_run_at when a job is executed', async () => {
|
||||||
|
state.jobsById['j1'] = { id: 'j1', enabled: true, userId: 'u1', provider: [] };
|
||||||
|
state.jobsList = [state.jobsById['j1']];
|
||||||
|
state.users = [{ id: 'u1', isAdmin: false }];
|
||||||
|
|
||||||
|
await initService();
|
||||||
|
|
||||||
|
const before = Date.now();
|
||||||
|
bus.emit('jobs:runOne', { jobId: 'j1' });
|
||||||
|
await new Promise((r) => setTimeout(r, 0));
|
||||||
|
const after = Date.now();
|
||||||
|
|
||||||
|
expect(calls.lastRunUpdates.length).toBe(1);
|
||||||
|
const [update] = calls.lastRunUpdates;
|
||||||
|
expect(update.id).toBe('j1');
|
||||||
|
expect(update.timestamp).toBeGreaterThanOrEqual(before);
|
||||||
|
expect(update.timestamp).toBeLessThanOrEqual(after);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
89
test/services/logger.test.js
Normal file
89
test/services/logger.test.js
Normal file
@@ -0,0 +1,89 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (c) 2026 by Christian Kellner.
|
||||||
|
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||||
|
import path from 'node:path';
|
||||||
|
|
||||||
|
describe('services/logger.js - debug log sink', () => {
|
||||||
|
let logger;
|
||||||
|
let setDebugLogSink;
|
||||||
|
let consoleSpies;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
vi.resetModules();
|
||||||
|
const mod = await import(path.resolve('lib/services/logger.js'));
|
||||||
|
logger = mod.default;
|
||||||
|
setDebugLogSink = mod.setDebugLogSink;
|
||||||
|
|
||||||
|
// Silence console output so test runner stdout stays readable while still
|
||||||
|
// letting us inspect what the logger emitted if a test wants to.
|
||||||
|
consoleSpies = {
|
||||||
|
debug: vi.spyOn(console, 'debug').mockImplementation(() => {}),
|
||||||
|
info: vi.spyOn(console, 'info').mockImplementation(() => {}),
|
||||||
|
warn: vi.spyOn(console, 'warn').mockImplementation(() => {}),
|
||||||
|
error: vi.spyOn(console, 'error').mockImplementation(() => {}),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
setDebugLogSink(null);
|
||||||
|
for (const spy of Object.values(consoleSpies)) spy.mockRestore();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('is a no-op for the sink when none is registered', () => {
|
||||||
|
// Just make sure nothing throws.
|
||||||
|
expect(() => logger.info('hello')).not.toThrow();
|
||||||
|
expect(() => logger.error(new Error('boom'))).not.toThrow();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('forwards every log level (including debug) to the registered sink', () => {
|
||||||
|
const captured = [];
|
||||||
|
setDebugLogSink((entry) => captured.push(entry));
|
||||||
|
|
||||||
|
logger.debug('debug-line');
|
||||||
|
logger.info('info-line');
|
||||||
|
logger.warn('warn-line');
|
||||||
|
logger.error('error-line');
|
||||||
|
|
||||||
|
expect(captured).toHaveLength(4);
|
||||||
|
expect(captured.map((c) => c.level)).toEqual(['debug', 'info', 'warn', 'error']);
|
||||||
|
expect(captured[0].message).toContain('debug-line');
|
||||||
|
expect(captured[1].message).toContain('info-line');
|
||||||
|
expect(captured[2].message).toContain('warn-line');
|
||||||
|
expect(captured[3].message).toContain('error-line');
|
||||||
|
for (const c of captured) {
|
||||||
|
expect(typeof c.ts).toBe('number');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('serializes Error stacks for the sink instead of "[object Object]"', () => {
|
||||||
|
const captured = [];
|
||||||
|
setDebugLogSink((entry) => captured.push(entry));
|
||||||
|
|
||||||
|
logger.error(new Error('boom'));
|
||||||
|
|
||||||
|
expect(captured).toHaveLength(1);
|
||||||
|
expect(captured[0].message).toContain('Error: boom');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('stops forwarding once the sink is unregistered', () => {
|
||||||
|
const captured = [];
|
||||||
|
setDebugLogSink((entry) => captured.push(entry));
|
||||||
|
logger.info('one');
|
||||||
|
setDebugLogSink(null);
|
||||||
|
logger.info('two');
|
||||||
|
|
||||||
|
expect(captured).toHaveLength(1);
|
||||||
|
expect(captured[0].message).toContain('one');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not break the caller when the sink throws', () => {
|
||||||
|
setDebugLogSink(() => {
|
||||||
|
throw new Error('sink exploded');
|
||||||
|
});
|
||||||
|
expect(() => logger.info('still works')).not.toThrow();
|
||||||
|
expect(consoleSpies.info).toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
});
|
||||||
64
test/storage/jobStorage.test.js
Normal file
64
test/storage/jobStorage.test.js
Normal file
@@ -0,0 +1,64 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (c) 2026 by Christian Kellner.
|
||||||
|
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { vi, describe, it, expect, beforeEach } from 'vitest';
|
||||||
|
|
||||||
|
// Mock SqliteConnection so we can assert which SQL the storage layer runs
|
||||||
|
// without spinning up a real SQLite DB.
|
||||||
|
|
||||||
|
const calls = {
|
||||||
|
execute: [],
|
||||||
|
query: [],
|
||||||
|
};
|
||||||
|
|
||||||
|
const sqliteMock = {
|
||||||
|
execute: (sql, params) => {
|
||||||
|
calls.execute.push({ sql, params });
|
||||||
|
return { changes: 1 };
|
||||||
|
},
|
||||||
|
query: (sql, params) => {
|
||||||
|
calls.query.push({ sql, params });
|
||||||
|
if (sqliteMock.__queryHandler) return sqliteMock.__queryHandler(sql, params);
|
||||||
|
return [];
|
||||||
|
},
|
||||||
|
__queryHandler: null,
|
||||||
|
};
|
||||||
|
|
||||||
|
vi.mock('../../lib/services/storage/SqliteConnection.js', () => ({
|
||||||
|
default: sqliteMock,
|
||||||
|
}));
|
||||||
|
|
||||||
|
describe('jobStorage.getJobs', () => {
|
||||||
|
let jobStorage;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
calls.execute.length = 0;
|
||||||
|
calls.query.length = 0;
|
||||||
|
sqliteMock.__queryHandler = null;
|
||||||
|
jobStorage = await import('../../lib/services/storage/jobStorage.js');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('filters out disabled jobs by default (WHERE j.enabled = 1)', () => {
|
||||||
|
jobStorage.getJobs();
|
||||||
|
expect(calls.query).toHaveLength(1);
|
||||||
|
expect(calls.query[0].sql).toMatch(/WHERE j\.enabled = 1/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('includes disabled jobs when includeDisabled is true', () => {
|
||||||
|
jobStorage.getJobs({ includeDisabled: true });
|
||||||
|
expect(calls.query).toHaveLength(1);
|
||||||
|
expect(calls.query[0].sql).not.toMatch(/WHERE j\.enabled = 1/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('coerces the enabled column to a boolean', () => {
|
||||||
|
sqliteMock.__queryHandler = () => [
|
||||||
|
{ id: 'enabled-job', enabled: 1 },
|
||||||
|
{ id: 'disabled-job', enabled: 0 },
|
||||||
|
];
|
||||||
|
const jobs = jobStorage.getJobs({ includeDisabled: true });
|
||||||
|
expect(jobs.find((j) => j.id === 'enabled-job').enabled).toBe(true);
|
||||||
|
expect(jobs.find((j) => j.id === 'disabled-job').enabled).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
244
test/storage/listingStatus.test.js
Normal file
244
test/storage/listingStatus.test.js
Normal file
@@ -0,0 +1,244 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (c) 2026 by Christian Kellner.
|
||||||
|
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { vi, describe, it, expect, beforeEach } from 'vitest';
|
||||||
|
|
||||||
|
// We mock SqliteConnection so we can assert which SQL the storage layer
|
||||||
|
// runs and with which params, without spinning up a real SQLite DB.
|
||||||
|
|
||||||
|
const calls = {
|
||||||
|
execute: [],
|
||||||
|
query: [],
|
||||||
|
};
|
||||||
|
|
||||||
|
const sqliteMock = {
|
||||||
|
execute: (sql, params) => {
|
||||||
|
calls.execute.push({ sql, params });
|
||||||
|
// Default: pretend 1 row was affected (so setListingStatus reports success).
|
||||||
|
return { changes: 1 };
|
||||||
|
},
|
||||||
|
query: (sql, params) => {
|
||||||
|
calls.query.push({ sql, params });
|
||||||
|
// Return shape varies by test — overridden via queryHandler when needed.
|
||||||
|
if (sqliteMock.__queryHandler) return sqliteMock.__queryHandler(sql, params);
|
||||||
|
return [];
|
||||||
|
},
|
||||||
|
__queryHandler: null,
|
||||||
|
};
|
||||||
|
|
||||||
|
vi.mock('../../lib/services/storage/SqliteConnection.js', () => ({
|
||||||
|
default: sqliteMock,
|
||||||
|
}));
|
||||||
|
|
||||||
|
describe('listingsStorage.setListingStatus', () => {
|
||||||
|
let listingsStorage;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
calls.execute.length = 0;
|
||||||
|
calls.query.length = 0;
|
||||||
|
sqliteMock.__queryHandler = null;
|
||||||
|
listingsStorage = await import('../../lib/services/storage/listingsStorage.js');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('runs an UPDATE storing a JSON payload with status and setAt', () => {
|
||||||
|
const before = Date.now();
|
||||||
|
const changes = listingsStorage.setListingStatus('listing-1', 'Applied');
|
||||||
|
const after = Date.now();
|
||||||
|
expect(changes).toBe(1);
|
||||||
|
expect(calls.execute).toHaveLength(1);
|
||||||
|
expect(calls.execute[0].sql).toMatch(/UPDATE listings SET status = @status WHERE id = @id/);
|
||||||
|
expect(calls.execute[0].params.id).toBe('listing-1');
|
||||||
|
const parsed = JSON.parse(calls.execute[0].params.status);
|
||||||
|
expect(parsed.status).toBe('applied');
|
||||||
|
expect(parsed.setAt).toBeGreaterThanOrEqual(before);
|
||||||
|
expect(parsed.setAt).toBeLessThanOrEqual(after);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts null to clear the status (no JSON wrapping)', () => {
|
||||||
|
listingsStorage.setListingStatus('listing-2', null);
|
||||||
|
expect(calls.execute[0].params).toEqual({ id: 'listing-2', status: null });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects invalid statuses', () => {
|
||||||
|
expect(() => listingsStorage.setListingStatus('listing-3', 'maybe')).toThrow(/Invalid listing status/);
|
||||||
|
expect(calls.execute).toHaveLength(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns 0 when no id is supplied (no SQL is run)', () => {
|
||||||
|
const result = listingsStorage.setListingStatus(null, 'applied');
|
||||||
|
expect(result).toBe(0);
|
||||||
|
expect(calls.execute).toHaveLength(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('listingsStorage.queryListings statusFilter', () => {
|
||||||
|
let listingsStorage;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
calls.execute.length = 0;
|
||||||
|
calls.query.length = 0;
|
||||||
|
// Return empty rows for both the count and the page-fetch queries.
|
||||||
|
sqliteMock.__queryHandler = (sql) => {
|
||||||
|
if (/COUNT\(1\)/.test(sql)) return [{ cnt: 0 }];
|
||||||
|
return [];
|
||||||
|
};
|
||||||
|
listingsStorage = await import('../../lib/services/storage/listingsStorage.js');
|
||||||
|
});
|
||||||
|
|
||||||
|
it("adds 'l.status IS NULL' to WHERE when statusFilter is 'none'", () => {
|
||||||
|
listingsStorage.queryListings({ statusFilter: 'none', userId: 'u1', isAdmin: true });
|
||||||
|
const pageQuery = calls.query.find((c) => !/COUNT\(1\)/.test(c.sql));
|
||||||
|
expect(pageQuery.sql).toMatch(/\(l\.status IS NULL\)/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('extracts the inner status field via json_extract for a concrete status', () => {
|
||||||
|
listingsStorage.queryListings({ statusFilter: 'applied', userId: 'u1', isAdmin: true });
|
||||||
|
const pageQuery = calls.query.find((c) => !/COUNT\(1\)/.test(c.sql));
|
||||||
|
expect(pageQuery.sql).toMatch(/json_extract\(l\.status, '\$\.status'\) = @statusValue/);
|
||||||
|
expect(pageQuery.params.statusValue).toBe('applied');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('ignores unknown statusFilter values silently', () => {
|
||||||
|
listingsStorage.queryListings({ statusFilter: 'bogus', userId: 'u1', isAdmin: true });
|
||||||
|
const pageQuery = calls.query.find((c) => !/COUNT\(1\)/.test(c.sql));
|
||||||
|
expect(pageQuery.sql).not.toMatch(/status/i);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('parses the JSON status payload of returned rows into an object', () => {
|
||||||
|
sqliteMock.__queryHandler = (sql) => {
|
||||||
|
if (/COUNT\(1\)/.test(sql)) return [{ cnt: 2 }];
|
||||||
|
return [
|
||||||
|
{ id: 'a', status: JSON.stringify({ status: 'applied', setAt: 1700000000000 }) },
|
||||||
|
{ id: 'b', status: null },
|
||||||
|
];
|
||||||
|
};
|
||||||
|
const result = listingsStorage.queryListings({ userId: 'u1', isAdmin: true });
|
||||||
|
expect(result.result[0].status).toEqual({ status: 'applied', setAt: 1700000000000 });
|
||||||
|
expect(result.result[1].status).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('listingsStorage.queryListings hiddenOnly', () => {
|
||||||
|
let listingsStorage;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
calls.execute.length = 0;
|
||||||
|
calls.query.length = 0;
|
||||||
|
sqliteMock.__queryHandler = (sql) => {
|
||||||
|
if (/COUNT\(1\)/.test(sql)) return [{ cnt: 0 }];
|
||||||
|
return [];
|
||||||
|
};
|
||||||
|
listingsStorage = await import('../../lib/services/storage/listingsStorage.js');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('filters by manually_deleted = 0 by default', () => {
|
||||||
|
listingsStorage.queryListings({ userId: 'u1', isAdmin: true });
|
||||||
|
const pageQuery = calls.query.find((c) => !/COUNT\(1\)/.test(c.sql));
|
||||||
|
expect(pageQuery.sql).toMatch(/\(l\.manually_deleted = 0\)/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('filters by manually_deleted = 1 when hiddenOnly is true', () => {
|
||||||
|
listingsStorage.queryListings({ userId: 'u1', isAdmin: true, hiddenOnly: true });
|
||||||
|
const pageQuery = calls.query.find((c) => !/COUNT\(1\)/.test(c.sql));
|
||||||
|
expect(pageQuery.sql).toMatch(/\(l\.manually_deleted = 1\)/);
|
||||||
|
expect(pageQuery.sql).not.toMatch(/\(l\.manually_deleted = 0\)/);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('listingsStorage.restoreListingsById', () => {
|
||||||
|
let listingsStorage;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
calls.execute.length = 0;
|
||||||
|
calls.query.length = 0;
|
||||||
|
sqliteMock.__queryHandler = null;
|
||||||
|
listingsStorage = await import('../../lib/services/storage/listingsStorage.js');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('clears the manually_deleted flag for the given ids', () => {
|
||||||
|
listingsStorage.restoreListingsById(['a', 'b']);
|
||||||
|
expect(calls.execute).toHaveLength(1);
|
||||||
|
expect(calls.execute[0].sql).toMatch(/UPDATE listings\s+SET manually_deleted = 0\s+WHERE id IN \(\?,\?\)/);
|
||||||
|
expect(calls.execute[0].params).toEqual(['a', 'b']);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('is a no-op when ids are missing or empty', () => {
|
||||||
|
listingsStorage.restoreListingsById([]);
|
||||||
|
listingsStorage.restoreListingsById(undefined);
|
||||||
|
expect(calls.execute).toHaveLength(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('listingsStorage.getListingById', () => {
|
||||||
|
let listingsStorage;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
calls.execute.length = 0;
|
||||||
|
calls.query.length = 0;
|
||||||
|
listingsStorage = await import('../../lib/services/storage/listingsStorage.js');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('parses the JSON status payload of the returned row', () => {
|
||||||
|
sqliteMock.__queryHandler = () => [
|
||||||
|
{ id: 'a', status: JSON.stringify({ status: 'rejected', setAt: 1700000000001 }) },
|
||||||
|
];
|
||||||
|
const row = listingsStorage.getListingById('a', 'u1', true);
|
||||||
|
expect(row.status).toEqual({ status: 'rejected', setAt: 1700000000001 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns null status untouched', () => {
|
||||||
|
sqliteMock.__queryHandler = () => [{ id: 'a', status: null }];
|
||||||
|
const row = listingsStorage.getListingById('a', 'u1', true);
|
||||||
|
expect(row.status).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns null when no row is found', () => {
|
||||||
|
sqliteMock.__queryHandler = () => [];
|
||||||
|
const row = listingsStorage.getListingById('missing', 'u1', true);
|
||||||
|
expect(row).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('watchListStorage.ensureWatch', () => {
|
||||||
|
let watchListStorage;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
calls.execute.length = 0;
|
||||||
|
calls.query.length = 0;
|
||||||
|
sqliteMock.__queryHandler = null;
|
||||||
|
watchListStorage = await import('../../lib/services/storage/watchListStorage.js');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('inserts and reports watched=true on first call', () => {
|
||||||
|
// After INSERT, createWatch queries for existence and gets a row back.
|
||||||
|
sqliteMock.__queryHandler = () => [{ ok: 1 }];
|
||||||
|
const result = watchListStorage.ensureWatch('listing-1', 'user-1');
|
||||||
|
expect(result).toEqual({ watched: true });
|
||||||
|
// INSERT should have been issued.
|
||||||
|
expect(calls.execute.some((c) => /INSERT INTO watch_list/.test(c.sql))).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns watched=true when an entry already exists', () => {
|
||||||
|
// Simulate ON CONFLICT being a no-op: execute reports no changes, then SELECT confirms row exists.
|
||||||
|
sqliteMock.execute = (sql, params) => {
|
||||||
|
calls.execute.push({ sql, params });
|
||||||
|
return { changes: 0 };
|
||||||
|
};
|
||||||
|
sqliteMock.__queryHandler = () => [{ ok: 1 }];
|
||||||
|
const result = watchListStorage.ensureWatch('listing-2', 'user-2');
|
||||||
|
expect(result).toEqual({ watched: true });
|
||||||
|
// Restore execute to default for subsequent tests.
|
||||||
|
sqliteMock.execute = (sql, params) => {
|
||||||
|
calls.execute.push({ sql, params });
|
||||||
|
return { changes: 1 };
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns watched=false when listingId or userId is missing', () => {
|
||||||
|
expect(watchListStorage.ensureWatch(null, 'u')).toEqual({ watched: false });
|
||||||
|
expect(watchListStorage.ensureWatch('l', null)).toEqual({ watched: false });
|
||||||
|
expect(calls.execute).toHaveLength(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -155,6 +155,7 @@ const routes = {
|
|||||||
'GET /api/dashboard': dashboard,
|
'GET /api/dashboard': dashboard,
|
||||||
'GET /api/demo': { demoMode: false },
|
'GET /api/demo': { demoMode: false },
|
||||||
'POST /api/user/settings/news-hash': {},
|
'POST /api/user/settings/news-hash': {},
|
||||||
|
'POST /api/user/settings/listing-deletion-preference': {},
|
||||||
};
|
};
|
||||||
|
|
||||||
const server = http.createServer((req, res) => {
|
const server = http.createServer((req, res) => {
|
||||||
|
|||||||
170
ui/src/App.jsx
170
ui/src/App.jsx
@@ -11,14 +11,14 @@ import GeneralSettings from './views/generalSettings/GeneralSettings';
|
|||||||
import JobMutation from './views/jobs/mutation/JobMutation';
|
import JobMutation from './views/jobs/mutation/JobMutation';
|
||||||
import UserMutator from './views/user/mutation/UserMutator';
|
import UserMutator from './views/user/mutation/UserMutator';
|
||||||
import { useActions, useSelector } from './services/state/store';
|
import { useActions, useSelector } from './services/state/store';
|
||||||
import { Routes, Route, Navigate } from 'react-router-dom';
|
import { Routes, Route, Navigate, useLocation } from 'react-router-dom';
|
||||||
import Login from './views/login/Login';
|
import Login from './views/login/Login';
|
||||||
import Users from './views/user/Users';
|
import Users from './views/user/Users';
|
||||||
import Jobs from './views/jobs/Jobs';
|
import Jobs from './views/jobs/Jobs';
|
||||||
|
|
||||||
import './App.less';
|
import './App.less';
|
||||||
import TrackingModal from './components/tracking/TrackingModal.jsx';
|
import TrackingModal from './components/tracking/TrackingModal.jsx';
|
||||||
import { Banner } from '@douyinfe/semi-ui-19';
|
import { Banner, LocaleProvider } from '@douyinfe/semi-ui-19';
|
||||||
import VersionBanner from './components/version/VersionBanner.jsx';
|
import VersionBanner from './components/version/VersionBanner.jsx';
|
||||||
import Listings from './views/listings/Listings.jsx';
|
import Listings from './views/listings/Listings.jsx';
|
||||||
import MapView from './views/listings/Map.jsx';
|
import MapView from './views/listings/Map.jsx';
|
||||||
@@ -29,13 +29,27 @@ import WatchlistManagement from './views/listings/management/WatchlistManagement
|
|||||||
import Dashboard from './views/dashboard/Dashboard.jsx';
|
import Dashboard from './views/dashboard/Dashboard.jsx';
|
||||||
import ListingDetail from './views/listings/ListingDetail.jsx';
|
import ListingDetail from './views/listings/ListingDetail.jsx';
|
||||||
import NewsModal from './components/news/NewsModal.jsx';
|
import NewsModal from './components/news/NewsModal.jsx';
|
||||||
|
import { I18nProvider, availableLanguages } from './services/i18n/i18n.jsx';
|
||||||
|
import DebugLoggingBanner from './components/debug/DebugLoggingBanner.jsx';
|
||||||
|
|
||||||
|
const semiLocaleModules = import.meta.glob('/node_modules/@douyinfe/semi-ui-19/lib/es/locale/source/*.js', {
|
||||||
|
eager: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
const semiLocales = {};
|
||||||
|
for (const [path, mod] of Object.entries(semiLocaleModules)) {
|
||||||
|
const name = path.match(/\/source\/(\w+)\.js$/)?.[1];
|
||||||
|
if (name) semiLocales[name] = mod.default ?? mod;
|
||||||
|
}
|
||||||
|
|
||||||
export default function FredyApp() {
|
export default function FredyApp() {
|
||||||
|
const location = useLocation();
|
||||||
const actions = useActions();
|
const actions = useActions();
|
||||||
const [loading, setLoading] = React.useState(true);
|
const [loading, setLoading] = React.useState(true);
|
||||||
const currentUser = useSelector((state) => state.user.currentUser);
|
const currentUser = useSelector((state) => state.user.currentUser);
|
||||||
const versionUpdate = useSelector((state) => state.versionUpdate.versionUpdate);
|
const versionUpdate = useSelector((state) => state.versionUpdate.versionUpdate);
|
||||||
const settings = useSelector((state) => state.generalSettings.settings);
|
const settings = useSelector((state) => state.generalSettings.settings);
|
||||||
|
const language = useSelector((state) => state.userSettings.settings.language);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
async function init() {
|
async function init() {
|
||||||
@@ -63,78 +77,90 @@ export default function FredyApp() {
|
|||||||
const isAdmin = () => currentUser != null && currentUser.isAdmin;
|
const isAdmin = () => currentUser != null && currentUser.isAdmin;
|
||||||
const { Sider, Content } = Layout;
|
const { Sider, Content } = Layout;
|
||||||
|
|
||||||
return loading ? null : needsLogin() ? (
|
return loading ? null : (
|
||||||
<Routes>
|
<I18nProvider language={language ?? 'en'}>
|
||||||
<Route path="/login" element={<Login />} />
|
<LocaleProvider
|
||||||
<Route path="*" element={<Navigate to="/login" replace />} />
|
locale={
|
||||||
</Routes>
|
semiLocales[availableLanguages.find((l) => l.code === (language ?? 'en'))?.semiLocale] ?? semiLocales['en_US']
|
||||||
) : (
|
}
|
||||||
<Layout className="app">
|
>
|
||||||
<Sider>
|
{needsLogin() ? (
|
||||||
<Navigation isAdmin={isAdmin()} />
|
|
||||||
</Sider>
|
|
||||||
<Layout className="app__main">
|
|
||||||
<Content className="app__content">
|
|
||||||
{versionUpdate?.newVersion && <VersionBanner />}
|
|
||||||
{settings.demoMode && (
|
|
||||||
<>
|
|
||||||
<Banner
|
|
||||||
fullMode={true}
|
|
||||||
type="info"
|
|
||||||
bordered
|
|
||||||
closeIcon={null}
|
|
||||||
description="You're currently viewing the demo version of Fredy. Jobs won't scrape websites, and any changes you make will be reverted at midnight."
|
|
||||||
/>
|
|
||||||
<br />
|
|
||||||
</>
|
|
||||||
)}
|
|
||||||
{settings.analyticsEnabled === null && !settings.demoMode && <TrackingModal />}
|
|
||||||
{!settings.demoMode && <NewsModal />}
|
|
||||||
<Routes>
|
<Routes>
|
||||||
<Route path="/403" element={<InsufficientPermission />} />
|
<Route path="/login" element={<Login />} />
|
||||||
<Route path="/jobs/new" element={<JobMutation />} />
|
<Route path="*" element={<Navigate state={{ from: location }} to="/login" replace />} />
|
||||||
<Route path="/jobs/edit/:jobId" element={<JobMutation />} />
|
|
||||||
<Route path="/dashboard" element={<Dashboard />} />
|
|
||||||
<Route path="/jobs" element={<Jobs />} />
|
|
||||||
<Route path="/listings" element={<Listings />} />
|
|
||||||
<Route path="/listings/listing/:listingId" element={<ListingDetail />} />
|
|
||||||
<Route path="/map" element={<MapView />} />
|
|
||||||
<Route path="/watchlistManagement" element={<WatchlistManagement />} />
|
|
||||||
|
|
||||||
{/* Permission-aware routes */}
|
|
||||||
<Route
|
|
||||||
path="/users/new"
|
|
||||||
element={
|
|
||||||
<PermissionAwareRoute currentUser={currentUser}>
|
|
||||||
<UserMutator />
|
|
||||||
</PermissionAwareRoute>
|
|
||||||
}
|
|
||||||
/>
|
|
||||||
<Route
|
|
||||||
path="/users/edit/:userId"
|
|
||||||
element={
|
|
||||||
<PermissionAwareRoute currentUser={currentUser}>
|
|
||||||
<UserMutator />
|
|
||||||
</PermissionAwareRoute>
|
|
||||||
}
|
|
||||||
/>
|
|
||||||
<Route
|
|
||||||
path="/users"
|
|
||||||
element={
|
|
||||||
<PermissionAwareRoute currentUser={currentUser}>
|
|
||||||
<Users />
|
|
||||||
</PermissionAwareRoute>
|
|
||||||
}
|
|
||||||
/>
|
|
||||||
<Route path="/userSettings" element={<Navigate to="/generalSettings" replace />} />
|
|
||||||
<Route path="/generalSettings" element={<GeneralSettings />} />
|
|
||||||
|
|
||||||
<Route path="/" element={<Navigate to="/dashboard" replace />} />
|
|
||||||
</Routes>
|
</Routes>
|
||||||
</Content>
|
) : (
|
||||||
<FredyFooter />
|
<Layout className="app">
|
||||||
</Layout>
|
<Sider>
|
||||||
</Layout>
|
<Navigation isAdmin={isAdmin()} />
|
||||||
|
</Sider>
|
||||||
|
<Layout className="app__main">
|
||||||
|
<Content className="app__content">
|
||||||
|
{versionUpdate?.newVersion && <VersionBanner />}
|
||||||
|
<DebugLoggingBanner />
|
||||||
|
{settings.demoMode && (
|
||||||
|
<>
|
||||||
|
<Banner
|
||||||
|
fullMode={true}
|
||||||
|
type="info"
|
||||||
|
bordered
|
||||||
|
closeIcon={null}
|
||||||
|
description="You're currently viewing the demo version of Fredy. Jobs won't scrape websites, and any changes you make will be reverted at midnight."
|
||||||
|
/>
|
||||||
|
<br />
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
{settings.analyticsEnabled === null && !settings.demoMode && <TrackingModal />}
|
||||||
|
{!settings.demoMode && <NewsModal />}
|
||||||
|
<Routes>
|
||||||
|
<Route path="/403" element={<InsufficientPermission />} />
|
||||||
|
<Route path="/jobs/new" element={<JobMutation />} />
|
||||||
|
<Route path="/jobs/edit/:jobId" element={<JobMutation />} />
|
||||||
|
<Route path="/dashboard" element={<Dashboard />} />
|
||||||
|
<Route path="/jobs" element={<Jobs />} />
|
||||||
|
<Route path="/listings" element={<Listings />} />
|
||||||
|
<Route path="/listings/watchlist" element={<Listings mode="watchlist" />} />
|
||||||
|
<Route path="/listings/listing/:listingId" element={<ListingDetail />} />
|
||||||
|
<Route path="/map" element={<MapView />} />
|
||||||
|
<Route path="/watchlistManagement" element={<WatchlistManagement />} />
|
||||||
|
|
||||||
|
{/* Permission-aware routes */}
|
||||||
|
<Route
|
||||||
|
path="/users/new"
|
||||||
|
element={
|
||||||
|
<PermissionAwareRoute currentUser={currentUser}>
|
||||||
|
<UserMutator />
|
||||||
|
</PermissionAwareRoute>
|
||||||
|
}
|
||||||
|
/>
|
||||||
|
<Route
|
||||||
|
path="/users/edit/:userId"
|
||||||
|
element={
|
||||||
|
<PermissionAwareRoute currentUser={currentUser}>
|
||||||
|
<UserMutator />
|
||||||
|
</PermissionAwareRoute>
|
||||||
|
}
|
||||||
|
/>
|
||||||
|
<Route
|
||||||
|
path="/users"
|
||||||
|
element={
|
||||||
|
<PermissionAwareRoute currentUser={currentUser}>
|
||||||
|
<Users />
|
||||||
|
</PermissionAwareRoute>
|
||||||
|
}
|
||||||
|
/>
|
||||||
|
<Route path="/userSettings" element={<Navigate to="/generalSettings" replace />} />
|
||||||
|
<Route path="/generalSettings" element={<GeneralSettings />} />
|
||||||
|
|
||||||
|
<Route path="/" element={<Navigate to="/dashboard" replace />} />
|
||||||
|
</Routes>
|
||||||
|
</Content>
|
||||||
|
<FredyFooter />
|
||||||
|
</Layout>
|
||||||
|
</Layout>
|
||||||
|
)}
|
||||||
|
</LocaleProvider>
|
||||||
|
</I18nProvider>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
BIN
ui/src/assets/news/1.jpg
Normal file
BIN
ui/src/assets/news/1.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 112 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 835 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 367 KiB |
@@ -1,16 +1,11 @@
|
|||||||
{
|
{
|
||||||
"key": "00e6b81777a275f5a140fc9101cb943810db6a69f6eb3927319c5aee0c876221",
|
"key": "00e6b81777a275f5a140fc9101cb94eef0db6a69f6eb3927319c5aee0c876221",
|
||||||
"content":
|
"content":
|
||||||
[
|
[
|
||||||
{
|
{
|
||||||
"title": "Table overview for listings",
|
"title": "Fredy goes multilingual!",
|
||||||
"text": "Thanks to https://github.com/datenwurm, we now have a table overview for listings. If you decide to use the table view, the decision will be stored.",
|
"text": "Fredy now supports multiple languages (Starting with german and english). You can select the language in the user-settings.",
|
||||||
"media": "1.png"
|
"media": "1.jpg"
|
||||||
},
|
|
||||||
{
|
|
||||||
"title": "Table overview for jobs",
|
|
||||||
"text": "Based on datenwurm's, work, I created a table overview for jobs. If you decide to use the table view, the decision will be stored.",
|
|
||||||
"media": "2.png"
|
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,8 +3,9 @@
|
|||||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { useState } from 'react';
|
import { useState, useEffect } from 'react';
|
||||||
import { Modal, Radio, RadioGroup, Typography } from '@douyinfe/semi-ui-19';
|
import { Modal, Radio, RadioGroup, Typography, Checkbox } from '@douyinfe/semi-ui-19';
|
||||||
|
import { useTranslation } from '../services/i18n/i18n.jsx';
|
||||||
|
|
||||||
const { Text } = Typography;
|
const { Text } = Typography;
|
||||||
|
|
||||||
@@ -12,56 +13,71 @@ const ListingDeletionModal = ({
|
|||||||
visible,
|
visible,
|
||||||
onConfirm,
|
onConfirm,
|
||||||
onCancel,
|
onCancel,
|
||||||
title = 'Delete Listings',
|
title,
|
||||||
showOptions = true,
|
showOptions = true,
|
||||||
message = 'How would you like to delete the selected listing(s)?',
|
message,
|
||||||
|
defaultDeleteType = 'soft',
|
||||||
}) => {
|
}) => {
|
||||||
|
const t = useTranslation();
|
||||||
|
const resolvedTitle = title ?? t('listing.deletion.title');
|
||||||
|
const resolvedMessage = message ?? t('listing.deletion.message');
|
||||||
const [deleteType, setDeleteType] = useState('soft');
|
const [deleteType, setDeleteType] = useState('soft');
|
||||||
|
const [remember, setRemember] = useState(false);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (visible) {
|
||||||
|
setDeleteType(defaultDeleteType);
|
||||||
|
setRemember(false);
|
||||||
|
}
|
||||||
|
}, [visible, defaultDeleteType]);
|
||||||
|
|
||||||
const handleOk = () => {
|
const handleOk = () => {
|
||||||
onConfirm(!showOptions || deleteType === 'hard');
|
if (showOptions) {
|
||||||
|
onConfirm(deleteType === 'hard', remember);
|
||||||
|
} else {
|
||||||
|
onConfirm(true);
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<Modal
|
<Modal
|
||||||
title={title}
|
title={resolvedTitle}
|
||||||
visible={visible}
|
visible={visible}
|
||||||
onOk={handleOk}
|
onOk={handleOk}
|
||||||
onCancel={onCancel}
|
onCancel={onCancel}
|
||||||
okText="Confirm"
|
okText={t('listing.deletion.confirm')}
|
||||||
cancelText="Cancel"
|
cancelText={t('listing.deletion.cancel')}
|
||||||
style={{ maxWidth: '500px' }}
|
style={{ maxWidth: '500px' }}
|
||||||
>
|
>
|
||||||
<div style={{ marginBottom: 16 }}>
|
<div style={{ marginBottom: 16 }}>
|
||||||
<Text>{message}</Text>
|
<Text>{resolvedMessage}</Text>
|
||||||
</div>
|
</div>
|
||||||
{showOptions && (
|
{showOptions && (
|
||||||
<RadioGroup value={deleteType} onChange={(e) => setDeleteType(e.target.value)} style={{ width: '100%' }}>
|
<>
|
||||||
<Radio value="soft" style={{ alignItems: 'flex-start', width: '100%' }}>
|
<RadioGroup value={deleteType} onChange={(e) => setDeleteType(e.target.value)} style={{ width: '100%' }}>
|
||||||
<div style={{ marginLeft: 8 }}>
|
<Radio value="soft" style={{ alignItems: 'flex-start', width: '100%' }}>
|
||||||
<Text strong>Mark as deleted (Soft Delete)</Text>
|
<div style={{ marginLeft: 8 }}>
|
||||||
<br />
|
<Text strong>{t('listing.deletion.softLabel')}</Text>
|
||||||
<Text type="secondary">
|
|
||||||
Listings are kept in the database but marked as hidden. They will <b>not</b> re-appear during the next
|
|
||||||
scraping session.
|
|
||||||
</Text>
|
|
||||||
</div>
|
|
||||||
</Radio>
|
|
||||||
<Radio value="hard" style={{ marginTop: 16, alignItems: 'flex-start', width: '100%' }}>
|
|
||||||
<div style={{ marginLeft: 8 }}>
|
|
||||||
<Text strong>Remove from database (Hard Delete)</Text>
|
|
||||||
<br />
|
|
||||||
<Text type="secondary">
|
|
||||||
Listings are completely removed from the database.
|
|
||||||
<br />
|
<br />
|
||||||
<Text type="warning">
|
<Text type="secondary">{t('listing.deletion.softDescription')}</Text>
|
||||||
Consequence: They might re-appear when scraping the next time because Fredy won't know they were
|
</div>
|
||||||
previously found.
|
</Radio>
|
||||||
|
<Radio value="hard" style={{ marginTop: 16, alignItems: 'flex-start', width: '100%' }}>
|
||||||
|
<div style={{ marginLeft: 8 }}>
|
||||||
|
<Text strong>{t('listing.deletion.hardLabel')}</Text>
|
||||||
|
<br />
|
||||||
|
<Text type="secondary">
|
||||||
|
{t('listing.deletion.hardDescription')}
|
||||||
|
<br />
|
||||||
|
<Text type="warning">{t('listing.deletion.hardConsequence')}</Text>
|
||||||
</Text>
|
</Text>
|
||||||
</Text>
|
</div>
|
||||||
</div>
|
</Radio>
|
||||||
</Radio>
|
</RadioGroup>
|
||||||
</RadioGroup>
|
<Checkbox checked={remember} onChange={(e) => setRemember(e.target.checked)} style={{ marginTop: 16 }}>
|
||||||
|
{t('listing.deletion.rememberChoice')}
|
||||||
|
</Checkbox>
|
||||||
|
</>
|
||||||
)}
|
)}
|
||||||
</Modal>
|
</Modal>
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -8,10 +8,12 @@ import { Pie } from 'react-chartjs-2';
|
|||||||
import { Chart as ChartJS, ArcElement, Tooltip, Legend, Title as ChartTitle } from 'chart.js';
|
import { Chart as ChartJS, ArcElement, Tooltip, Legend, Title as ChartTitle } from 'chart.js';
|
||||||
|
|
||||||
import './ChartCard.less';
|
import './ChartCard.less';
|
||||||
|
import { useTranslation } from '../../services/i18n/i18n.jsx';
|
||||||
|
|
||||||
ChartJS.register(ArcElement, Tooltip, Legend, ChartTitle);
|
ChartJS.register(ArcElement, Tooltip, Legend, ChartTitle);
|
||||||
|
|
||||||
export default function PieChartCard({ data = [] }) {
|
export default function PieChartCard({ data = [] }) {
|
||||||
|
const t = useTranslation();
|
||||||
const { labels, values } = React.useMemo(() => {
|
const { labels, values } = React.useMemo(() => {
|
||||||
if (data && typeof data === 'object' && !Array.isArray(data)) {
|
if (data && typeof data === 'object' && !Array.isArray(data)) {
|
||||||
const lbls = Array.isArray(data.labels) ? data.labels : [];
|
const lbls = Array.isArray(data.labels) ? data.labels : [];
|
||||||
@@ -92,6 +94,12 @@ export default function PieChartCard({ data = [] }) {
|
|||||||
const isEmpty = !labels || labels.length === 0 || !values || values.length === 0;
|
const isEmpty = !labels || labels.length === 0 || !values || values.length === 0;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<>{isEmpty ? <div className="chartCard__no__data">No Data</div> : <Pie data={chartData} options={options} />}</>
|
<>
|
||||||
|
{isEmpty ? (
|
||||||
|
<div className="chartCard__no__data">{t('dashboard.noData')}</div>
|
||||||
|
) : (
|
||||||
|
<Pie data={chartData} options={options} />
|
||||||
|
)}
|
||||||
|
</>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
58
ui/src/components/debug/DebugLoggingBanner.jsx
Normal file
58
ui/src/components/debug/DebugLoggingBanner.jsx
Normal file
@@ -0,0 +1,58 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (c) 2026 by Christian Kellner.
|
||||||
|
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { useEffect, useState } from 'react';
|
||||||
|
import { Banner } from '@douyinfe/semi-ui-19';
|
||||||
|
|
||||||
|
import { useTranslation } from '../../services/i18n/i18n.jsx';
|
||||||
|
import { fetchDebugActive } from '../../services/debugLoggingClient.js';
|
||||||
|
|
||||||
|
const POLL_INTERVAL_MS = 15000;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Persistent, non-dismissable red banner shown on every page while the admin opt-in
|
||||||
|
* "Debug Logging" feature is active. Polls the lightweight `/api/debug/active` probe
|
||||||
|
* so every authenticated user (not just admins) sees the warning, without exposing
|
||||||
|
* the rest of the settings payload.
|
||||||
|
*
|
||||||
|
* Polling interval is intentionally generous (15s) because the value only changes
|
||||||
|
* when an admin toggles the feature, which happens at human speeds. The Debug tab
|
||||||
|
* itself uses its own 3s polling for the live progress bar inside Settings.
|
||||||
|
*
|
||||||
|
* @returns {JSX.Element|null}
|
||||||
|
*/
|
||||||
|
export default function DebugLoggingBanner() {
|
||||||
|
const t = useTranslation();
|
||||||
|
const [active, setActive] = useState(false);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
let cancelled = false;
|
||||||
|
const tick = async () => {
|
||||||
|
try {
|
||||||
|
const res = await fetchDebugActive();
|
||||||
|
if (!cancelled) setActive(Boolean(res?.enabled));
|
||||||
|
} catch {
|
||||||
|
// Best-effort probe: an unauthenticated 401 (e.g. session expired) simply
|
||||||
|
// hides the banner until the next successful poll.
|
||||||
|
}
|
||||||
|
};
|
||||||
|
tick();
|
||||||
|
const id = window.setInterval(tick, POLL_INTERVAL_MS);
|
||||||
|
return () => {
|
||||||
|
cancelled = true;
|
||||||
|
window.clearInterval(id);
|
||||||
|
};
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
if (!active) return null;
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<Banner fullMode={true} type="danger" bordered closeIcon={null} description={t('app.debugLoggingBanner')} />
|
||||||
|
<br />
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
DebugLoggingBanner.displayName = 'DebugLoggingBanner';
|
||||||
@@ -6,16 +6,18 @@
|
|||||||
import './FredyFooter.less';
|
import './FredyFooter.less';
|
||||||
import { useSelector } from '../../services/state/store.js';
|
import { useSelector } from '../../services/state/store.js';
|
||||||
import { Layout } from '@douyinfe/semi-ui-19';
|
import { Layout } from '@douyinfe/semi-ui-19';
|
||||||
|
import { useTranslation } from '../../services/i18n/i18n.jsx';
|
||||||
|
|
||||||
export default function FredyFooter() {
|
export default function FredyFooter() {
|
||||||
|
const t = useTranslation();
|
||||||
const { Footer } = Layout;
|
const { Footer } = Layout;
|
||||||
const version = useSelector((state) => state.versionUpdate.versionUpdate);
|
const version = useSelector((state) => state.versionUpdate.versionUpdate);
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<Footer className="fredyFooter">
|
<Footer className="fredyFooter">
|
||||||
<span className="fredyFooter__version">Fredy v{version?.localFredyVersion || 'N/A'}</span>
|
<span className="fredyFooter__version">Fredy v{version?.localFredyVersion || t('common.na')}</span>
|
||||||
<span className="fredyFooter__credit">
|
<span className="fredyFooter__credit">
|
||||||
Made with ❤️ by{' '}
|
{t('footer.madeWith')}{' '}
|
||||||
<a href="https://github.com/orangecoding" target="_blank" rel="noreferrer">
|
<a href="https://github.com/orangecoding" target="_blank" rel="noreferrer">
|
||||||
Christian Kellner
|
Christian Kellner
|
||||||
</a>
|
</a>
|
||||||
|
|||||||
@@ -48,18 +48,22 @@ import { IllustrationNoResult, IllustrationNoResultDark } from '@douyinfe/semi-i
|
|||||||
import JobsTable from '../../table/JobsTable.jsx';
|
import JobsTable from '../../table/JobsTable.jsx';
|
||||||
|
|
||||||
import './JobGrid.less';
|
import './JobGrid.less';
|
||||||
|
import { useTranslation } from '../../../services/i18n/i18n.jsx';
|
||||||
|
|
||||||
const { Text, Title } = Typography;
|
const { Text, Title } = Typography;
|
||||||
|
|
||||||
const getPopoverContent = (text) => <article className="jobPopoverContent">{text}</article>;
|
const getPopoverContent = (text) => <article className="jobPopoverContent">{text}</article>;
|
||||||
|
|
||||||
const JobGrid = () => {
|
const JobGrid = () => {
|
||||||
|
const t = useTranslation();
|
||||||
const jobsData = useSelector((state) => state.jobsData);
|
const jobsData = useSelector((state) => state.jobsData);
|
||||||
const actions = useActions();
|
const actions = useActions();
|
||||||
const navigate = useNavigate();
|
const navigate = useNavigate();
|
||||||
|
|
||||||
const userSettings = useSelector((state) => state.userSettings.settings);
|
const userSettings = useSelector((state) => state.userSettings.settings);
|
||||||
const viewMode = userSettings?.jobs_view_mode ?? 'grid';
|
const viewMode = userSettings?.jobs_view_mode ?? 'grid';
|
||||||
|
const listingDeletionPref = userSettings?.listing_deletion_preference;
|
||||||
|
const defaultDeleteType = listingDeletionPref?.hardDelete ? 'hard' : 'soft';
|
||||||
|
|
||||||
const [page, setPage] = useState(1);
|
const [page, setPage] = useState(1);
|
||||||
const pageSize = 12;
|
const pageSize = 12;
|
||||||
@@ -102,7 +106,7 @@ const JobGrid = () => {
|
|||||||
actions.jobsData.setJobRunning(data.jobId, !!data.running);
|
actions.jobsData.setJobRunning(data.jobId, !!data.running);
|
||||||
// notify finish if it was triggered by this view
|
// notify finish if it was triggered by this view
|
||||||
if (pendingJobIdRef.current === data.jobId && data.running === false) {
|
if (pendingJobIdRef.current === data.jobId && data.running === false) {
|
||||||
Toast.success('Job finished');
|
Toast.success(t('jobs.toastFinished'));
|
||||||
pendingJobIdRef.current = null;
|
pendingJobIdRef.current = null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -142,26 +146,34 @@ const JobGrid = () => {
|
|||||||
};
|
};
|
||||||
|
|
||||||
const onListingRemoval = (jobId) => {
|
const onListingRemoval = (jobId) => {
|
||||||
setPendingDeletion({ type: 'listings', jobId });
|
const deletion = { type: 'listings', jobId };
|
||||||
|
if (listingDeletionPref?.skipPrompt) {
|
||||||
|
confirmDeletion(listingDeletionPref.hardDelete, false, deletion);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setPendingDeletion(deletion);
|
||||||
setDeleteModalVisible(true);
|
setDeleteModalVisible(true);
|
||||||
};
|
};
|
||||||
|
|
||||||
const confirmDeletion = async (hardDelete) => {
|
const confirmDeletion = async (hardDelete, remember, deletion = pendingDeletion) => {
|
||||||
const { type, jobId } = pendingDeletion;
|
const { type, jobId } = deletion;
|
||||||
try {
|
try {
|
||||||
|
if (remember && type === 'listings') {
|
||||||
|
await actions.userSettings.setListingDeletionPreference({ skipPrompt: true, hardDelete });
|
||||||
|
}
|
||||||
if (type === 'job') {
|
if (type === 'job') {
|
||||||
await xhrDelete('/api/jobs', { jobId });
|
await xhrDelete('/api/jobs', { jobId });
|
||||||
Toast.success('Job and listings successfully removed');
|
Toast.success(t('jobs.toastDeletedWithListings'));
|
||||||
} else if (type === 'listings') {
|
} else if (type === 'listings') {
|
||||||
await xhrDelete('/api/listings/job', { jobId, hardDelete });
|
await xhrDelete('/api/listings/job', { jobId, hardDelete });
|
||||||
Toast.success('Listings successfully removed');
|
Toast.success(t('jobs.toastListingsDeleted'));
|
||||||
}
|
}
|
||||||
loadData();
|
loadData();
|
||||||
if (type === 'job') {
|
if (type === 'job') {
|
||||||
actions.jobsData.getJobs(); // refresh select list too
|
actions.jobsData.getJobs(); // refresh select list too
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
Toast.error(error.message || 'Error performing deletion');
|
Toast.error(error.message || t('jobs.toastDeleteError'));
|
||||||
} finally {
|
} finally {
|
||||||
setDeleteModalVisible(false);
|
setDeleteModalVisible(false);
|
||||||
setPendingDeletion(null);
|
setPendingDeletion(null);
|
||||||
@@ -171,8 +183,9 @@ const JobGrid = () => {
|
|||||||
const onJobStatusChanged = async (jobId, status) => {
|
const onJobStatusChanged = async (jobId, status) => {
|
||||||
try {
|
try {
|
||||||
await xhrPut(`/api/jobs/${jobId}/status`, { status });
|
await xhrPut(`/api/jobs/${jobId}/status`, { status });
|
||||||
Toast.success('Job status successfully changed');
|
Toast.success(t('jobs.toastStatusChanged'));
|
||||||
loadData();
|
loadData();
|
||||||
|
actions.jobsData.getJobs(); // refresh the jobs slice read by the edit form so its switch isn't stale
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
Toast.error(error.error);
|
Toast.error(error.error);
|
||||||
}
|
}
|
||||||
@@ -182,21 +195,21 @@ const JobGrid = () => {
|
|||||||
try {
|
try {
|
||||||
const response = await xhrPost(`/api/jobs/${jobId}/run`);
|
const response = await xhrPost(`/api/jobs/${jobId}/run`);
|
||||||
if (response.status === 202) {
|
if (response.status === 202) {
|
||||||
Toast.success('Job run started');
|
Toast.success(t('jobs.toastRunStarted'));
|
||||||
} else {
|
} else {
|
||||||
Toast.info('Job run requested');
|
Toast.info(t('jobs.toastRunRequested'));
|
||||||
}
|
}
|
||||||
pendingJobIdRef.current = jobId;
|
pendingJobIdRef.current = jobId;
|
||||||
loadData();
|
loadData();
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (error?.status === 409) {
|
if (error?.status === 409) {
|
||||||
Toast.warning(error?.json?.message || 'Job is already running');
|
Toast.warning(error?.json?.message || t('jobs.toastAlreadyRunning'));
|
||||||
} else if (error?.status === 403) {
|
} else if (error?.status === 403) {
|
||||||
Toast.error('You are not allowed to run this job');
|
Toast.error(t('jobs.toastNotAllowed'));
|
||||||
} else if (error?.status === 404) {
|
} else if (error?.status === 404) {
|
||||||
Toast.error('Job not found');
|
Toast.error(t('jobs.toastNotFound'));
|
||||||
} else {
|
} else {
|
||||||
Toast.error('Failed to trigger job');
|
Toast.error(t('jobs.toastRunFailed'));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -212,7 +225,7 @@ const JobGrid = () => {
|
|||||||
className="jobGrid__topbar__search"
|
className="jobGrid__topbar__search"
|
||||||
prefix={<IconSearch />}
|
prefix={<IconSearch />}
|
||||||
showClear
|
showClear
|
||||||
placeholder="Search"
|
placeholder={t('jobs.searchPlaceholder')}
|
||||||
onChange={handleFilterChange}
|
onChange={handleFilterChange}
|
||||||
/>
|
/>
|
||||||
|
|
||||||
@@ -225,39 +238,44 @@ const JobGrid = () => {
|
|||||||
setActivityFilter(v === 'all' ? null : v === 'true');
|
setActivityFilter(v === 'all' ? null : v === 'true');
|
||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
<Radio value="all">All</Radio>
|
<Radio value="all">{t('jobs.filterAll')}</Radio>
|
||||||
<Radio value="true">Active</Radio>
|
<Radio value="true">{t('jobs.filterActive')}</Radio>
|
||||||
<Radio value="false">Inactive</Radio>
|
<Radio value="false">{t('jobs.filterInactive')}</Radio>
|
||||||
</RadioGroup>
|
</RadioGroup>
|
||||||
|
|
||||||
<Select prefix="Sort by" style={{ width: 200 }} value={sortField} onChange={(val) => setSortField(val)}>
|
<Select
|
||||||
<Select.Option value="name">Name</Select.Option>
|
prefix={t('jobs.sortPrefix')}
|
||||||
<Select.Option value="numberOfFoundListings">Number of Listings</Select.Option>
|
style={{ width: 200 }}
|
||||||
<Select.Option value="enabled">Status</Select.Option>
|
value={sortField}
|
||||||
|
onChange={(val) => setSortField(val)}
|
||||||
|
>
|
||||||
|
<Select.Option value="name">{t('jobs.sortByName')}</Select.Option>
|
||||||
|
<Select.Option value="numberOfFoundListings">{t('jobs.sortByListings')}</Select.Option>
|
||||||
|
<Select.Option value="enabled">{t('jobs.sortByStatus')}</Select.Option>
|
||||||
</Select>
|
</Select>
|
||||||
|
|
||||||
<Button
|
<Button
|
||||||
icon={sortDir === 'asc' ? <IconArrowUp /> : <IconArrowDown />}
|
icon={sortDir === 'asc' ? <IconArrowUp /> : <IconArrowDown />}
|
||||||
onClick={() => setSortDir((d) => (d === 'asc' ? 'desc' : 'asc'))}
|
onClick={() => setSortDir((d) => (d === 'asc' ? 'desc' : 'asc'))}
|
||||||
title={sortDir === 'asc' ? 'Ascending' : 'Descending'}
|
title={sortDir === 'asc' ? t('jobs.sortAscending') : t('jobs.sortDescending')}
|
||||||
/>
|
/>
|
||||||
|
|
||||||
<div className="jobGrid__topbar__view-toggle">
|
<div className="jobGrid__topbar__view-toggle">
|
||||||
<Tooltip content="Grid view">
|
<Tooltip content={t('jobs.tooltipGridView')}>
|
||||||
<Button
|
<Button
|
||||||
icon={<IconGridView />}
|
icon={<IconGridView />}
|
||||||
theme={viewMode === 'grid' ? 'solid' : 'borderless'}
|
theme={viewMode === 'grid' ? 'solid' : 'borderless'}
|
||||||
onClick={() => actions.userSettings.setJobsViewMode('grid')}
|
onClick={() => actions.userSettings.setJobsViewMode('grid')}
|
||||||
aria-label="Grid view"
|
aria-label={t('common.ariaGridView')}
|
||||||
aria-pressed={viewMode === 'grid'}
|
aria-pressed={viewMode === 'grid'}
|
||||||
/>
|
/>
|
||||||
</Tooltip>
|
</Tooltip>
|
||||||
<Tooltip content="Table view">
|
<Tooltip content={t('jobs.tooltipTableView')}>
|
||||||
<Button
|
<Button
|
||||||
icon={<IconList />}
|
icon={<IconList />}
|
||||||
theme={viewMode === 'table' ? 'solid' : 'borderless'}
|
theme={viewMode === 'table' ? 'solid' : 'borderless'}
|
||||||
onClick={() => actions.userSettings.setJobsViewMode('table')}
|
onClick={() => actions.userSettings.setJobsViewMode('table')}
|
||||||
aria-label="Table view"
|
aria-label={t('common.ariaTableView')}
|
||||||
aria-pressed={viewMode === 'table'}
|
aria-pressed={viewMode === 'table'}
|
||||||
/>
|
/>
|
||||||
</Tooltip>
|
</Tooltip>
|
||||||
@@ -268,7 +286,7 @@ const JobGrid = () => {
|
|||||||
<Empty
|
<Empty
|
||||||
image={<IllustrationNoResult />}
|
image={<IllustrationNoResult />}
|
||||||
darkModeImage={<IllustrationNoResultDark />}
|
darkModeImage={<IllustrationNoResultDark />}
|
||||||
description="No jobs available yet..."
|
description={t('jobs.empty')}
|
||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
@@ -286,7 +304,7 @@ const JobGrid = () => {
|
|||||||
</div>
|
</div>
|
||||||
<div style={{ display: 'flex', alignItems: 'center', gap: 6, flexShrink: 0 }}>
|
<div style={{ display: 'flex', alignItems: 'center', gap: 6, flexShrink: 0 }}>
|
||||||
{job.isOnlyShared && (
|
{job.isOnlyShared && (
|
||||||
<Popover content={getPopoverContent('This job has been shared with you — read only.')}>
|
<Popover content={getPopoverContent(t('jobs.cardSharedReadOnly'))}>
|
||||||
<div>
|
<div>
|
||||||
<IconAlertTriangle style={{ color: 'rgba(var(--semi-yellow-7), 1)' }} />
|
<IconAlertTriangle style={{ color: 'rgba(var(--semi-yellow-7), 1)' }} />
|
||||||
</div>
|
</div>
|
||||||
@@ -294,7 +312,7 @@ const JobGrid = () => {
|
|||||||
)}
|
)}
|
||||||
{job.running && (
|
{job.running && (
|
||||||
<Tag color="green" variant="light" size="small">
|
<Tag color="green" variant="light" size="small">
|
||||||
RUNNING
|
{t('jobs.cardRunning')}
|
||||||
</Tag>
|
</Tag>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
@@ -304,19 +322,19 @@ const JobGrid = () => {
|
|||||||
<div className="jobGrid__card__stat jobGrid__card__stat--blue">
|
<div className="jobGrid__card__stat jobGrid__card__stat--blue">
|
||||||
<span className="jobGrid__card__stat__number">{job.numberOfFoundListings || 0}</span>
|
<span className="jobGrid__card__stat__number">{job.numberOfFoundListings || 0}</span>
|
||||||
<span className="jobGrid__card__stat__label">
|
<span className="jobGrid__card__stat__label">
|
||||||
<IconHome size="small" /> Listings
|
<IconHome size="small" /> {t('jobs.cardListings')}
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
<div className="jobGrid__card__stat jobGrid__card__stat--orange">
|
<div className="jobGrid__card__stat jobGrid__card__stat--orange">
|
||||||
<span className="jobGrid__card__stat__number">{job.provider?.length || 0}</span>
|
<span className="jobGrid__card__stat__number">{job.provider?.length || 0}</span>
|
||||||
<span className="jobGrid__card__stat__label">
|
<span className="jobGrid__card__stat__label">
|
||||||
<IconBriefcase size="small" /> Providers
|
<IconBriefcase size="small" /> {t('jobs.cardProviders')}
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
<div className="jobGrid__card__stat jobGrid__card__stat--purple">
|
<div className="jobGrid__card__stat jobGrid__card__stat--purple">
|
||||||
<span className="jobGrid__card__stat__number">{job.notificationAdapter?.length || 0}</span>
|
<span className="jobGrid__card__stat__number">{job.notificationAdapter?.length || 0}</span>
|
||||||
<span className="jobGrid__card__stat__label">
|
<span className="jobGrid__card__stat__label">
|
||||||
<IconBell size="small" /> Adapters
|
<IconBell size="small" /> {t('jobs.cardAdapters')}
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@@ -332,11 +350,11 @@ const JobGrid = () => {
|
|||||||
size="small"
|
size="small"
|
||||||
/>
|
/>
|
||||||
<Text type="secondary" size="small">
|
<Text type="secondary" size="small">
|
||||||
Active
|
{t('jobs.cardActive')}
|
||||||
</Text>
|
</Text>
|
||||||
</div>
|
</div>
|
||||||
<div className="jobGrid__actions">
|
<div className="jobGrid__actions">
|
||||||
<Popover content={getPopoverContent('Run Job')}>
|
<Popover content={getPopoverContent(t('jobs.popoverRunJob'))}>
|
||||||
<div>
|
<div>
|
||||||
<Button
|
<Button
|
||||||
type="primary"
|
type="primary"
|
||||||
@@ -349,7 +367,7 @@ const JobGrid = () => {
|
|||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
</Popover>
|
</Popover>
|
||||||
<Popover content={getPopoverContent('Edit a Job')}>
|
<Popover content={getPopoverContent(t('jobs.popoverEditJob'))}>
|
||||||
<div>
|
<div>
|
||||||
<Button
|
<Button
|
||||||
type="secondary"
|
type="secondary"
|
||||||
@@ -360,7 +378,7 @@ const JobGrid = () => {
|
|||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
</Popover>
|
</Popover>
|
||||||
<Popover content={getPopoverContent('Clone Job')}>
|
<Popover content={getPopoverContent(t('jobs.popoverCloneJob'))}>
|
||||||
<div>
|
<div>
|
||||||
<Button
|
<Button
|
||||||
type="tertiary"
|
type="tertiary"
|
||||||
@@ -371,7 +389,7 @@ const JobGrid = () => {
|
|||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
</Popover>
|
</Popover>
|
||||||
<Popover content={getPopoverContent('Delete all found Listings of this Job')}>
|
<Popover content={getPopoverContent(t('jobs.popoverDeleteListings'))}>
|
||||||
<div>
|
<div>
|
||||||
<Button
|
<Button
|
||||||
type="danger"
|
type="danger"
|
||||||
@@ -382,7 +400,7 @@ const JobGrid = () => {
|
|||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
</Popover>
|
</Popover>
|
||||||
<Popover content={getPopoverContent('Delete Job')}>
|
<Popover content={getPopoverContent(t('jobs.popoverDeleteJob'))}>
|
||||||
<div>
|
<div>
|
||||||
<Button
|
<Button
|
||||||
type="danger"
|
type="danger"
|
||||||
@@ -423,13 +441,10 @@ const JobGrid = () => {
|
|||||||
)}
|
)}
|
||||||
<ListingDeletionModal
|
<ListingDeletionModal
|
||||||
visible={deleteModalVisible}
|
visible={deleteModalVisible}
|
||||||
title={pendingDeletion?.type === 'job' ? 'Delete Job' : 'Delete Listings'}
|
title={pendingDeletion?.type === 'job' ? t('jobs.deletion.title') : t('listing.deletion.title')}
|
||||||
showOptions={pendingDeletion?.type !== 'job'}
|
showOptions={pendingDeletion?.type !== 'job'}
|
||||||
message={
|
defaultDeleteType={defaultDeleteType}
|
||||||
pendingDeletion?.type === 'job'
|
message={pendingDeletion?.type === 'job' ? t('jobs.deletion.message') : t('listing.deletion.message')}
|
||||||
? 'Are you sure you want to delete this job? All associated listings will be removed from the database.'
|
|
||||||
: 'How would you like to delete the selected listing(s)?'
|
|
||||||
}
|
|
||||||
onConfirm={confirmDeletion}
|
onConfirm={confirmDeletion}
|
||||||
onCancel={() => {
|
onCancel={() => {
|
||||||
setDeleteModalVisible(false);
|
setDeleteModalVisible(false);
|
||||||
|
|||||||
@@ -16,113 +16,153 @@ import {
|
|||||||
} from '@douyinfe/semi-icons';
|
} from '@douyinfe/semi-icons';
|
||||||
import no_image from '../../../assets/no_image.png';
|
import no_image from '../../../assets/no_image.png';
|
||||||
import * as timeService from '../../../services/time/timeService.js';
|
import * as timeService from '../../../services/time/timeService.js';
|
||||||
|
import StatusControl from '../../listings/StatusControl.jsx';
|
||||||
|
|
||||||
import './ListingsGrid.less';
|
import './ListingsGrid.less';
|
||||||
|
import { useTranslation, useLocale } from '../../../services/i18n/i18n.jsx';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param {{ listings: object[], onWatch: Function, onNavigate: Function, onDelete: Function }} props
|
* @param {{ listings: object[], onWatch: Function, onNavigate: Function, onDelete: Function, onRestore?: Function, isHiddenView?: boolean, onStatusChange: Function }} props
|
||||||
*/
|
*/
|
||||||
const ListingsGrid = ({ listings, onWatch, onNavigate, onDelete }) => (
|
const ListingsGrid = ({ listings, onWatch, onNavigate, onDelete, onRestore, isHiddenView = false, onStatusChange }) => {
|
||||||
<div className="listingsGrid__grid">
|
const t = useTranslation();
|
||||||
{listings.map((item) => (
|
const locale = useLocale();
|
||||||
<div
|
return (
|
||||||
key={item.id}
|
<div className="listingsGrid__grid">
|
||||||
className="listingsGrid__card"
|
{listings.map((item) => (
|
||||||
style={{ cursor: 'pointer' }}
|
<div
|
||||||
role="button"
|
key={item.id}
|
||||||
tabIndex={0}
|
className="listingsGrid__card"
|
||||||
onClick={() => onNavigate(item.id)}
|
style={{ cursor: 'pointer' }}
|
||||||
onKeyDown={(e) => {
|
role="button"
|
||||||
if (e.key === 'Enter' || e.key === ' ') onNavigate(item.id);
|
tabIndex={0}
|
||||||
}}
|
onClick={() => onNavigate(item.id)}
|
||||||
>
|
onKeyDown={(e) => {
|
||||||
<div className="listingsGrid__card__image-wrapper">
|
if (e.key === 'Enter' || e.key === ' ') onNavigate(item.id);
|
||||||
<img
|
}}
|
||||||
src={item.image_url || no_image}
|
>
|
||||||
alt={item.title}
|
<div className="listingsGrid__card__image-wrapper">
|
||||||
onError={(e) => {
|
<img
|
||||||
e.target.src = no_image;
|
src={item.image_url || no_image}
|
||||||
}}
|
alt={item.title}
|
||||||
/>
|
onError={(e) => {
|
||||||
{!item.is_active && (
|
e.target.src = no_image;
|
||||||
<div className="listingsGrid__card__inactive-watermark">
|
}}
|
||||||
<span>Inactive</span>
|
/>
|
||||||
</div>
|
{!item.is_active && (
|
||||||
)}
|
<div className="listingsGrid__card__inactive-watermark">
|
||||||
<button
|
<span>{t('listings.cardInactive')}</span>
|
||||||
type="button"
|
</div>
|
||||||
className="listingsGrid__card__star"
|
)}
|
||||||
onClick={(e) => onWatch(e, item)}
|
<Tooltip
|
||||||
aria-label={item.isWatched === 1 ? 'Remove from watchlist' : 'Add to watchlist'}
|
content={
|
||||||
>
|
item.isWatched === 1 ? t('listings.tooltipRemoveFromWatchlist') : t('listings.tooltipAddToWatchlist')
|
||||||
{item.isWatched === 1 ? <IconStar /> : <IconStarStroked />}
|
}
|
||||||
</button>
|
>
|
||||||
</div>
|
<button
|
||||||
|
type="button"
|
||||||
<div className="listingsGrid__card__body">
|
className="listingsGrid__card__star"
|
||||||
<div className="listingsGrid__card__title" title={item.title}>
|
onClick={(e) => onWatch(e, item)}
|
||||||
{item.title}
|
aria-label={
|
||||||
|
item.isWatched === 1 ? t('listings.tooltipRemoveFromWatchlist') : t('listings.tooltipAddToWatchlist')
|
||||||
|
}
|
||||||
|
>
|
||||||
|
{item.isWatched === 1 ? <IconStar /> : <IconStarStroked />}
|
||||||
|
</button>
|
||||||
|
</Tooltip>
|
||||||
</div>
|
</div>
|
||||||
{item.price && (
|
|
||||||
<div className="listingsGrid__card__price">
|
<div className="listingsGrid__card__body">
|
||||||
<IconCart size="small" />
|
<div className="listingsGrid__card__title" title={item.title}>
|
||||||
{item.price}
|
{item.title}
|
||||||
</div>
|
</div>
|
||||||
)}
|
{item.price && (
|
||||||
{item.address && (
|
<div className="listingsGrid__card__price">
|
||||||
|
<IconCart size="small" />
|
||||||
|
{item.price}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{item.address && (
|
||||||
|
<div className="listingsGrid__card__meta">
|
||||||
|
<IconMapPin />
|
||||||
|
{item.address}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
<div className="listingsGrid__card__meta">
|
<div className="listingsGrid__card__meta">
|
||||||
<IconMapPin />
|
<IconBriefcase />
|
||||||
{item.address}
|
{item.provider}
|
||||||
</div>
|
</div>
|
||||||
)}
|
<div className="listingsGrid__card__provider">{timeService.format(item.created_at, false, locale)}</div>
|
||||||
<div className="listingsGrid__card__meta">
|
|
||||||
<IconBriefcase />
|
|
||||||
{item.provider}
|
|
||||||
</div>
|
</div>
|
||||||
<div className="listingsGrid__card__provider">{timeService.format(item.created_at, false)}</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div className="listingsGrid__card__actions" onClick={(e) => e.stopPropagation()}>
|
<div className="listingsGrid__card__actions" onClick={(e) => e.stopPropagation()}>
|
||||||
<Tooltip content="Original Listing">
|
<StatusControl
|
||||||
<Button
|
status={item.status?.status ?? null}
|
||||||
size="small"
|
compact
|
||||||
icon={<IconLink />}
|
onChange={(next) => onStatusChange?.(item, next)}
|
||||||
style={{ color: '#60a5fa' }}
|
onTriggerClick={(e) => e.stopPropagation()}
|
||||||
theme="borderless"
|
|
||||||
onClick={(e) => {
|
|
||||||
e.stopPropagation();
|
|
||||||
window.open(item.link, '_blank');
|
|
||||||
}}
|
|
||||||
/>
|
/>
|
||||||
</Tooltip>
|
<Tooltip content={t('listings.tooltipOriginalListing')}>
|
||||||
<Tooltip content="View in Fredy">
|
<Button
|
||||||
<Button
|
size="small"
|
||||||
size="small"
|
icon={<IconLink />}
|
||||||
icon={<IconEyeOpened />}
|
style={{ color: '#60a5fa' }}
|
||||||
style={{ color: '#34d399' }}
|
theme="borderless"
|
||||||
theme="borderless"
|
onClick={(e) => {
|
||||||
onClick={(e) => {
|
e.stopPropagation();
|
||||||
e.stopPropagation();
|
window.open(item.link, '_blank');
|
||||||
onNavigate(item.id);
|
}}
|
||||||
}}
|
/>
|
||||||
/>
|
</Tooltip>
|
||||||
</Tooltip>
|
<Tooltip content={t('listings.tooltipViewInFredy')}>
|
||||||
<Tooltip content="Remove">
|
<Button
|
||||||
<Button
|
size="small"
|
||||||
size="small"
|
icon={<IconEyeOpened />}
|
||||||
icon={<IconDelete />}
|
style={{ color: '#34d399' }}
|
||||||
style={{ color: '#fb7185' }}
|
theme="borderless"
|
||||||
theme="borderless"
|
onClick={(e) => {
|
||||||
onClick={(e) => {
|
e.stopPropagation();
|
||||||
e.stopPropagation();
|
onNavigate(item.id);
|
||||||
onDelete(item.id);
|
}}
|
||||||
}}
|
/>
|
||||||
/>
|
</Tooltip>
|
||||||
</Tooltip>
|
{isHiddenView ? (
|
||||||
|
<Tooltip content={t('listings.tooltipUndelete')}>
|
||||||
|
<Button
|
||||||
|
size="small"
|
||||||
|
icon={
|
||||||
|
<span className="listingsGrid__strike" aria-hidden="true">
|
||||||
|
<IconDelete />
|
||||||
|
</span>
|
||||||
|
}
|
||||||
|
style={{ color: '#34d399' }}
|
||||||
|
theme="borderless"
|
||||||
|
onClick={(e) => {
|
||||||
|
e.stopPropagation();
|
||||||
|
onRestore?.(item.id);
|
||||||
|
}}
|
||||||
|
aria-label={t('listings.tooltipUndelete')}
|
||||||
|
/>
|
||||||
|
</Tooltip>
|
||||||
|
) : (
|
||||||
|
<Tooltip content={t('listings.tooltipRemove')}>
|
||||||
|
<Button
|
||||||
|
size="small"
|
||||||
|
icon={<IconDelete />}
|
||||||
|
style={{ color: '#fb7185' }}
|
||||||
|
theme="borderless"
|
||||||
|
onClick={(e) => {
|
||||||
|
e.stopPropagation();
|
||||||
|
onDelete(item.id);
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</Tooltip>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
))}
|
||||||
))}
|
</div>
|
||||||
</div>
|
);
|
||||||
);
|
};
|
||||||
|
|
||||||
export default ListingsGrid;
|
export default ListingsGrid;
|
||||||
|
|||||||
@@ -11,12 +11,11 @@
|
|||||||
border: 1px solid @color-border !important;
|
border: 1px solid @color-border !important;
|
||||||
border-radius: @radius-card !important;
|
border-radius: @radius-card !important;
|
||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
transition: transform @transition-card, box-shadow @transition-card;
|
transition: box-shadow @transition-card;
|
||||||
display: flex;
|
display: flex;
|
||||||
flex-direction: column;
|
flex-direction: column;
|
||||||
|
|
||||||
&:hover {
|
&:hover {
|
||||||
transform: translateY(-2px);
|
|
||||||
box-shadow: 0 6px 24px -4px rgba(0,0,0,0.6);
|
box-shadow: 0 6px 24px -4px rgba(0,0,0,0.6);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -140,4 +139,23 @@
|
|||||||
border-radius: @radius-chip !important;
|
border-radius: @radius-chip !important;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
&__strike {
|
||||||
|
position: relative;
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
|
||||||
|
&::after {
|
||||||
|
content: '';
|
||||||
|
position: absolute;
|
||||||
|
left: -2px;
|
||||||
|
right: -2px;
|
||||||
|
top: 50%;
|
||||||
|
height: 2px;
|
||||||
|
background: currentColor;
|
||||||
|
transform: rotate(-45deg);
|
||||||
|
pointer-events: none;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,7 +10,18 @@ import {
|
|||||||
parseString,
|
parseString,
|
||||||
parseNullableBoolean,
|
parseNullableBoolean,
|
||||||
} from '../../hooks/useSearchParamState.js';
|
} from '../../hooks/useSearchParamState.js';
|
||||||
import { Button, Pagination, Toast, Input, Select, Empty, Radio, RadioGroup, Tooltip } from '@douyinfe/semi-ui-19';
|
import {
|
||||||
|
Button,
|
||||||
|
Pagination,
|
||||||
|
Toast,
|
||||||
|
Input,
|
||||||
|
Select,
|
||||||
|
Empty,
|
||||||
|
Radio,
|
||||||
|
RadioGroup,
|
||||||
|
Tooltip,
|
||||||
|
Banner,
|
||||||
|
} from '@douyinfe/semi-ui-19';
|
||||||
import { IconSearch, IconArrowUp, IconArrowDown, IconGridView, IconList } from '@douyinfe/semi-icons';
|
import { IconSearch, IconArrowUp, IconArrowDown, IconGridView, IconList } from '@douyinfe/semi-icons';
|
||||||
import { useNavigate, useSearchParams } from 'react-router-dom';
|
import { useNavigate, useSearchParams } from 'react-router-dom';
|
||||||
import ListingDeletionModal from '../ListingDeletionModal.jsx';
|
import ListingDeletionModal from '../ListingDeletionModal.jsx';
|
||||||
@@ -22,8 +33,11 @@ import ListingsTable from '../table/ListingsTable.jsx';
|
|||||||
import { IllustrationNoResult, IllustrationNoResultDark } from '@douyinfe/semi-illustrations';
|
import { IllustrationNoResult, IllustrationNoResultDark } from '@douyinfe/semi-illustrations';
|
||||||
|
|
||||||
import './ListingsOverview.less';
|
import './ListingsOverview.less';
|
||||||
|
import { useTranslation } from '../../services/i18n/i18n.jsx';
|
||||||
|
|
||||||
const ListingsOverview = () => {
|
const ListingsOverview = ({ mode = 'all' }) => {
|
||||||
|
const t = useTranslation();
|
||||||
|
const isWatchlistMode = mode === 'watchlist';
|
||||||
const listingsData = useSelector((state) => state.listingsData);
|
const listingsData = useSelector((state) => state.listingsData);
|
||||||
const providers = useSelector((state) => state.provider);
|
const providers = useSelector((state) => state.provider);
|
||||||
const jobs = useSelector((state) => state.jobsData.jobs);
|
const jobs = useSelector((state) => state.jobsData.jobs);
|
||||||
@@ -33,6 +47,8 @@ const ListingsOverview = () => {
|
|||||||
const sp = useSearchParams();
|
const sp = useSearchParams();
|
||||||
|
|
||||||
const viewMode = userSettings?.listings_view_mode ?? 'grid';
|
const viewMode = userSettings?.listings_view_mode ?? 'grid';
|
||||||
|
const listingDeletionPref = userSettings?.listing_deletion_preference;
|
||||||
|
const defaultDeleteType = listingDeletionPref?.hardDelete ? 'hard' : 'soft';
|
||||||
|
|
||||||
const [page, setPage] = useSearchParamState(sp, 'page', 1, parseNumber);
|
const [page, setPage] = useSearchParamState(sp, 'page', 1, parseNumber);
|
||||||
const pageSize = 40;
|
const pageSize = 40;
|
||||||
@@ -44,9 +60,16 @@ const ListingsOverview = () => {
|
|||||||
const [jobNameFilter, setJobNameFilter] = useSearchParamState(sp, 'job', null, parseString);
|
const [jobNameFilter, setJobNameFilter] = useSearchParamState(sp, 'job', null, parseString);
|
||||||
const [activityFilter, setActivityFilter] = useSearchParamState(sp, 'active', null, parseNullableBoolean);
|
const [activityFilter, setActivityFilter] = useSearchParamState(sp, 'active', null, parseNullableBoolean);
|
||||||
const [providerFilter, setProviderFilter] = useSearchParamState(sp, 'provider', null, parseString);
|
const [providerFilter, setProviderFilter] = useSearchParamState(sp, 'provider', null, parseString);
|
||||||
|
const [statusFilter, setStatusFilter] = useSearchParamState(sp, 'status', null, parseString);
|
||||||
|
const [hiddenOnly, setHiddenOnly] = useSearchParamState(sp, 'hidden', false, parseNullableBoolean);
|
||||||
const [deleteModalVisible, setDeleteModalVisible] = useState(false);
|
const [deleteModalVisible, setDeleteModalVisible] = useState(false);
|
||||||
const [listingToDelete, setListingToDelete] = useState(null);
|
const [listingToDelete, setListingToDelete] = useState(null);
|
||||||
|
|
||||||
|
const isHiddenView = hiddenOnly === true;
|
||||||
|
|
||||||
|
// In watchlist mode the watch filter is forced to "watched only" — regardless of the URL.
|
||||||
|
const effectiveWatchListFilter = isWatchlistMode ? true : watchListFilter;
|
||||||
|
|
||||||
const loadData = () => {
|
const loadData = () => {
|
||||||
actions.listingsData.getListingsData({
|
actions.listingsData.getListingsData({
|
||||||
page,
|
page,
|
||||||
@@ -54,13 +77,32 @@ const ListingsOverview = () => {
|
|||||||
sortfield: sortField,
|
sortfield: sortField,
|
||||||
sortdir: sortDir,
|
sortdir: sortDir,
|
||||||
freeTextFilter,
|
freeTextFilter,
|
||||||
filter: { watchListFilter, jobNameFilter, activityFilter, providerFilter },
|
filter: {
|
||||||
|
watchListFilter: effectiveWatchListFilter,
|
||||||
|
jobNameFilter,
|
||||||
|
activityFilter: isHiddenView ? null : activityFilter,
|
||||||
|
providerFilter,
|
||||||
|
statusFilter,
|
||||||
|
hiddenOnly: isHiddenView ? true : undefined,
|
||||||
|
},
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
loadData();
|
loadData();
|
||||||
}, [page, sortField, sortDir, freeTextFilter, providerFilter, activityFilter, jobNameFilter, watchListFilter]);
|
}, [
|
||||||
|
page,
|
||||||
|
sortField,
|
||||||
|
sortDir,
|
||||||
|
freeTextFilter,
|
||||||
|
providerFilter,
|
||||||
|
activityFilter,
|
||||||
|
jobNameFilter,
|
||||||
|
watchListFilter,
|
||||||
|
statusFilter,
|
||||||
|
hiddenOnly,
|
||||||
|
isWatchlistMode,
|
||||||
|
]);
|
||||||
|
|
||||||
const handleFilterChange = useMemo(
|
const handleFilterChange = useMemo(
|
||||||
() =>
|
() =>
|
||||||
@@ -82,28 +124,62 @@ const ListingsOverview = () => {
|
|||||||
e.stopPropagation();
|
e.stopPropagation();
|
||||||
try {
|
try {
|
||||||
await xhrPost('/api/listings/watch', { listingId: item.id });
|
await xhrPost('/api/listings/watch', { listingId: item.id });
|
||||||
Toast.success(item.isWatched === 1 ? 'Listing removed from Watchlist' : 'Listing added to Watchlist');
|
Toast.success(
|
||||||
|
item.isWatched === 1 ? t('listings.toastRemovedFromWatchlist') : t('listings.toastAddedToWatchlist'),
|
||||||
|
);
|
||||||
loadData();
|
loadData();
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
console.error(e);
|
console.error(e);
|
||||||
Toast.error('Failed to operate Watchlist');
|
Toast.error(t('listings.toastWatchlistError'));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleStatusChange = async (item, nextStatus) => {
|
||||||
|
try {
|
||||||
|
await actions.listingsData.setListingStatus(item.id, nextStatus);
|
||||||
|
Toast.success(nextStatus ? `Marked as ${nextStatus}` : t('listings.toastStatusCleared'));
|
||||||
|
loadData();
|
||||||
|
} catch (e) {
|
||||||
|
console.error(e);
|
||||||
|
Toast.error(t('listings.toastStatusUpdateError'));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const handleDelete = (id) => {
|
const handleDelete = (id) => {
|
||||||
|
if (listingDeletionPref?.skipPrompt) {
|
||||||
|
confirmDeletion(listingDeletionPref.hardDelete, false, id);
|
||||||
|
return;
|
||||||
|
}
|
||||||
setListingToDelete(id);
|
setListingToDelete(id);
|
||||||
setDeleteModalVisible(true);
|
setDeleteModalVisible(true);
|
||||||
};
|
};
|
||||||
|
|
||||||
const handleNavigate = (id) => navigate(`/listings/listing/${id}`);
|
const handleRestore = async (id) => {
|
||||||
|
|
||||||
const confirmDeletion = async (hardDelete) => {
|
|
||||||
try {
|
try {
|
||||||
await xhrDelete('/api/listings/', { ids: [listingToDelete], hardDelete });
|
await actions.listingsData.restoreListings([id]);
|
||||||
Toast.success('Listing successfully removed');
|
Toast.success(t('listings.toastRestored'));
|
||||||
|
loadData();
|
||||||
|
} catch (e) {
|
||||||
|
console.error(e);
|
||||||
|
Toast.error(t('listings.toastRestoreError'));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleNavigate = (id) => {
|
||||||
|
if (isHiddenView) return;
|
||||||
|
navigate(`/listings/listing/${id}`);
|
||||||
|
};
|
||||||
|
|
||||||
|
const confirmDeletion = async (hardDelete, remember, id = listingToDelete) => {
|
||||||
|
try {
|
||||||
|
if (remember) {
|
||||||
|
await actions.userSettings.setListingDeletionPreference({ skipPrompt: true, hardDelete });
|
||||||
|
}
|
||||||
|
await xhrDelete('/api/listings/', { ids: [id], hardDelete });
|
||||||
|
Toast.success(t('listings.toastDeleted'));
|
||||||
loadData();
|
loadData();
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
Toast.error(error.message || 'Error deleting listing');
|
Toast.error(error.message || t('listings.toastDeleteError'));
|
||||||
} finally {
|
} finally {
|
||||||
setDeleteModalVisible(false);
|
setDeleteModalVisible(false);
|
||||||
setListingToDelete(null);
|
setListingToDelete(null);
|
||||||
@@ -112,129 +188,222 @@ const ListingsOverview = () => {
|
|||||||
|
|
||||||
const listings = listingsData?.result || [];
|
const listings = listingsData?.result || [];
|
||||||
|
|
||||||
|
const activityRadioValue = isHiddenView ? 'hidden' : activityFilter === null ? 'all' : String(activityFilter);
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="listingsOverview">
|
<div className="listingsOverview">
|
||||||
<div className="listingsOverview__topbar">
|
<div className="listingsOverview__topbar">
|
||||||
<Input
|
<Tooltip content={t('listings.filterSearchHelp')} trigger="hover" position="top">
|
||||||
className="listingsOverview__topbar__search"
|
<span className="listingsOverview__topbar__tooltipWrap listingsOverview__topbar__search">
|
||||||
prefix={<IconSearch />}
|
<Input
|
||||||
showClear
|
prefix={<IconSearch />}
|
||||||
placeholder="Search"
|
showClear
|
||||||
defaultValue={freeTextFilter ?? ''}
|
placeholder={t('listings.searchPlaceholder')}
|
||||||
onChange={handleFilterChange}
|
defaultValue={freeTextFilter ?? ''}
|
||||||
/>
|
onChange={handleFilterChange}
|
||||||
|
/>
|
||||||
|
</span>
|
||||||
|
</Tooltip>
|
||||||
|
|
||||||
<RadioGroup
|
<Tooltip content={t('listings.filterActivityHelp')} trigger="hover" position="top">
|
||||||
type="button"
|
<span className="listingsOverview__topbar__tooltipWrap">
|
||||||
buttonSize="middle"
|
<RadioGroup
|
||||||
value={activityFilter === null ? 'all' : String(activityFilter)}
|
type="button"
|
||||||
onChange={(e) => {
|
buttonSize="middle"
|
||||||
const v = e.target.value;
|
value={activityRadioValue}
|
||||||
setActivityFilter(v === 'all' ? null : v === 'true');
|
onChange={(e) => {
|
||||||
setPage(1);
|
const v = e.target.value;
|
||||||
}}
|
if (v === 'hidden') {
|
||||||
|
setHiddenOnly(true);
|
||||||
|
setActivityFilter(null);
|
||||||
|
} else {
|
||||||
|
setHiddenOnly(false);
|
||||||
|
setActivityFilter(v === 'all' ? null : v === 'true');
|
||||||
|
}
|
||||||
|
setPage(1);
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<Radio value="all">{t('listings.filterAll')}</Radio>
|
||||||
|
<Radio value="true">{t('listings.filterActive')}</Radio>
|
||||||
|
<Radio value="false">{t('listings.filterInactive')}</Radio>
|
||||||
|
<Radio value="hidden">{t('listings.filterHidden')}</Radio>
|
||||||
|
</RadioGroup>
|
||||||
|
</span>
|
||||||
|
</Tooltip>
|
||||||
|
|
||||||
|
{!isWatchlistMode && (
|
||||||
|
<Tooltip content={t('listings.filterWatchHelp')} trigger="hover" position="top">
|
||||||
|
<span className="listingsOverview__topbar__tooltipWrap">
|
||||||
|
<RadioGroup
|
||||||
|
type="button"
|
||||||
|
buttonSize="middle"
|
||||||
|
value={watchListFilter === null ? 'all' : String(watchListFilter)}
|
||||||
|
onChange={(e) => {
|
||||||
|
const v = e.target.value;
|
||||||
|
setWatchListFilter(v === 'all' ? null : v === 'true');
|
||||||
|
setPage(1);
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<Radio value="all">{t('listings.filterAll')}</Radio>
|
||||||
|
<Radio value="true">{t('listings.filterWatched')}</Radio>
|
||||||
|
<Radio value="false">{t('listings.filterUnwatched')}</Radio>
|
||||||
|
</RadioGroup>
|
||||||
|
</span>
|
||||||
|
</Tooltip>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<Tooltip content={t('listings.filterStatusHelp')} trigger="hover" position="top">
|
||||||
|
<span className="listingsOverview__topbar__tooltipWrap">
|
||||||
|
<Select
|
||||||
|
placeholder={t('listings.filterStatusPlaceholder')}
|
||||||
|
showClear
|
||||||
|
onChange={(val) => {
|
||||||
|
setStatusFilter(val ?? null);
|
||||||
|
setPage(1);
|
||||||
|
}}
|
||||||
|
value={statusFilter}
|
||||||
|
style={{ width: 150 }}
|
||||||
|
>
|
||||||
|
<Select.Option value="applied">{t('listings.filterStatusApplied')}</Select.Option>
|
||||||
|
<Select.Option value="rejected">{t('listings.filterStatusRejected')}</Select.Option>
|
||||||
|
<Select.Option value="accepted">{t('listings.filterStatusAccepted')}</Select.Option>
|
||||||
|
<Select.Option value="none">{t('listings.filterStatusNone')}</Select.Option>
|
||||||
|
</Select>
|
||||||
|
</span>
|
||||||
|
</Tooltip>
|
||||||
|
|
||||||
|
<Tooltip content={t('listings.filterProviderHelp')} trigger="hover" position="top">
|
||||||
|
<span className="listingsOverview__topbar__tooltipWrap">
|
||||||
|
<Select
|
||||||
|
placeholder={t('listings.filterProviderPlaceholder')}
|
||||||
|
showClear
|
||||||
|
onChange={(val) => {
|
||||||
|
setProviderFilter(val);
|
||||||
|
setPage(1);
|
||||||
|
}}
|
||||||
|
value={providerFilter}
|
||||||
|
style={{ width: 130 }}
|
||||||
|
>
|
||||||
|
{providers?.map((p) => (
|
||||||
|
<Select.Option key={p.id} value={p.id}>
|
||||||
|
{p.name}
|
||||||
|
</Select.Option>
|
||||||
|
))}
|
||||||
|
</Select>
|
||||||
|
</span>
|
||||||
|
</Tooltip>
|
||||||
|
|
||||||
|
<Tooltip content={t('listings.filterJobHelp')} trigger="hover" position="top">
|
||||||
|
<span className="listingsOverview__topbar__tooltipWrap">
|
||||||
|
<Select
|
||||||
|
placeholder={t('listings.filterJobPlaceholder')}
|
||||||
|
showClear
|
||||||
|
onChange={(val) => {
|
||||||
|
setJobNameFilter(val);
|
||||||
|
setPage(1);
|
||||||
|
}}
|
||||||
|
value={jobNameFilter}
|
||||||
|
style={{ width: 130 }}
|
||||||
|
>
|
||||||
|
{jobs?.map((j) => (
|
||||||
|
<Select.Option key={j.id} value={j.id}>
|
||||||
|
{j.name}
|
||||||
|
</Select.Option>
|
||||||
|
))}
|
||||||
|
</Select>
|
||||||
|
</span>
|
||||||
|
</Tooltip>
|
||||||
|
|
||||||
|
<Tooltip content={t('listings.filterSortHelp')} trigger="hover" position="top">
|
||||||
|
<span className="listingsOverview__topbar__tooltipWrap listingsOverview__topbar__sort">
|
||||||
|
<Select
|
||||||
|
prefix={t('listings.sortPrefix')}
|
||||||
|
style={{ width: 220 }}
|
||||||
|
value={sortField}
|
||||||
|
onChange={(val) => setSortField(val)}
|
||||||
|
>
|
||||||
|
<Select.Option value="job_name">{t('listings.sortByJobName')}</Select.Option>
|
||||||
|
<Select.Option value="created_at">{t('listings.sortByDate')}</Select.Option>
|
||||||
|
<Select.Option value="price">{t('listings.sortByPrice')}</Select.Option>
|
||||||
|
<Select.Option value="provider">{t('listings.sortByProvider')}</Select.Option>
|
||||||
|
</Select>
|
||||||
|
</span>
|
||||||
|
</Tooltip>
|
||||||
|
|
||||||
|
<Tooltip
|
||||||
|
content={sortDir === 'asc' ? t('listings.sortAscending') : t('listings.sortDescending')}
|
||||||
|
trigger="hover"
|
||||||
|
position="top"
|
||||||
>
|
>
|
||||||
<Radio value="all">All</Radio>
|
<span className="listingsOverview__topbar__tooltipWrap">
|
||||||
<Radio value="true">Active</Radio>
|
<Button
|
||||||
<Radio value="false">Inactive</Radio>
|
icon={sortDir === 'asc' ? <IconArrowUp /> : <IconArrowDown />}
|
||||||
</RadioGroup>
|
onClick={() => setSortDir(sortDir === 'asc' ? 'desc' : 'asc')}
|
||||||
|
aria-label={sortDir === 'asc' ? t('listings.sortAscending') : t('listings.sortDescending')}
|
||||||
<RadioGroup
|
/>
|
||||||
type="button"
|
</span>
|
||||||
buttonSize="middle"
|
</Tooltip>
|
||||||
value={watchListFilter === null ? 'all' : String(watchListFilter)}
|
|
||||||
onChange={(e) => {
|
|
||||||
const v = e.target.value;
|
|
||||||
setWatchListFilter(v === 'all' ? null : v === 'true');
|
|
||||||
setPage(1);
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
<Radio value="all">All</Radio>
|
|
||||||
<Radio value="true">Watched</Radio>
|
|
||||||
<Radio value="false">Unwatched</Radio>
|
|
||||||
</RadioGroup>
|
|
||||||
|
|
||||||
<Select
|
|
||||||
placeholder="Provider"
|
|
||||||
showClear
|
|
||||||
onChange={(val) => {
|
|
||||||
setProviderFilter(val);
|
|
||||||
setPage(1);
|
|
||||||
}}
|
|
||||||
value={providerFilter}
|
|
||||||
style={{ width: 130 }}
|
|
||||||
>
|
|
||||||
{providers?.map((p) => (
|
|
||||||
<Select.Option key={p.id} value={p.id}>
|
|
||||||
{p.name}
|
|
||||||
</Select.Option>
|
|
||||||
))}
|
|
||||||
</Select>
|
|
||||||
|
|
||||||
<Select
|
|
||||||
placeholder="Job"
|
|
||||||
showClear
|
|
||||||
onChange={(val) => {
|
|
||||||
setJobNameFilter(val);
|
|
||||||
setPage(1);
|
|
||||||
}}
|
|
||||||
value={jobNameFilter}
|
|
||||||
style={{ width: 130 }}
|
|
||||||
>
|
|
||||||
{jobs?.map((j) => (
|
|
||||||
<Select.Option key={j.id} value={j.id}>
|
|
||||||
{j.name}
|
|
||||||
</Select.Option>
|
|
||||||
))}
|
|
||||||
</Select>
|
|
||||||
|
|
||||||
<Select prefix="Sort by" style={{ width: 185 }} value={sortField} onChange={(val) => setSortField(val)}>
|
|
||||||
<Select.Option value="job_name">Job Name</Select.Option>
|
|
||||||
<Select.Option value="created_at">Listing Date</Select.Option>
|
|
||||||
<Select.Option value="price">Price</Select.Option>
|
|
||||||
<Select.Option value="provider">Provider</Select.Option>
|
|
||||||
</Select>
|
|
||||||
|
|
||||||
<Button
|
|
||||||
icon={sortDir === 'asc' ? <IconArrowUp /> : <IconArrowDown />}
|
|
||||||
onClick={() => setSortDir(sortDir === 'asc' ? 'desc' : 'asc')}
|
|
||||||
title={sortDir === 'asc' ? 'Ascending' : 'Descending'}
|
|
||||||
/>
|
|
||||||
|
|
||||||
<div className="listingsOverview__topbar__view-toggle">
|
<div className="listingsOverview__topbar__view-toggle">
|
||||||
<Tooltip content="Grid view">
|
<Tooltip content={t('listings.tooltipGridView')}>
|
||||||
<Button
|
<Button
|
||||||
icon={<IconGridView />}
|
icon={<IconGridView />}
|
||||||
theme={viewMode === 'grid' ? 'solid' : 'borderless'}
|
theme={viewMode === 'grid' ? 'solid' : 'borderless'}
|
||||||
onClick={() => actions.userSettings.setListingsViewMode('grid')}
|
onClick={() => actions.userSettings.setListingsViewMode('grid')}
|
||||||
aria-label="Grid view"
|
aria-label={t('common.ariaGridView')}
|
||||||
aria-pressed={viewMode === 'grid'}
|
aria-pressed={viewMode === 'grid'}
|
||||||
/>
|
/>
|
||||||
</Tooltip>
|
</Tooltip>
|
||||||
<Tooltip content="Table view">
|
<Tooltip content={t('listings.tooltipTableView')}>
|
||||||
<Button
|
<Button
|
||||||
icon={<IconList />}
|
icon={<IconList />}
|
||||||
theme={viewMode === 'table' ? 'solid' : 'borderless'}
|
theme={viewMode === 'table' ? 'solid' : 'borderless'}
|
||||||
onClick={() => actions.userSettings.setListingsViewMode('table')}
|
onClick={() => actions.userSettings.setListingsViewMode('table')}
|
||||||
aria-label="Table view"
|
aria-label={t('common.ariaTableView')}
|
||||||
aria-pressed={viewMode === 'table'}
|
aria-pressed={viewMode === 'table'}
|
||||||
/>
|
/>
|
||||||
</Tooltip>
|
</Tooltip>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{isHiddenView && (
|
||||||
|
<Banner
|
||||||
|
type="info"
|
||||||
|
fullMode={false}
|
||||||
|
closeIcon={null}
|
||||||
|
description={t('listings.hiddenViewBanner')}
|
||||||
|
style={{ marginBottom: 12 }}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
|
||||||
{listings.length === 0 && (
|
{listings.length === 0 && (
|
||||||
<Empty
|
<Empty
|
||||||
image={<IllustrationNoResult />}
|
image={<IllustrationNoResult />}
|
||||||
darkModeImage={<IllustrationNoResultDark />}
|
darkModeImage={<IllustrationNoResultDark />}
|
||||||
description="No listings available yet..."
|
description={t('listings.empty')}
|
||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{viewMode === 'grid' ? (
|
{viewMode === 'grid' ? (
|
||||||
<ListingsGrid listings={listings} onWatch={handleWatch} onNavigate={handleNavigate} onDelete={handleDelete} />
|
<ListingsGrid
|
||||||
|
listings={listings}
|
||||||
|
onWatch={handleWatch}
|
||||||
|
onNavigate={handleNavigate}
|
||||||
|
onDelete={handleDelete}
|
||||||
|
onRestore={handleRestore}
|
||||||
|
isHiddenView={isHiddenView}
|
||||||
|
onStatusChange={handleStatusChange}
|
||||||
|
/>
|
||||||
) : (
|
) : (
|
||||||
<ListingsTable listings={listings} onWatch={handleWatch} onNavigate={handleNavigate} onDelete={handleDelete} />
|
<ListingsTable
|
||||||
|
listings={listings}
|
||||||
|
onWatch={handleWatch}
|
||||||
|
onNavigate={handleNavigate}
|
||||||
|
onDelete={handleDelete}
|
||||||
|
onRestore={handleRestore}
|
||||||
|
isHiddenView={isHiddenView}
|
||||||
|
onStatusChange={handleStatusChange}
|
||||||
|
/>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{listings.length > 0 && (
|
{listings.length > 0 && (
|
||||||
@@ -251,6 +420,7 @@ const ListingsOverview = () => {
|
|||||||
|
|
||||||
<ListingDeletionModal
|
<ListingDeletionModal
|
||||||
visible={deleteModalVisible}
|
visible={deleteModalVisible}
|
||||||
|
defaultDeleteType={defaultDeleteType}
|
||||||
onConfirm={confirmDeletion}
|
onConfirm={confirmDeletion}
|
||||||
onCancel={() => {
|
onCancel={() => {
|
||||||
setDeleteModalVisible(false);
|
setDeleteModalVisible(false);
|
||||||
|
|||||||
@@ -8,6 +8,15 @@
|
|||||||
margin-bottom: @space-4;
|
margin-bottom: @space-4;
|
||||||
flex-wrap: wrap;
|
flex-wrap: wrap;
|
||||||
|
|
||||||
|
&__tooltipWrap {
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
|
||||||
|
> * {
|
||||||
|
width: 100%;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
&__search {
|
&__search {
|
||||||
min-width: 200px;
|
min-width: 200px;
|
||||||
flex: 1;
|
flex: 1;
|
||||||
@@ -19,6 +28,14 @@
|
|||||||
flex-shrink: 0;
|
flex-shrink: 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
&__sort {
|
||||||
|
flex-shrink: 0;
|
||||||
|
|
||||||
|
.semi-select-prefix {
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@media (max-width: 768px) {
|
@media (max-width: 768px) {
|
||||||
.listingsOverview__topbar__search {
|
.listingsOverview__topbar__search {
|
||||||
width: 100%;
|
width: 100%;
|
||||||
|
|||||||
113
ui/src/components/listings/StatusControl.jsx
Normal file
113
ui/src/components/listings/StatusControl.jsx
Normal file
@@ -0,0 +1,113 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (c) 2026 by Christian Kellner.
|
||||||
|
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { useState } from 'react';
|
||||||
|
import { Dropdown, Button, Tooltip } from '@douyinfe/semi-ui-19';
|
||||||
|
import { IconChevronDown } from '@douyinfe/semi-icons';
|
||||||
|
|
||||||
|
import './StatusControl.less';
|
||||||
|
import { useTranslation } from '../../services/i18n/i18n.jsx';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @typedef {('applied'|'rejected'|'accepted'|null)} ListingStatus
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Shared control for setting a listing's user-decision status
|
||||||
|
* (Applied / Rejected / Accepted).
|
||||||
|
*
|
||||||
|
* Both compact (table/grid rows) and full (listing detail header) modes
|
||||||
|
* render a Button that picks up the project's CI tokens via the
|
||||||
|
* .status-btn classes, with a small size variant for compact contexts.
|
||||||
|
*
|
||||||
|
* @param {Object} props
|
||||||
|
* @param {ListingStatus} props.status - The current status value.
|
||||||
|
* @param {(next: ListingStatus) => void} props.onChange - Called with the new status when the user picks one.
|
||||||
|
* @param {boolean} [props.compact=false] - When true, renders smaller for table/grid rows; full size otherwise.
|
||||||
|
* @param {(e: React.MouseEvent) => void} [props.onTriggerClick] - Optional click handler to stop propagation on the trigger.
|
||||||
|
*/
|
||||||
|
export default function StatusControl({ status = null, onChange, compact = false, onTriggerClick }) {
|
||||||
|
const t = useTranslation();
|
||||||
|
const [open, setOpen] = useState(false);
|
||||||
|
const [tooltipOpen, setTooltipOpen] = useState(false);
|
||||||
|
|
||||||
|
const STATUS_OPTIONS = [
|
||||||
|
{ value: null, label: t('listings.status.none') },
|
||||||
|
{ value: 'applied', label: t('listings.status.applied') },
|
||||||
|
{ value: 'rejected', label: t('listings.status.rejected') },
|
||||||
|
{ value: 'accepted', label: t('listings.status.accepted') },
|
||||||
|
];
|
||||||
|
|
||||||
|
const STATUS_TOOLTIP = t('listings.status.tooltip');
|
||||||
|
|
||||||
|
const optionFor = (status) => STATUS_OPTIONS.find((o) => o.value === status) ?? STATUS_OPTIONS[0];
|
||||||
|
|
||||||
|
const current = optionFor(status);
|
||||||
|
|
||||||
|
const handlePick = (next) => {
|
||||||
|
setOpen(false);
|
||||||
|
if (next === status) return;
|
||||||
|
onChange?.(next);
|
||||||
|
};
|
||||||
|
|
||||||
|
const menu = (
|
||||||
|
<Dropdown.Menu>
|
||||||
|
{STATUS_OPTIONS.map((opt) => (
|
||||||
|
<Dropdown.Item
|
||||||
|
key={opt.value ?? '__none__'}
|
||||||
|
active={opt.value === status}
|
||||||
|
onClick={() => handlePick(opt.value)}
|
||||||
|
>
|
||||||
|
{opt.label}
|
||||||
|
</Dropdown.Item>
|
||||||
|
))}
|
||||||
|
</Dropdown.Menu>
|
||||||
|
);
|
||||||
|
|
||||||
|
const className = ['status-btn', compact ? 'status-btn--compact' : null, status ? `status-btn--${status}` : null]
|
||||||
|
.filter(Boolean)
|
||||||
|
.join(' ');
|
||||||
|
|
||||||
|
const trigger = (
|
||||||
|
<Tooltip
|
||||||
|
content={STATUS_TOOLTIP}
|
||||||
|
position="top"
|
||||||
|
trigger="custom"
|
||||||
|
visible={tooltipOpen && !open}
|
||||||
|
onVisibleChange={setTooltipOpen}
|
||||||
|
>
|
||||||
|
<Button
|
||||||
|
size={compact ? 'small' : 'default'}
|
||||||
|
theme="borderless"
|
||||||
|
icon={<IconChevronDown />}
|
||||||
|
iconPosition="right"
|
||||||
|
onMouseEnter={() => setTooltipOpen(true)}
|
||||||
|
onMouseLeave={() => setTooltipOpen(false)}
|
||||||
|
onClick={(e) => {
|
||||||
|
onTriggerClick?.(e);
|
||||||
|
setTooltipOpen(false);
|
||||||
|
setOpen((o) => !o);
|
||||||
|
}}
|
||||||
|
className={className}
|
||||||
|
>
|
||||||
|
{status ? current.label : t('listings.status.statusLabel')}
|
||||||
|
</Button>
|
||||||
|
</Tooltip>
|
||||||
|
);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Dropdown
|
||||||
|
trigger="custom"
|
||||||
|
visible={open}
|
||||||
|
onVisibleChange={setOpen}
|
||||||
|
onClickOutSide={() => setOpen(false)}
|
||||||
|
position="bottom"
|
||||||
|
render={menu}
|
||||||
|
stopPropagation
|
||||||
|
>
|
||||||
|
<span className="status-btn__anchor">{trigger}</span>
|
||||||
|
</Dropdown>
|
||||||
|
);
|
||||||
|
}
|
||||||
64
ui/src/components/listings/StatusControl.less
Normal file
64
ui/src/components/listings/StatusControl.less
Normal file
@@ -0,0 +1,64 @@
|
|||||||
|
@import '../../tokens.less';
|
||||||
|
|
||||||
|
// Wrapper span used as the Dropdown's positioning anchor so the menu opens
|
||||||
|
// directly below the visible button rather than the implicit wrapper of the
|
||||||
|
// hover tooltip (which can have a different bounding box).
|
||||||
|
.status-btn__anchor {
|
||||||
|
display: inline-block;
|
||||||
|
line-height: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// StatusControl shared base. Matches dimensions and border treatment
|
||||||
|
// of the surrounding Watched / Open listing / Delete buttons in the
|
||||||
|
// detail view, and shrinks via the --compact modifier for table rows
|
||||||
|
// and grid cards.
|
||||||
|
.status-btn {
|
||||||
|
color: @color-muted !important;
|
||||||
|
border: 1px solid @color-border-bright !important;
|
||||||
|
border-radius: @radius-btn !important;
|
||||||
|
background: transparent !important;
|
||||||
|
transition: color @transition-fast, border-color @transition-fast, background @transition-fast;
|
||||||
|
|
||||||
|
&:hover {
|
||||||
|
color: @color-text !important;
|
||||||
|
background: rgba(255, 255, 255, 0.06) !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
&--compact {
|
||||||
|
height: 24px !important;
|
||||||
|
padding: 0 8px !important;
|
||||||
|
font-size: @text-sm !important;
|
||||||
|
border-radius: @radius-chip !important;
|
||||||
|
|
||||||
|
.semi-icon {
|
||||||
|
font-size: 12px !important;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
&--applied {
|
||||||
|
color: @color-info !important;
|
||||||
|
border-color: rgba(96, 165, 250, 0.4) !important;
|
||||||
|
background: rgba(96, 165, 250, 0.08) !important;
|
||||||
|
&:hover {
|
||||||
|
background: rgba(96, 165, 250, 0.14) !important;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
&--rejected {
|
||||||
|
color: @color-error !important;
|
||||||
|
border-color: rgba(251, 113, 133, 0.4) !important;
|
||||||
|
background: rgba(251, 113, 133, 0.08) !important;
|
||||||
|
&:hover {
|
||||||
|
background: rgba(251, 113, 133, 0.14) !important;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
&--accepted {
|
||||||
|
color: @color-success !important;
|
||||||
|
border-color: rgba(52, 211, 153, 0.4) !important;
|
||||||
|
background: rgba(52, 211, 153, 0.08) !important;
|
||||||
|
&:hover {
|
||||||
|
background: rgba(52, 211, 153, 0.14) !important;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -8,6 +8,7 @@ import maplibregl from 'maplibre-gl';
|
|||||||
import 'maplibre-gl/dist/maplibre-gl.css';
|
import 'maplibre-gl/dist/maplibre-gl.css';
|
||||||
import '@mapbox/mapbox-gl-draw/dist/mapbox-gl-draw.css';
|
import '@mapbox/mapbox-gl-draw/dist/mapbox-gl-draw.css';
|
||||||
import { fixMapboxDrawCompatibility, addDrawingControl, setupAreaFilterEventListeners } from './MapDrawingExtension.js';
|
import { fixMapboxDrawCompatibility, addDrawingControl, setupAreaFilterEventListeners } from './MapDrawingExtension.js';
|
||||||
|
import { getBoundsFromCoords } from '../../views/listings/mapUtils.js';
|
||||||
import './Map.less';
|
import './Map.less';
|
||||||
|
|
||||||
export const GERMANY_BOUNDS = [
|
export const GERMANY_BOUNDS = [
|
||||||
@@ -66,6 +67,7 @@ export default function Map({
|
|||||||
const mapContainerRef = useRef(null);
|
const mapContainerRef = useRef(null);
|
||||||
const mapRef = useRef(null);
|
const mapRef = useRef(null);
|
||||||
const drawRef = useRef(null);
|
const drawRef = useRef(null);
|
||||||
|
const hasFittedToInitialAreaRef = useRef(false);
|
||||||
|
|
||||||
// Initialize map - ONLY when container changes, never reinitialize
|
// Initialize map - ONLY when container changes, never reinitialize
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
@@ -128,6 +130,17 @@ export default function Map({
|
|||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Error loading spatial filter:', error);
|
console.error('Error loading spatial filter:', error);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!hasFittedToInitialAreaRef.current) {
|
||||||
|
const coords = initialSpatialFilter.features.flatMap((feature) =>
|
||||||
|
feature.geometry?.type === 'Polygon' ? feature.geometry.coordinates.flat() : [],
|
||||||
|
);
|
||||||
|
const bounds = getBoundsFromCoords(coords);
|
||||||
|
if (bounds) {
|
||||||
|
mapRef.current.fitBounds(bounds, { padding: 50, maxZoom: 15, duration: 0 });
|
||||||
|
hasFittedToInitialAreaRef.current = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Setup drawing event listeners
|
// Setup drawing event listeners
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user