mirror of
https://github.com/orangecoding/fredy.git
synced 2026-06-16 12:31:07 +00:00
Compare commits
28 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2465514b7a | ||
|
|
9dde377fe6 | ||
|
|
28a3a7f372 | ||
|
|
e859250545 | ||
|
|
4dd0370ec1 | ||
|
|
51b4e51f3f | ||
|
|
fa1899765c | ||
|
|
d43c5b3f97 | ||
|
|
7fd8be07a2 | ||
|
|
2926ee7e08 | ||
|
|
9506d1a9db | ||
|
|
feaa06c132 | ||
|
|
ad46500d4e | ||
|
|
3c209a8f97 | ||
|
|
398259ff20 | ||
|
|
cf030bfa39 | ||
|
|
5dc976c7e3 | ||
|
|
05f1bc61c9 | ||
|
|
6e8a35a836 | ||
|
|
87771655a8 | ||
|
|
87b5673bf0 | ||
|
|
9291155cc2 | ||
|
|
ac90d4122b | ||
|
|
790c559316 | ||
|
|
2a815c92e6 | ||
|
|
cef9b5c8fc | ||
|
|
1e2476a375 | ||
|
|
78b762bd9e |
@@ -1,7 +1,47 @@
|
||||
# Dependencies (will be installed fresh in container)
|
||||
node_modules/
|
||||
npm-debug.log
|
||||
test/
|
||||
|
||||
# Database and config (mounted as volumes)
|
||||
db/
|
||||
conf/
|
||||
|
||||
# Git
|
||||
.git/
|
||||
.github/
|
||||
.gitignore
|
||||
|
||||
# IDE and editor
|
||||
.idea/
|
||||
.vscode/
|
||||
*.swp
|
||||
*.swo
|
||||
.DS_Store
|
||||
|
||||
# Testing
|
||||
test/
|
||||
|
||||
# Documentation
|
||||
doc/
|
||||
*.md
|
||||
!README.md
|
||||
|
||||
# Development config files
|
||||
.babelrc
|
||||
.husky/
|
||||
.nvmrc
|
||||
.prettierrc
|
||||
.prettierignore
|
||||
eslint.config.js
|
||||
|
||||
# Docker files (not needed inside container)
|
||||
Dockerfile
|
||||
docker-compose.yml
|
||||
docker-test.sh
|
||||
.dockerignore
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
npm-debug.log
|
||||
|
||||
# Build artifacts (built fresh in container)
|
||||
dist/
|
||||
|
||||
69
Dockerfile
69
Dockerfile
@@ -1,27 +1,59 @@
|
||||
FROM node:22-slim
|
||||
# ================================
|
||||
# Stage 1: Build stage
|
||||
# ================================
|
||||
FROM node:22-alpine AS builder
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
# Install build dependencies needed for native modules (better-sqlite3)
|
||||
RUN apk add --no-cache python3 make g++
|
||||
|
||||
# Copy package files first for better layer caching
|
||||
COPY package.json yarn.lock ./
|
||||
|
||||
# Install all dependencies (including devDependencies for building)
|
||||
RUN yarn config set network-timeout 600000 \
|
||||
&& yarn --frozen-lockfile
|
||||
|
||||
# Copy source files needed for build
|
||||
COPY index.html vite.config.js ./
|
||||
COPY ui ./ui
|
||||
COPY lib ./lib
|
||||
|
||||
# Build frontend assets
|
||||
RUN yarn build:frontend
|
||||
|
||||
# ================================
|
||||
# Stage 2: Production stage
|
||||
# ================================
|
||||
FROM node:22-alpine
|
||||
|
||||
WORKDIR /fredy
|
||||
|
||||
# Install Chromium and curl without extra recommended packages and clean apt cache
|
||||
# curl is needed for the health check
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends chromium curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
# Install Chromium and curl (for healthcheck)
|
||||
# Using Alpine's chromium package which is much smaller
|
||||
RUN apk add --no-cache chromium curl
|
||||
|
||||
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
|
||||
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
|
||||
ENV NODE_ENV=production \
|
||||
PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
|
||||
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
|
||||
|
||||
# Copy lockfiles first to leverage cache for dependencies
|
||||
COPY package.json yarn.lock .
|
||||
# Install build dependencies for native modules, then remove them after yarn install
|
||||
COPY package.json yarn.lock ./
|
||||
|
||||
# Set Yarn timeout, install dependencies and PM2 globally
|
||||
RUN yarn config set network-timeout 600000 \
|
||||
&& yarn --frozen-lockfile \
|
||||
&& yarn global add pm2
|
||||
RUN apk add --no-cache --virtual .build-deps python3 make g++ \
|
||||
&& yarn config set network-timeout 600000 \
|
||||
&& yarn --frozen-lockfile --production \
|
||||
&& yarn cache clean \
|
||||
&& apk del .build-deps
|
||||
|
||||
# Copy application source and build production assets
|
||||
COPY . .
|
||||
RUN yarn build:frontend
|
||||
# Copy built frontend from builder stage
|
||||
COPY --from=builder /build/ui/public ./ui/public
|
||||
|
||||
# Copy application source (only what's needed at runtime)
|
||||
COPY index.js ./
|
||||
COPY index.html ./
|
||||
COPY lib ./lib
|
||||
|
||||
# Prepare runtime directories and symlinks for data and config
|
||||
RUN mkdir -p /db /conf \
|
||||
@@ -34,5 +66,4 @@ EXPOSE 9998
|
||||
VOLUME /db
|
||||
VOLUME /conf
|
||||
|
||||
# Start application using PM2 runtime
|
||||
CMD ["pm2-runtime", "index.js"]
|
||||
CMD ["node", "index.js"]
|
||||
|
||||
227
LICENSE
227
LICENSE
@@ -1,21 +1,214 @@
|
||||
MIT License
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
Copyright (c) 2025 Christian Kellner
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
1. Definitions.
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor
|
||||
be liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
|
||||
Additional License Condition – Commons Clause
|
||||
|
||||
The Licensed Work is provided under the terms of this license and is also
|
||||
subject to the following additional condition ("Commons Clause"):
|
||||
|
||||
"License Condition v1.0":
|
||||
|
||||
The Licensed Work and its derivative works may not be used by any person or
|
||||
organization to Sell the Licensed Work (as defined below).
|
||||
|
||||
"Sell" or "Selling" means practicing any or all of the rights granted to you
|
||||
under the License to provide to third parties, for a fee or other consideration
|
||||
(including without limitation fees for hosting or consulting/support services
|
||||
related to the Software), a product or service whose value derives, entirely or
|
||||
substantially, from the functionality of the Licensed Work.
|
||||
|
||||
A non-exhaustive list of activities considered "Selling" includes:
|
||||
- Using the Licensed Work to provide paid hosted services or managed services
|
||||
- Distributing the Licensed Work as part of a commercial product or service
|
||||
for which a fee is charged primarily for the value of the Licensed Work
|
||||
|
||||
This restriction does not apply to the use of the Licensed Work for internal
|
||||
business purposes or non-commercial use.
|
||||
|
||||
|
||||
Attribution and Naming Clause
|
||||
|
||||
Any derivative work based on this software must include clear and visible
|
||||
attribution to the original project "Fredy" and its author(s).
|
||||
Derivative works may not be distributed, published, or presented under a
|
||||
different name or branding without the explicit written permission of the
|
||||
original copyright holder.
|
||||
|
||||
|
||||
Copyright (c) 2026 Christian Kellner
|
||||
Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
|
||||
@@ -107,6 +107,10 @@ yarn run start:frontend # in another terminal
|
||||
|
||||
👉 Open <http://localhost:9998>
|
||||
|
||||
### With Unraid
|
||||
|
||||
Should you use [Unraid](https://unraid.net/), you can now install Fredy from the community store :)
|
||||
|
||||
**Default Login:**
|
||||
- Username: `admin`
|
||||
- Password: `admin`
|
||||
@@ -115,7 +119,7 @@ yarn run start:frontend # in another terminal
|
||||
|
||||
## 📸 Screenshots
|
||||
|
||||
| Fredy Main Overview | Job Configuration | Found Listings |
|
||||
| Fredy Maps View | Dashboard | Found Listings |
|
||||
|--------------------------------------------------|-----------------------------------------------------------------------|-----------------------------------------------------------------------------|
|
||||
|  |  |  |
|
||||
|
||||
@@ -202,7 +206,7 @@ flowchart TD
|
||||
F2["Adapter 2"]
|
||||
end
|
||||
|
||||
A1 --> B["FredyPipeline"]
|
||||
A1 --> B["FredyPipelineExecutioner"]
|
||||
A2 --> B
|
||||
A3 --> B
|
||||
B --> C1 & C2 & C3
|
||||
|
||||
52
copyright.js
Normal file
52
copyright.js
Normal file
@@ -0,0 +1,52 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
|
||||
const COPYRIGHT = `/*
|
||||
* Copyright (c) ${new Date().getFullYear()} by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
`;
|
||||
|
||||
async function getAllFiles(dir = '.') {
|
||||
const entries = await fs.readdir(dir, { withFileTypes: true });
|
||||
let files = [];
|
||||
for (let entry of entries) {
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
if (entry.name === 'node_modules' || entry.name.startsWith('.')) continue;
|
||||
files = files.concat(await getAllFiles(fullPath));
|
||||
} else if (fullPath.endsWith('.js') || fullPath.endsWith('.jsx')) {
|
||||
files.push(fullPath);
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
/* eslint-disable no-console */
|
||||
async function addCopyright(files) {
|
||||
const oldCopyrightRegex =
|
||||
/^(\/\*\n \* Copyright \(c\) \d{4} by Christian Kellner\.\n \* Licensed under Apache-2.0 with Commons Clause and Attribution\/Naming Clause\n \*\/\n\n)+/;
|
||||
for (let file of files) {
|
||||
try {
|
||||
let content = await fs.readFile(file, 'utf8');
|
||||
const strippedContent = content.replace(oldCopyrightRegex, '');
|
||||
const newContent = COPYRIGHT + strippedContent;
|
||||
if (content !== newContent) {
|
||||
await fs.writeFile(file, newContent);
|
||||
console.log(`Added/Updated copyright in ${file}`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`Error processing ${file}: ${err}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
/* eslint-enable no-console */
|
||||
|
||||
const filesToProcess = process.argv.length > 2 ? process.argv.slice(2) : await getAllFiles();
|
||||
await addCopyright(filesToProcess);
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 197 KiB After Width: | Height: | Size: 3.7 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 512 KiB After Width: | Height: | Size: 4.7 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 372 KiB After Width: | Height: | Size: 402 KiB |
@@ -1,22 +1,26 @@
|
||||
services:
|
||||
fredy:
|
||||
container_name: fredy
|
||||
# build from empty build folder to reduce size of image
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: ghcr.io/orangecoding/fredy
|
||||
# map existing config and database
|
||||
environment:
|
||||
- NODE_ENV=production
|
||||
volumes:
|
||||
- ./conf:/conf
|
||||
- ./db:/db
|
||||
ports:
|
||||
- "9998:9998"
|
||||
restart: unless-stopped
|
||||
# Resource limits to prevent runaway memory usage from Chromium
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 1G
|
||||
healthcheck:
|
||||
# The container will immediately stop when health check fails after retries
|
||||
test: ["CMD-SHELL", "curl --fail --silent --show-error --max-time 5 http://localhost:9998/ || exit 1"]
|
||||
test: ["CMD", "curl", "--fail", "--silent", "--show-error", "--max-time", "5", "http://localhost:9998/"]
|
||||
interval: 120s
|
||||
timeout: 10s
|
||||
retries: 1
|
||||
start_period: 10s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
// eslint.config.js
|
||||
import js from '@eslint/js';
|
||||
import prettier from 'eslint-config-prettier';
|
||||
|
||||
66
index.js
66
index.js
@@ -1,19 +1,21 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { checkIfConfigIsAccessible, getProviders, refreshConfig } from './lib/utils.js';
|
||||
import * as similarityCache from './lib/services/similarity-check/similarityCache.js';
|
||||
import * as jobStorage from './lib/services/storage/jobStorage.js';
|
||||
import FredyPipeline from './lib/FredyPipeline.js';
|
||||
import { duringWorkingHoursOrNotSet } from './lib/utils.js';
|
||||
import { runMigrations } from './lib/services/storage/migrations/migrate.js';
|
||||
import { ensureDemoUserExists, ensureAdminUserExists } from './lib/services/storage/userStorage.js';
|
||||
import { cleanupDemoAtMidnight } from './lib/services/crons/demoCleanup-cron.js';
|
||||
import { initTrackerCron } from './lib/services/crons/tracker-cron.js';
|
||||
import logger from './lib/services/logger.js';
|
||||
import { bus } from './lib/services/events/event-bus.js';
|
||||
import { initActiveCheckerCron } from './lib/services/crons/listing-alive-cron.js';
|
||||
import { initGeocodingCron } from './lib/services/crons/geocoding-cron.js';
|
||||
import { getSettings } from './lib/services/storage/settingsStorage.js';
|
||||
import SqliteConnection from './lib/services/storage/SqliteConnection.js';
|
||||
import SqliteConnection, { computeDbPath } from './lib/services/storage/SqliteConnection.js';
|
||||
import { initJobExecutionService } from './lib/services/jobs/jobExecutionService.js';
|
||||
|
||||
//in the config, we store the path of the sqlite file, thus we must check if it is available
|
||||
const isConfigAccessible = await checkIfConfigIsAccessible();
|
||||
@@ -32,12 +34,10 @@ await runMigrations();
|
||||
|
||||
const settings = await getSettings();
|
||||
|
||||
// Ensure sqlite directory exists before loading anything else (based on config.sqlitepath)
|
||||
const rawDir = settings.sqlitepath || '/db';
|
||||
const relDir = rawDir.startsWith('/') ? rawDir.slice(1) : rawDir;
|
||||
const absDir = path.isAbsolute(relDir) ? relDir : path.join(process.cwd(), relDir);
|
||||
if (!fs.existsSync(absDir)) {
|
||||
fs.mkdirSync(absDir, { recursive: true });
|
||||
// Ensure the sqlite directory exists before loading anything else (based on config.sqlitepath)
|
||||
const { dir: sqliteDir } = await computeDbPath();
|
||||
if (!fs.existsSync(sqliteDir)) {
|
||||
fs.mkdirSync(sqliteDir, { recursive: true });
|
||||
}
|
||||
|
||||
// Load provider modules once at startup
|
||||
@@ -57,48 +57,14 @@ if (settings.demoMode) {
|
||||
cleanupDemoAtMidnight();
|
||||
}
|
||||
|
||||
logger.info(`Started Fredy successfully. Ui can be accessed via http://localhost:${settings.port}`);
|
||||
|
||||
ensureAdminUserExists();
|
||||
ensureDemoUserExists();
|
||||
await initTrackerCron();
|
||||
//do not wait for this to finish, let it run in the background
|
||||
initActiveCheckerCron();
|
||||
initGeocodingCron();
|
||||
|
||||
bus.on('jobs:runAll', () => {
|
||||
logger.debug('Running Fredy Job manually');
|
||||
execute();
|
||||
});
|
||||
logger.info(`Started Fredy successfully. Ui can be accessed via http://localhost:${settings.port}`);
|
||||
|
||||
const execute = () => {
|
||||
const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(settings, Date.now());
|
||||
if (!settings.demoMode) {
|
||||
if (isDuringWorkingHoursOrNotSet) {
|
||||
settings.lastRun = Date.now();
|
||||
jobStorage
|
||||
.getJobs()
|
||||
.filter((job) => job.enabled)
|
||||
.forEach((job) => {
|
||||
job.provider
|
||||
.filter((p) => providers.find((loaded) => loaded.metaInformation.id === p.id) != null)
|
||||
.forEach(async (prov) => {
|
||||
const matchedProvider = providers.find((loaded) => loaded.metaInformation.id === prov.id);
|
||||
matchedProvider.init(prov, job.blacklist);
|
||||
await new FredyPipeline(
|
||||
matchedProvider.config,
|
||||
job.notificationAdapter,
|
||||
prov.id,
|
||||
job.id,
|
||||
similarityCache,
|
||||
).execute();
|
||||
});
|
||||
});
|
||||
} else {
|
||||
logger.debug('Working hours set. Skipping as outside of working hours.');
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
setInterval(execute, INTERVAL);
|
||||
//start once at startup
|
||||
execute();
|
||||
// Initialize the lean Job Execution Service (schedules and bus listeners)
|
||||
initJobExecutionService({ providers, settings, intervalMs: INTERVAL });
|
||||
|
||||
@@ -1,9 +1,19 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { NoNewListingsWarning } from './errors.js';
|
||||
import { storeListings, getKnownListingHashesForJobAndProvider } from './services/storage/listingsStorage.js';
|
||||
import { getJob } from './services/storage/jobStorage.js';
|
||||
import * as notify from './notification/notify.js';
|
||||
import Extractor from './services/extractor/extractor.js';
|
||||
import urlModifier from './services/queryStringMutator.js';
|
||||
import logger from './services/logger.js';
|
||||
import { geocodeAddress } from './services/geocoding/geoCodingService.js';
|
||||
import { distanceMeters } from './services/listings/distanceCalculator.js';
|
||||
import { getUserSettings } from './services/storage/settingsStorage.js';
|
||||
import { updateListingDistance } from './services/storage/listingsStorage.js';
|
||||
|
||||
/**
|
||||
* @typedef {Object} Listing
|
||||
@@ -35,7 +45,7 @@ import logger from './services/logger.js';
|
||||
* 7) Filter out entries similar to already seen ones
|
||||
* 8) Dispatch notifications
|
||||
*/
|
||||
class FredyPipeline {
|
||||
class FredyPipelineExecutioner {
|
||||
/**
|
||||
* Create a new runtime instance for a single provider/job execution.
|
||||
*
|
||||
@@ -74,12 +84,33 @@ class FredyPipeline {
|
||||
.then(this._normalize.bind(this))
|
||||
.then(this._filter.bind(this))
|
||||
.then(this._findNew.bind(this))
|
||||
.then(this._geocode.bind(this))
|
||||
.then(this._save.bind(this))
|
||||
.then(this._calculateDistance.bind(this))
|
||||
.then(this._filterBySimilarListings.bind(this))
|
||||
.then(this._notify.bind(this))
|
||||
.catch(this._handleError.bind(this));
|
||||
}
|
||||
|
||||
/**
|
||||
* Geocode new listings.
|
||||
*
|
||||
* @param {Listing[]} newListings New listings to geocode.
|
||||
* @returns {Promise<Listing[]>} Resolves with the listings (potentially with added coordinates).
|
||||
*/
|
||||
async _geocode(newListings) {
|
||||
for (const listing of newListings) {
|
||||
if (listing.address) {
|
||||
const coords = await geocodeAddress(listing.address);
|
||||
if (coords) {
|
||||
listing.latitude = coords.lat;
|
||||
listing.longitude = coords.lng;
|
||||
}
|
||||
}
|
||||
}
|
||||
return newListings;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch listings from the provider, using the default Extractor flow unless
|
||||
* a provider-specific getListings override is supplied.
|
||||
@@ -175,6 +206,42 @@ class FredyPipeline {
|
||||
return newListings;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate distance for new listings.
|
||||
*
|
||||
* @param {Listing[]} listings
|
||||
* @returns {Listing[]}
|
||||
* @private
|
||||
*/
|
||||
_calculateDistance(listings) {
|
||||
if (listings.length === 0) return [];
|
||||
|
||||
const job = getJob(this._jobKey);
|
||||
const userId = job?.userId;
|
||||
|
||||
if (userId == null || typeof userId !== 'string') {
|
||||
logger.debug('Skipping distance calculation: userId is missing or invalid');
|
||||
return listings;
|
||||
}
|
||||
|
||||
const userSettings = getUserSettings(userId);
|
||||
const homeAddress = userSettings?.home_address;
|
||||
|
||||
if (!homeAddress || !homeAddress.coords) {
|
||||
return listings;
|
||||
}
|
||||
|
||||
const { lat, lng } = homeAddress.coords;
|
||||
for (const listing of listings) {
|
||||
if (listing.latitude != null && listing.longitude != null) {
|
||||
const dist = distanceMeters(lat, lng, listing.latitude, listing.longitude);
|
||||
updateListingDistance(listing.id, dist);
|
||||
listing.distance_to_destination = dist;
|
||||
}
|
||||
}
|
||||
return listings;
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove listings that are similar to already known entries according to the similarity cache.
|
||||
* Adds the remaining listings to the cache.
|
||||
@@ -213,4 +280,4 @@ class FredyPipeline {
|
||||
}
|
||||
}
|
||||
|
||||
export default FredyPipeline;
|
||||
export default FredyPipelineExecutioner;
|
||||
@@ -1,11 +1,16 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { notificationAdapterRouter } from './routes/notificationAdapterRouter.js';
|
||||
import { authInterceptor, cookieSession, adminInterceptor } from './security.js';
|
||||
import { generalSettingsRouter } from './routes/generalSettingsRoute.js';
|
||||
import { analyticsRouter } from './routes/analyticsRouter.js';
|
||||
import { providerRouter } from './routes/providerRouter.js';
|
||||
import { versionRouter } from './routes/versionRouter.js';
|
||||
import { loginRouter } from './routes/loginRoute.js';
|
||||
import { userRouter } from './routes/userRoute.js';
|
||||
import { userSettingsRouter } from './routes/userSettingsRoute.js';
|
||||
import { jobRouter } from './routes/jobRouter.js';
|
||||
import bodyParser from 'body-parser';
|
||||
import restana from 'restana';
|
||||
@@ -16,7 +21,8 @@ import { demoRouter } from './routes/demoRouter.js';
|
||||
import logger from '../services/logger.js';
|
||||
import { listingsRouter } from './routes/listingsRouter.js';
|
||||
import { getSettings } from '../services/storage/settingsStorage.js';
|
||||
import { featureRouter } from './routes/featureRouter.js';
|
||||
import { dashboardRouter } from './routes/dashboardRouter.js';
|
||||
import { backupRouter } from './routes/backupRouter.js';
|
||||
const service = restana();
|
||||
const staticService = files(path.join(getDirName(), '../ui/public'));
|
||||
const PORT = (await getSettings()).port || 9998;
|
||||
@@ -28,19 +34,22 @@ service.use('/api/admin', authInterceptor());
|
||||
service.use('/api/jobs', authInterceptor());
|
||||
service.use('/api/version', authInterceptor());
|
||||
service.use('/api/listings', authInterceptor());
|
||||
service.use('/api/dashboard', authInterceptor());
|
||||
service.use('/api/user/settings', authInterceptor());
|
||||
|
||||
// /admin can only be accessed when user is having admin permissions
|
||||
service.use('/api/admin', adminInterceptor());
|
||||
service.use('/api/jobs/notificationAdapter', notificationAdapterRouter);
|
||||
service.use('/api/admin/generalSettings', generalSettingsRouter);
|
||||
service.use('/api/admin/backup', backupRouter);
|
||||
service.use('/api/jobs/provider', providerRouter);
|
||||
service.use('/api/jobs/insights', analyticsRouter);
|
||||
service.use('/api/admin/users', userRouter);
|
||||
service.use('/api/user/settings', userSettingsRouter);
|
||||
service.use('/api/version', versionRouter);
|
||||
service.use('/api/jobs', jobRouter);
|
||||
service.use('/api/login', loginRouter);
|
||||
service.use('/api/listings', listingsRouter);
|
||||
service.use('/api/features', featureRouter);
|
||||
service.use('/api/dashboard', dashboardRouter);
|
||||
//this route is unsecured intentionally as it is being queried from the login page
|
||||
service.use('/api/demo', demoRouter);
|
||||
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
import restana from 'restana';
|
||||
import * as listingStorage from '../../services/storage/listingsStorage.js';
|
||||
const service = restana();
|
||||
const analyticsRouter = service.newRouter();
|
||||
analyticsRouter.get('/:jobId', async (req, res) => {
|
||||
const { jobId } = req.params;
|
||||
res.body = listingStorage.getListingProviderDataForAnalytics(jobId) || {};
|
||||
res.send();
|
||||
});
|
||||
export { analyticsRouter };
|
||||
75
lib/api/routes/backupRouter.js
Normal file
75
lib/api/routes/backupRouter.js
Normal file
@@ -0,0 +1,75 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import restana from 'restana';
|
||||
import {
|
||||
buildBackupFileName,
|
||||
createBackupZip,
|
||||
precheckRestore,
|
||||
restoreFromZip,
|
||||
} from '../../services/storage/backupRestoreService.js';
|
||||
|
||||
/**
|
||||
* Backup & Restore Admin Router
|
||||
*
|
||||
* Endpoints:
|
||||
* - GET /api/admin/backup
|
||||
* Returns the current database as a zip download. Content-Type: application/zip
|
||||
* - POST /api/admin/backup/restore?dryRun=true
|
||||
* Accepts a zip file (raw body). Returns a compatibility report, does not restore.
|
||||
* - POST /api/admin/backup/restore?force=true|false
|
||||
* Accepts a zip file (raw body). Restores the database; when incompatible and force=false, returns 400.
|
||||
*/
|
||||
const service = restana();
|
||||
const backupRouter = service.newRouter();
|
||||
|
||||
backupRouter.get('/', async (req, res) => {
|
||||
const zipBuffer = await createBackupZip();
|
||||
const fileName = await buildBackupFileName();
|
||||
res.setHeader('Content-Type', 'application/zip');
|
||||
res.setHeader('Content-Disposition', `attachment; filename="${fileName}"`);
|
||||
res.send(zipBuffer);
|
||||
});
|
||||
|
||||
/**
|
||||
* Read the full request body as a Buffer. Used for raw zip uploads.
|
||||
* @param {import('http').IncomingMessage} req
|
||||
* @returns {Promise<Buffer>}
|
||||
*/
|
||||
function readBody(req) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const chunks = [];
|
||||
req.on('data', (c) => chunks.push(c));
|
||||
req.on('end', () => resolve(Buffer.concat(chunks)));
|
||||
req.on('error', (e) => reject(e));
|
||||
});
|
||||
}
|
||||
|
||||
// Upload endpoint. Accepts raw zip (Content-Type: application/zip or application/octet-stream)
|
||||
// Query parameters:
|
||||
// - dryRun=true => only validate and return compatibility info
|
||||
// - force=true => proceed even if incompatible
|
||||
backupRouter.post('/restore', async (req, res) => {
|
||||
const { dryRun = 'false', force = 'false' } = req.query || {};
|
||||
const doDryRun = String(dryRun) === 'true';
|
||||
const doForce = String(force) === 'true';
|
||||
const body = await readBody(req);
|
||||
|
||||
if (doDryRun) {
|
||||
res.body = await precheckRestore(body);
|
||||
return res.send();
|
||||
}
|
||||
|
||||
try {
|
||||
res.body = await restoreFromZip(body, { force: doForce });
|
||||
return res.send();
|
||||
} catch (e) {
|
||||
res.statusCode = 400;
|
||||
res.body = { message: e?.message || 'Restore failed', details: e?.payload || null };
|
||||
return res.send();
|
||||
}
|
||||
});
|
||||
|
||||
export { backupRouter };
|
||||
71
lib/api/routes/dashboardRouter.js
Normal file
71
lib/api/routes/dashboardRouter.js
Normal file
@@ -0,0 +1,71 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import restana from 'restana';
|
||||
import * as jobStorage from '../../services/storage/jobStorage.js';
|
||||
import * as userStorage from '../../services/storage/userStorage.js';
|
||||
import { getListingsKpisForJobIds, getProviderDistributionForJobIds } from '../../services/storage/listingsStorage.js';
|
||||
import { getSettings } from '../../services/storage/settingsStorage.js';
|
||||
|
||||
const service = restana();
|
||||
export const dashboardRouter = service.newRouter();
|
||||
|
||||
function isAdmin(req) {
|
||||
const user = req.session?.currentUser ? userStorage.getUser(req.session.currentUser) : null;
|
||||
return !!user?.isAdmin;
|
||||
}
|
||||
|
||||
function getAccessibleJobs(req) {
|
||||
const currentUser = req.session.currentUser;
|
||||
const admin = isAdmin(req);
|
||||
return jobStorage
|
||||
.getJobs()
|
||||
.filter((job) => admin || job.userId === currentUser || job.shared_with_user.includes(currentUser));
|
||||
}
|
||||
|
||||
function cap(val) {
|
||||
return String(val).charAt(0).toUpperCase() + String(val).slice(1);
|
||||
}
|
||||
|
||||
dashboardRouter.get('/', async (req, res) => {
|
||||
const jobs = getAccessibleJobs(req);
|
||||
const settings = await getSettings();
|
||||
|
||||
// KPIs
|
||||
const totalJobs = jobs.length;
|
||||
const totalListings = jobs.reduce((sum, j) => sum + (j.numberOfFoundListings || 0), 0);
|
||||
const jobIds = jobs.map((j) => j.id);
|
||||
const { numberOfActiveListings, avgPriceOfListings } = getListingsKpisForJobIds(jobIds);
|
||||
// Build Pie data in a simple shape the frontend can consume directly
|
||||
// Shape: { labels: string[], values: number[] } with values as percentages
|
||||
const providerPieRaw = getProviderDistributionForJobIds(jobIds);
|
||||
const providerPie = Array.isArray(providerPieRaw)
|
||||
? {
|
||||
labels: providerPieRaw.map((p) => cap(p.type)),
|
||||
values: providerPieRaw.map((p) => Number(p.value) || 0),
|
||||
}
|
||||
: providerPieRaw && typeof providerPieRaw === 'object'
|
||||
? {
|
||||
labels: Array.isArray(providerPieRaw.labels) ? providerPieRaw.labels : [],
|
||||
values: Array.isArray(providerPieRaw.values) ? providerPieRaw.values : [],
|
||||
}
|
||||
: { labels: [], values: [] };
|
||||
|
||||
res.body = {
|
||||
general: {
|
||||
interval: settings.interval,
|
||||
lastRun: settings.lastRun || null,
|
||||
nextRun: settings.lastRun == null ? 0 : settings.lastRun + settings.interval * 60000,
|
||||
},
|
||||
kpis: {
|
||||
totalJobs,
|
||||
totalListings,
|
||||
numberOfActiveListings,
|
||||
avgPriceOfListings,
|
||||
},
|
||||
pie: providerPie,
|
||||
};
|
||||
res.send();
|
||||
});
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import restana from 'restana';
|
||||
import { getSettings } from '../../services/storage/settingsStorage.js';
|
||||
const service = restana();
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
import restana from 'restana';
|
||||
import getFeatures from '../../features.js';
|
||||
const service = restana();
|
||||
const featureRouter = service.newRouter();
|
||||
|
||||
featureRouter.get('/', async (req, res) => {
|
||||
const features = getFeatures();
|
||||
res.body = Object.assign({}, { features });
|
||||
res.send();
|
||||
});
|
||||
|
||||
export { featureRouter };
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import restana from 'restana';
|
||||
import { getDirName } from '../../utils.js';
|
||||
import fs from 'fs';
|
||||
|
||||
@@ -1,10 +1,16 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import restana from 'restana';
|
||||
import * as jobStorage from '../../services/storage/jobStorage.js';
|
||||
import * as userStorage from '../../services/storage/userStorage.js';
|
||||
import { isAdmin } from '../security.js';
|
||||
import logger from '../../services/logger.js';
|
||||
import { bus } from '../../services/events/event-bus.js';
|
||||
import { getSettings } from '../../services/storage/settingsStorage.js';
|
||||
import { isRunning as isJobRunning } from '../../services/jobs/run-state.js';
|
||||
import { addClient as addSseClient, removeClient } from '../../services/sse/sse-broker.js';
|
||||
|
||||
const service = restana();
|
||||
const jobRouter = service.newRouter();
|
||||
@@ -33,6 +39,7 @@ jobRouter.get('/', async (req, res) => {
|
||||
.map((job) => {
|
||||
return {
|
||||
...job,
|
||||
running: isJobRunning(job.id),
|
||||
isOnlyShared:
|
||||
!isUserAdmin &&
|
||||
job.userId !== req.session.currentUser &&
|
||||
@@ -43,18 +50,113 @@ jobRouter.get('/', async (req, res) => {
|
||||
res.send();
|
||||
});
|
||||
|
||||
jobRouter.get('/processingTimes', async (req, res) => {
|
||||
const settings = await getSettings();
|
||||
res.body = {
|
||||
interval: settings.interval,
|
||||
lastRun: settings.lastRun || null,
|
||||
jobRouter.get('/data', async (req, res) => {
|
||||
const { page, pageSize = 50, activityFilter, sortfield = null, sortdir = 'asc', freeTextFilter } = req.query || {};
|
||||
|
||||
// normalize booleans
|
||||
const toBool = (v) => {
|
||||
if (v === true || v === 'true' || v === 1 || v === '1') return true;
|
||||
if (v === false || v === 'false' || v === 0 || v === '0') return false;
|
||||
return null;
|
||||
};
|
||||
const normalizedActivity = toBool(activityFilter);
|
||||
|
||||
const queryResult = jobStorage.queryJobs({
|
||||
page: page ? parseInt(page, 10) : 1,
|
||||
pageSize: pageSize ? parseInt(pageSize, 10) : 50,
|
||||
freeTextFilter: freeTextFilter || null,
|
||||
activityFilter: normalizedActivity,
|
||||
sortField: sortfield || null,
|
||||
sortDir: sortdir === 'desc' ? 'desc' : 'asc',
|
||||
userId: req.session.currentUser,
|
||||
isAdmin: isAdmin(req),
|
||||
});
|
||||
|
||||
const isUserAdmin = isAdmin(req);
|
||||
|
||||
// Map result to include runtime status
|
||||
queryResult.result = queryResult.result.map((job) => {
|
||||
return {
|
||||
...job,
|
||||
running: isJobRunning(job.id),
|
||||
isOnlyShared:
|
||||
!isUserAdmin &&
|
||||
job.userId !== req.session.currentUser &&
|
||||
job.shared_with_user.includes(req.session.currentUser),
|
||||
};
|
||||
});
|
||||
|
||||
res.body = queryResult;
|
||||
res.send();
|
||||
});
|
||||
|
||||
// Server-Sent Events for job status updates
|
||||
jobRouter.get('/events', async (req, res) => {
|
||||
const userId = req.session.currentUser;
|
||||
if (userId == null) {
|
||||
res.send({ message: 'Unauthorized' }, 401);
|
||||
return;
|
||||
}
|
||||
// SSE headers
|
||||
res.setHeader('Content-Type', 'text/event-stream');
|
||||
res.setHeader('Cache-Control', 'no-cache');
|
||||
res.setHeader('Connection', 'keep-alive');
|
||||
try {
|
||||
// Initial comment to establish stream
|
||||
res.write(': connected\n\n');
|
||||
addSseClient(userId, res);
|
||||
// Cleanup on close/aborted
|
||||
const onClose = () => removeClient(userId, res);
|
||||
// restana exposes original req/res; use both close and finish
|
||||
req.on('close', onClose);
|
||||
req.on('aborted', onClose);
|
||||
res.on('close', onClose);
|
||||
} catch (e) {
|
||||
logger.error('Error establishing SSE connection', e);
|
||||
try {
|
||||
res.end();
|
||||
} catch {
|
||||
//noop
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
jobRouter.post('/startAll', async (req, res) => {
|
||||
bus.emit('jobs:runAll');
|
||||
res.send();
|
||||
try {
|
||||
const userId = req.session.currentUser;
|
||||
// Emit only the userId; handler will decide based on admin/ownership
|
||||
bus.emit('jobs:runAll', { userId });
|
||||
res.send({ message: 'Run all accepted' }, 202);
|
||||
} catch (err) {
|
||||
logger.error('Failed to trigger startAll', err);
|
||||
res.send({ message: 'Unexpected error' }, 500);
|
||||
}
|
||||
});
|
||||
|
||||
// Trigger a single job run
|
||||
jobRouter.post('/:jobId/run', async (req, res) => {
|
||||
const { jobId } = req.params;
|
||||
try {
|
||||
const job = jobStorage.getJob(jobId);
|
||||
if (!job) {
|
||||
res.send({ message: 'Job not found' }, 404);
|
||||
return;
|
||||
}
|
||||
if (!doesJobBelongsToUser(job, req)) {
|
||||
res.send({ message: 'You are trying to run a job that is not associated to your user' }, 403);
|
||||
return;
|
||||
}
|
||||
if (isJobRunning(jobId)) {
|
||||
res.send({ message: 'Job is already running' }, 409);
|
||||
return;
|
||||
}
|
||||
// fire and forget; actual execution handled by index.js listener
|
||||
bus.emit('jobs:runOne', { jobId });
|
||||
res.send({ message: 'Job run accepted' }, 202);
|
||||
} catch (error) {
|
||||
logger.error(error);
|
||||
res.send({ message: 'Unexpected error triggering job' }, 500);
|
||||
}
|
||||
});
|
||||
|
||||
jobRouter.post('/', async (req, res) => {
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import restana from 'restana';
|
||||
import * as listingStorage from '../../services/storage/listingsStorage.js';
|
||||
import * as watchListStorage from '../../services/storage/watchListStorage.js';
|
||||
@@ -23,10 +28,14 @@ listingsRouter.get('/table', async (req, res) => {
|
||||
freeTextFilter,
|
||||
} = req.query || {};
|
||||
|
||||
// normalize booleans (accept true, 'true', 1, '1')
|
||||
const toBool = (v) => v === true || v === 'true' || v === 1 || v === '1';
|
||||
const normalizedActivity = toBool(activityFilter) ? true : null;
|
||||
const normalizedWatch = toBool(watchListFilter) ? true : null;
|
||||
// normalize booleans (accept true, 'true', 1, '1' for true; false, 'false', 0, '0' for false)
|
||||
const toBool = (v) => {
|
||||
if (v === true || v === 'true' || v === 1 || v === '1') return true;
|
||||
if (v === false || v === 'false' || v === 0 || v === '0') return false;
|
||||
return null;
|
||||
};
|
||||
const normalizedActivity = toBool(activityFilter);
|
||||
const normalizedWatch = toBool(watchListFilter);
|
||||
|
||||
let jobFilter = null;
|
||||
let jobIdFilter = null;
|
||||
@@ -54,6 +63,17 @@ listingsRouter.get('/table', async (req, res) => {
|
||||
res.send();
|
||||
});
|
||||
|
||||
listingsRouter.get('/map', async (req, res) => {
|
||||
const { jobId } = req.query || {};
|
||||
|
||||
res.body = listingStorage.getListingsForMap({
|
||||
jobId: nullOrEmpty(jobId) ? null : jobId,
|
||||
userId: req.session.currentUser,
|
||||
isAdmin: isAdminFn(req),
|
||||
});
|
||||
res.send();
|
||||
});
|
||||
|
||||
// Toggle watch state for the current user on a listing
|
||||
listingsRouter.post('/watch', async (req, res) => {
|
||||
try {
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import restana from 'restana';
|
||||
import * as userStorage from '../../services/storage/userStorage.js';
|
||||
import * as hasher from '../../services/security/hash.js';
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import restana from 'restana';
|
||||
const service = restana();
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import restana from 'restana';
|
||||
const service = restana();
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import restana from 'restana';
|
||||
import * as userStorage from '../../services/storage/userStorage.js';
|
||||
import * as jobStorage from '../../services/storage/jobStorage.js';
|
||||
|
||||
66
lib/api/routes/userSettingsRoute.js
Normal file
66
lib/api/routes/userSettingsRoute.js
Normal file
@@ -0,0 +1,66 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import restana from 'restana';
|
||||
import SqliteConnection from '../../services/storage/SqliteConnection.js';
|
||||
import { upsertSettings } from '../../services/storage/settingsStorage.js';
|
||||
import { geocodeAddress } from '../../services/geocoding/geoCodingService.js';
|
||||
import { autocompleteAddress } from '../../services/geocoding/autocompleteService.js';
|
||||
import { calculateDistanceForUser } from '../../services/geocoding/distanceService.js';
|
||||
import { fromJson } from '../../utils.js';
|
||||
import { trackFeature } from '../../services/tracking/Tracker.js';
|
||||
import { FEATURES } from '../../features.js';
|
||||
|
||||
const service = restana();
|
||||
const userSettingsRouter = service.newRouter();
|
||||
|
||||
userSettingsRouter.get('/', async (req, res) => {
|
||||
const userId = req.session.currentUser;
|
||||
const rows = SqliteConnection.query('SELECT name, value FROM settings WHERE user_id = @userId', { userId });
|
||||
const settings = {};
|
||||
for (const r of rows) {
|
||||
settings[r.name] = fromJson(r.value, null);
|
||||
}
|
||||
res.body = settings;
|
||||
res.send();
|
||||
});
|
||||
|
||||
userSettingsRouter.get('/autocomplete', async (req, res) => {
|
||||
const { q } = req.query;
|
||||
try {
|
||||
const results = await autocompleteAddress(q);
|
||||
res.body = results;
|
||||
res.send();
|
||||
} catch (error) {
|
||||
res.status(500).send({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
userSettingsRouter.post('/', async (req, res) => {
|
||||
const userId = req.session.currentUser;
|
||||
const { home_address } = req.body;
|
||||
|
||||
try {
|
||||
if (home_address) {
|
||||
await trackFeature(FEATURES.DISTANCE_ADDRESS_ENTERED);
|
||||
const coords = await geocodeAddress(home_address);
|
||||
if (coords && coords.lat !== -1) {
|
||||
upsertSettings({ home_address: { address: home_address, coords } }, userId);
|
||||
calculateDistanceForUser(userId);
|
||||
res.send({ success: true, coords });
|
||||
} else {
|
||||
res.status(400).send({ error: 'Could not geocode address' });
|
||||
}
|
||||
} else {
|
||||
// If address is empty, maybe clear it?
|
||||
upsertSettings({ home_address: null }, userId);
|
||||
res.send({ success: true });
|
||||
}
|
||||
} catch (error) {
|
||||
res.status(500).send({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
export { userSettingsRouter };
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import restana from 'restana';
|
||||
import fetch from 'node-fetch';
|
||||
import { getPackageVersion } from '../../utils.js';
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import * as userStorage from '../services/storage/userStorage.js';
|
||||
import cookieSession from 'cookie-session';
|
||||
import { nanoid } from 'nanoid';
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
export const DEFAULT_CONFIG = {
|
||||
interval: '60',
|
||||
port: 9998,
|
||||
workingHours: { from: '', to: '' },
|
||||
demoMode: false,
|
||||
analyticsEnabled: null,
|
||||
// Default path for sqlite storage directory. Interpreted relative to project root.
|
||||
sqlitepath: '/db',
|
||||
};
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
class ExtendableError extends Error {
|
||||
constructor(message) {
|
||||
super(message);
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
const FEATURES = {
|
||||
WATCHLIST_MANAGEMENT: false,
|
||||
};
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
export default function getFeatures() {
|
||||
return {
|
||||
...FEATURES,
|
||||
};
|
||||
}
|
||||
export const FEATURES = {
|
||||
DISTANCE_ADDRESS_ENTERED: 'DISTANCE_ADDRESS_ENTERED',
|
||||
};
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { markdown2Html } from '../../services/markdown.js';
|
||||
import { getJob } from '../../services/storage/jobStorage.js';
|
||||
import fetch from 'node-fetch';
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { markdown2Html } from '../../services/markdown.js';
|
||||
|
||||
export const send = ({ serviceName, newListings, jobKey }) => {
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import fetch from 'node-fetch';
|
||||
import { getJob } from '../../services/storage/jobStorage.js';
|
||||
import { markdown2Html } from '../../services/markdown.js';
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { markdown2Html } from '../../services/markdown.js';
|
||||
|
||||
const mapListing = (listing) => ({
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import mailjet from 'node-mailjet';
|
||||
import path from 'path';
|
||||
import fs from 'fs';
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { markdown2Html } from '../../services/markdown.js';
|
||||
import { getJob } from '../../services/storage/jobStorage.js';
|
||||
import fetch from 'node-fetch';
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { markdown2Html } from '../../services/markdown.js';
|
||||
import { getJob } from '../../services/storage/jobStorage.js';
|
||||
import fetch from 'node-fetch';
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { markdown2Html } from '../../services/markdown.js';
|
||||
import { getJob } from '../../services/storage/jobStorage.js';
|
||||
import fetch from 'node-fetch';
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import sgMail from '@sendgrid/mail';
|
||||
import { markdown2Html } from '../../services/markdown.js';
|
||||
import { normalizeImageUrl } from '../../utils.js';
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import Slack from 'slack';
|
||||
import { markdown2Html } from '../../services/markdown.js';
|
||||
import { normalizeImageUrl } from '../../utils.js';
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import fetch from 'node-fetch';
|
||||
import { markdown2Html } from '../../services/markdown.js';
|
||||
import { normalizeImageUrl } from '../../utils.js';
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { markdown2Html } from '../../services/markdown.js';
|
||||
import Database from 'better-sqlite3';
|
||||
import path from 'path';
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { markdown2Html } from '../../services/markdown.js';
|
||||
import { getJob } from '../../services/storage/jobStorage.js';
|
||||
import fetch from 'node-fetch';
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
const path = './adapter';
|
||||
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { buildHash, isOneOf } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
let appliedBlackList = [];
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { buildHash, isOneOf } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { isOneOf, buildHash } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
let appliedBlackList = [];
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
/**
|
||||
* ImmoScout provider using the mobile API to retrieve listings.
|
||||
*
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { isOneOf, buildHash } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { buildHash, isOneOf } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { buildHash, isOneOf } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { isOneOf, buildHash } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
let appliedBlackList = [];
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { isOneOf, buildHash } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { isOneOf, buildHash } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
let appliedBlackList = [];
|
||||
@@ -39,7 +44,7 @@ export const init = (sourceConfig, blacklist) => {
|
||||
|
||||
export const metaInformation = {
|
||||
name: 'OhneMakler',
|
||||
baseUrl: 'https://www.ohne-makler.net/immobilien',
|
||||
baseUrl: 'https://www.ohne-makler.net',
|
||||
id: 'ohneMakler',
|
||||
};
|
||||
export { config };
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { isOneOf, buildHash } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
let appliedBlackList = [];
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { isOneOf, buildHash } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
let appliedBlackList = [];
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { isOneOf, buildHash } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
|
||||
|
||||
58
lib/provider/wohnungsboerse.js
Normal file
58
lib/provider/wohnungsboerse.js
Normal file
@@ -0,0 +1,58 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import * as utils from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
|
||||
let appliedBlackList = [];
|
||||
|
||||
function normalize(o) {
|
||||
const id = o.link.split('/').pop();
|
||||
const price = o.price;
|
||||
const size = o.size;
|
||||
const rooms = o.rooms;
|
||||
const [city = '', part = ''] = (o.description || '').split('-').map((v) => v.trim());
|
||||
const address = `${part}, ${city}`;
|
||||
return Object.assign(o, { id, price, size, rooms, address });
|
||||
}
|
||||
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||
return o.id != null && o.title != null && titleNotBlacklisted && descNotBlacklisted && o.link.startsWith(o.link);
|
||||
}
|
||||
|
||||
const config = {
|
||||
url: null,
|
||||
sortByDateParam: null,
|
||||
waitForSelector: 'body',
|
||||
crawlContainer: '.search_result_container > a',
|
||||
crawlFields: {
|
||||
id: '*',
|
||||
title: 'h3 | trim',
|
||||
price: 'dl:nth-of-type(1) dd | removeNewline | trim',
|
||||
rooms: 'dl:nth-of-type(2) dd | removeNewline | trim',
|
||||
size: 'dl:nth-of-type(3) dd | removeNewline | trim',
|
||||
description: 'div.before\\:icon-location_marker | trim',
|
||||
link: '@href',
|
||||
imageUrl: 'img@src',
|
||||
},
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist,
|
||||
activeTester: checkIfListingIsActive,
|
||||
};
|
||||
|
||||
export const init = (sourceConfig, blacklistTerms) => {
|
||||
config.url = sourceConfig.url;
|
||||
appliedBlackList = blacklistTerms || [];
|
||||
};
|
||||
|
||||
export const metaInformation = {
|
||||
name: 'Wohnungsboerse',
|
||||
baseUrl: 'https://www.wohnungsboerse.net',
|
||||
id: 'wohnungsboerse',
|
||||
};
|
||||
|
||||
export { config };
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { removeJobsByUserId } from '../storage/jobStorage.js';
|
||||
import { getUsers } from '../storage/userStorage.js';
|
||||
import logger from '../logger.js';
|
||||
|
||||
39
lib/services/crons/geocoding-cron.js
Normal file
39
lib/services/crons/geocoding-cron.js
Normal file
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import cron from 'node-cron';
|
||||
import { getListingsToGeocode, updateListingGeocoordinates } from '../storage/listingsStorage.js';
|
||||
import { geocodeAddress, isGeocodingPaused } from '../geocoding/geoCodingService.js';
|
||||
import { getJobs } from '../storage/jobStorage.js';
|
||||
import { calculateDistanceForJob } from '../geocoding/distanceService.js';
|
||||
|
||||
async function runTask() {
|
||||
const listings = getListingsToGeocode();
|
||||
if (listings.length > 0) {
|
||||
for (const listing of listings) {
|
||||
if (isGeocodingPaused()) {
|
||||
break;
|
||||
}
|
||||
|
||||
const coords = await geocodeAddress(listing.address);
|
||||
if (coords) {
|
||||
updateListingGeocoordinates(listing.id, coords.lat, coords.lng);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//additional run
|
||||
const jobs = getJobs();
|
||||
for (const job of jobs) {
|
||||
calculateDistanceForJob(job.id, job.userId);
|
||||
}
|
||||
}
|
||||
|
||||
export async function initGeocodingCron() {
|
||||
// run directly on start
|
||||
await runTask();
|
||||
// then every 6 hours
|
||||
cron.schedule('0 */6 * * *', runTask);
|
||||
}
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import cron from 'node-cron';
|
||||
import runActiveChecker from '../listings/listingActiveService.js';
|
||||
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import cron from 'node-cron';
|
||||
import { inDevMode } from '../../utils.js';
|
||||
import { trackMainEvent } from '../tracking/Tracker.js';
|
||||
|
||||
@@ -1,2 +1,7 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { EventEmitter } from 'node:events';
|
||||
export const bus = new EventEmitter();
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { DEFAULT_HEADER } from './utils.js';
|
||||
|
||||
// Helper to safely coerce numbers
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { setDebug } from './utils.js';
|
||||
import puppeteerExtractor from './puppeteerExtractor.js';
|
||||
import { loadParser, parse } from './parser/parser.js';
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import * as cheerio from 'cheerio';
|
||||
import logger from '../../logger.js';
|
||||
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import puppeteer from 'puppeteer-extra';
|
||||
import StealthPlugin from 'puppeteer-extra-plugin-stealth';
|
||||
import { debug, botDetected } from './utils.js';
|
||||
@@ -99,7 +104,11 @@ export default async function execute(url, waitForSelector, options) {
|
||||
result = pageSource || (await page.content());
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn('Error executing with puppeteer executor', error);
|
||||
if (error?.message?.includes('Timeout')) {
|
||||
logger.debug('Error executing with puppeteer executor', error);
|
||||
} else {
|
||||
logger.warn('Error executing with puppeteer executor', error);
|
||||
}
|
||||
result = null;
|
||||
} finally {
|
||||
try {
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import logger from '../logger.js';
|
||||
|
||||
let debuggingOn = false;
|
||||
|
||||
26
lib/services/geocoding/autocompleteService.js
Normal file
26
lib/services/geocoding/autocompleteService.js
Normal file
@@ -0,0 +1,26 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { autocomplete as nominatimAutocomplete } from './client/nominatimClient.js';
|
||||
import logger from '../logger.js';
|
||||
|
||||
/**
|
||||
* Autocompletes an address using Nominatim.
|
||||
*
|
||||
* @param {string} query - The search query.
|
||||
* @returns {Promise<string[]>} List of matching addresses.
|
||||
*/
|
||||
export async function autocompleteAddress(query) {
|
||||
if (!query) {
|
||||
return [];
|
||||
}
|
||||
|
||||
try {
|
||||
return await nominatimAutocomplete(query);
|
||||
} catch (error) {
|
||||
logger.error('Error during address autocomplete:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
152
lib/services/geocoding/client/nominatimClient.js
Normal file
152
lib/services/geocoding/client/nominatimClient.js
Normal file
@@ -0,0 +1,152 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import os from 'os';
|
||||
import crypto from 'crypto';
|
||||
import https from 'https';
|
||||
import fetch from 'node-fetch';
|
||||
import pThrottle from 'p-throttle';
|
||||
import logger from '../../logger.js';
|
||||
|
||||
const API_URL = 'https://nominatim.openstreetmap.org/search';
|
||||
|
||||
const agent = new https.Agent({
|
||||
keepAlive: true,
|
||||
keepAliveMsecs: 1000,
|
||||
});
|
||||
|
||||
const throttle = pThrottle({
|
||||
limit: 1,
|
||||
interval: 1000,
|
||||
});
|
||||
|
||||
function computeMachineId() {
|
||||
const hostname = os.hostname() || 'unknown-host';
|
||||
const nets = os.networkInterfaces?.() || {};
|
||||
const macs = [];
|
||||
|
||||
for (const ifname of Object.keys(nets)) {
|
||||
for (const addr of nets[ifname] || []) {
|
||||
if (!addr) continue;
|
||||
if (addr.internal) continue;
|
||||
if (addr.mac && addr.mac !== '00:00:00:00:00:00') macs.push(addr.mac);
|
||||
}
|
||||
}
|
||||
|
||||
macs.sort();
|
||||
|
||||
const raw = [hostname, os.platform(), os.arch(), ...macs].join('|');
|
||||
|
||||
return crypto.createHash('sha256').update(raw).digest('hex').slice(0, 20);
|
||||
}
|
||||
|
||||
/**
|
||||
* Nominatim requires a specific User-Agent.
|
||||
* Since Fredy is self-hosted, we use a unique machine ID to make it specific.
|
||||
*/
|
||||
const userAgent = `Fredy-Self-Hosted (${computeMachineId()}; https://github.com/orangecoding/fredy)`;
|
||||
|
||||
let last429 = 0;
|
||||
const PAUSE_DURATION = 3600000; // 1 hour
|
||||
|
||||
/**
|
||||
* Geocodes an address using Nominatim.
|
||||
*
|
||||
* @param {string} address - The address to geocode.
|
||||
* @returns {Promise<{lat: number, lng: number}|null>} The geocoordinates or null if error. {lat: -1, lng: -1} if not found.
|
||||
*/
|
||||
async function doGeocode(address) {
|
||||
if (Date.now() - last429 < PAUSE_DURATION) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const url = `${API_URL}?q=${encodeURIComponent(address)}&format=json&countrycodes=de`;
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
agent,
|
||||
headers: {
|
||||
'User-Agent': userAgent,
|
||||
},
|
||||
});
|
||||
|
||||
if (response.status === 429) {
|
||||
logger.warn('Nominatim rate limit hit. Pausing for 1 hour.');
|
||||
last429 = Date.now();
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
logger.error(`Nominatim API error: ${response.status} ${response.statusText}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
if (Array.isArray(data) && data.length > 0) {
|
||||
const result = data[0];
|
||||
return {
|
||||
lat: parseFloat(result.lat),
|
||||
lng: parseFloat(result.lon),
|
||||
};
|
||||
}
|
||||
|
||||
return { lat: -1, lng: -1 };
|
||||
} catch (error) {
|
||||
logger.error('Error during Nominatim geocoding:', error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Autocompletes an address using Nominatim.
|
||||
*
|
||||
* @param {string} query - The search query.
|
||||
* @returns {Promise<string[]>} List of matching addresses.
|
||||
*/
|
||||
async function doAutocomplete(query) {
|
||||
if (Date.now() - last429 < PAUSE_DURATION) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const url = `${API_URL}?q=${encodeURIComponent(query)}&format=json&addressdetails=1&limit=5&countrycodes=de`;
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
agent,
|
||||
headers: {
|
||||
'User-Agent': userAgent,
|
||||
},
|
||||
});
|
||||
|
||||
if (response.status === 429) {
|
||||
logger.warn('Nominatim rate limit hit. Pausing for 1 hour.');
|
||||
last429 = Date.now();
|
||||
return [];
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
logger.error(`Nominatim API error: ${response.status} ${response.statusText}`);
|
||||
return [];
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
if (Array.isArray(data)) {
|
||||
return data.map((item) => item.display_name);
|
||||
}
|
||||
|
||||
return [];
|
||||
} catch (error) {
|
||||
logger.error('Error during Nominatim autocomplete:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export const geocode = throttle(doGeocode);
|
||||
|
||||
export const autocomplete = throttle(doAutocomplete);
|
||||
|
||||
export const isPaused = () => Date.now() - last429 < PAUSE_DURATION;
|
||||
61
lib/services/geocoding/distanceService.js
Normal file
61
lib/services/geocoding/distanceService.js
Normal file
@@ -0,0 +1,61 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { distanceMeters } from '../listings/distanceCalculator.js';
|
||||
import {
|
||||
getListingsToCalculateDistance,
|
||||
getListingsForUserToCalculateDistance,
|
||||
updateListingDistance,
|
||||
} from '../storage/listingsStorage.js';
|
||||
import { getUserSettings } from '../storage/settingsStorage.js';
|
||||
|
||||
/**
|
||||
* Calculates and updates distances for listings of a specific job.
|
||||
* Only processes listings where distance_to_destination is null.
|
||||
*
|
||||
* @param {string} jobId
|
||||
* @param {string} userId
|
||||
* @returns {void}
|
||||
*/
|
||||
export function calculateDistanceForJob(jobId, userId) {
|
||||
const userSettings = getUserSettings(userId);
|
||||
const homeAddress = userSettings.home_address;
|
||||
|
||||
if (!homeAddress || !homeAddress.coords) {
|
||||
return;
|
||||
}
|
||||
|
||||
const listings = getListingsToCalculateDistance(jobId);
|
||||
const { lat, lng } = homeAddress.coords;
|
||||
|
||||
for (const listing of listings) {
|
||||
const dist = distanceMeters(lat, lng, listing.latitude, listing.longitude);
|
||||
updateListingDistance(listing.id, dist);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates and updates distances for all active listings of a user.
|
||||
* Usually called when the user updates their home address.
|
||||
*
|
||||
* @param {string} userId
|
||||
* @returns {void}
|
||||
*/
|
||||
export function calculateDistanceForUser(userId) {
|
||||
const userSettings = getUserSettings(userId);
|
||||
const homeAddress = userSettings.home_address;
|
||||
|
||||
if (!homeAddress || !homeAddress.coords) {
|
||||
return;
|
||||
}
|
||||
|
||||
const listings = getListingsForUserToCalculateDistance(userId);
|
||||
const { lat, lng } = homeAddress.coords;
|
||||
|
||||
for (const listing of listings) {
|
||||
const dist = distanceMeters(lat, lng, listing.latitude, listing.longitude);
|
||||
updateListingDistance(listing.id, dist);
|
||||
}
|
||||
}
|
||||
43
lib/services/geocoding/geoCodingService.js
Normal file
43
lib/services/geocoding/geoCodingService.js
Normal file
@@ -0,0 +1,43 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { getGeocoordinatesByAddress } from '../storage/listingsStorage.js';
|
||||
import { geocode as nominatimGeocode, isPaused as isNominatimPaused } from './client/nominatimClient.js';
|
||||
import logger from '../logger.js';
|
||||
|
||||
/**
|
||||
* Geocodes an address using Nominatim or cached results from the database.
|
||||
*
|
||||
* @param {string} address - The address to geocode.
|
||||
* @returns {Promise<{lat: number, lng: number}|null>} The geocoordinates or null if error. {lat: -1, lng: -1} if not found.
|
||||
*/
|
||||
export async function geocodeAddress(address) {
|
||||
if (!address) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
// 1. Check if we already have this address geocoded in our database
|
||||
const cachedCoordinates = getGeocoordinatesByAddress(address);
|
||||
if (cachedCoordinates) {
|
||||
logger.debug(`Found cached geocoordinates for address: ${address}`);
|
||||
return cachedCoordinates;
|
||||
}
|
||||
|
||||
// 2. If not, use Nominatim
|
||||
return await nominatimGeocode(address);
|
||||
} catch (error) {
|
||||
logger.error('Error during geocoding:', error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if we are currently in a rate limit pause.
|
||||
* @returns {boolean}
|
||||
*/
|
||||
export function isGeocodingPaused() {
|
||||
return isNominatimPaused();
|
||||
}
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
/*
|
||||
Rent a flat
|
||||
Web:
|
||||
@@ -104,7 +109,9 @@ const REAL_ESTATE_TYPE = {
|
||||
const WEB_PATH_TO_APARTMENT_EQUIPMENT_MAP = {
|
||||
// Category "Balkon/Terrasse"
|
||||
'wohnung-mit-balkon-mieten': { equipment: ['balcony'] },
|
||||
'wohnung-kaufen-mit-balkon': { equipment: ['balcony'] },
|
||||
'wohnung-mit-garten-mieten': { equipment: ['garden'] },
|
||||
'eigentumswohnung-mit-garten': { equipment: ['garden'] },
|
||||
// Category "Wohnungstyp"
|
||||
'souterrainwohnung-mieten': { apartmenttypes: ['halfbasement'] },
|
||||
'erdgeschosswohnung-mieten': { apartmenttypes: ['groundfloor'] },
|
||||
|
||||
187
lib/services/jobs/jobExecutionService.js
Normal file
187
lib/services/jobs/jobExecutionService.js
Normal file
@@ -0,0 +1,187 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import logger from '../logger.js';
|
||||
import { bus } from '../events/event-bus.js';
|
||||
import * as jobStorage from '../storage/jobStorage.js';
|
||||
import * as userStorage from '../storage/userStorage.js';
|
||||
import { getUser } from '../storage/userStorage.js';
|
||||
import { duringWorkingHoursOrNotSet } from '../../utils.js';
|
||||
import FredyPipelineExecutioner from '../../FredyPipelineExecutioner.js';
|
||||
import * as similarityCache from '../similarity-check/similarityCache.js';
|
||||
import { isRunning, markFinished, markRunning } from './run-state.js';
|
||||
import { sendToUsers } from '../sse/sse-broker.js';
|
||||
|
||||
/**
|
||||
* Initializes the job execution service.
|
||||
* - Registers event-bus listeners for `jobs:runAll`, `jobs:runOne`, and `jobs:status`.
|
||||
* - Starts the periodic scheduler (if `intervalMs` > 0) and performs an initial run respecting working hours.
|
||||
* - Forwards job status updates to affected users via Server-Sent Events (SSE).
|
||||
*
|
||||
* This function is intentionally side-effectful and exposes no external API.
|
||||
*
|
||||
* @param {Object} deps - Dependencies required to initialize the service.
|
||||
* @param {Array<Object>} deps.providers - Loaded provider modules. Each module must expose `metaInformation.id`, `config`, and `init(config, blacklist)`.
|
||||
* @param {Object} deps.settings - Global settings object (read/write). Must include `demoMode`, `interval`, and working-hours attributes used by `duringWorkingHoursOrNotSet`.
|
||||
* @param {number} deps.intervalMs - Scheduler interval in milliseconds. If not finite or <= 0, the scheduler is not started.
|
||||
* @returns {void}
|
||||
*/
|
||||
export function initJobExecutionService({ providers, settings, intervalMs }) {
|
||||
// Forward job status via SSE to relevant recipients
|
||||
bus.on('jobs:status', ({ jobId, running }) => {
|
||||
try {
|
||||
const recipients = resolveRecipients(jobId);
|
||||
if (recipients.length > 0) {
|
||||
sendToUsers(recipients, 'jobStatus', { jobId, running });
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn('Failed to forward job status', jobId, err);
|
||||
}
|
||||
});
|
||||
|
||||
// Listen for "run all" requests (admin = all, user = own)
|
||||
bus.on('jobs:runAll', (payload) => {
|
||||
const userId = payload?.userId ?? null;
|
||||
const user = userId ? getUser(userId) : null;
|
||||
const isAdmin = !!user?.isAdmin;
|
||||
if (isAdmin) {
|
||||
logger.debug('Running all jobs manually (admin request)');
|
||||
} else if (userId) {
|
||||
logger.debug(`Running all jobs manually for user ${userId}`);
|
||||
} else {
|
||||
logger.debug('Running all jobs manually (no user provided)');
|
||||
}
|
||||
runAll(false, { userId, isAdmin });
|
||||
});
|
||||
|
||||
// Listen for single job run requests
|
||||
bus.on('jobs:runOne', ({ jobId }) => {
|
||||
logger.debug(`Running single job manually: ${jobId}`);
|
||||
// fire and forget, do not block the bus
|
||||
runSingle(jobId);
|
||||
});
|
||||
|
||||
// Start scheduler and initial run
|
||||
if (Number.isFinite(intervalMs) && intervalMs > 0) {
|
||||
setInterval(() => runAll(true), intervalMs);
|
||||
}
|
||||
// start once at startup, respecting working hours
|
||||
runAll(true);
|
||||
|
||||
/**
|
||||
* Resolve all recipients who should receive SSE updates for a job.
|
||||
* Includes job owner, users with whom the job is shared, and all admins.
|
||||
*
|
||||
* @param {string} jobId
|
||||
* @returns {string[]} unique userIds
|
||||
*/
|
||||
function resolveRecipients(jobId) {
|
||||
const job = jobStorage.getJob(jobId);
|
||||
if (!job) return [];
|
||||
const admins = (userStorage.getUsers && userStorage.getUsers(false)) || [];
|
||||
const adminIds = admins.filter((u) => u.isAdmin).map((u) => u.id);
|
||||
const shared = Array.isArray(job.shared_with_user) ? job.shared_with_user : [];
|
||||
const recipients = [job.userId, ...shared, ...adminIds].filter(Boolean);
|
||||
return Array.from(new Set(recipients));
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute all enabled jobs, optionally filtering by context (admin/owner) and respecting working hours.
|
||||
*
|
||||
* @param {boolean} [respectWorkingHours=true] - If true, skip execution when outside configured working hours.
|
||||
* @param {{userId?: string, isAdmin?: boolean}} [context] - Who requested the run; determines job filtering.
|
||||
* @returns {void}
|
||||
*/
|
||||
function runAll(respectWorkingHours = true, context = undefined) {
|
||||
if (settings.demoMode) return;
|
||||
const now = Date.now();
|
||||
const withinHours = duringWorkingHoursOrNotSet(settings, now);
|
||||
if (respectWorkingHours && !withinHours) {
|
||||
logger.debug('Working hours set. Skipping as outside of working hours.');
|
||||
return;
|
||||
}
|
||||
settings.lastRun = now;
|
||||
jobStorage
|
||||
.getJobs()
|
||||
.filter((job) => job.enabled)
|
||||
.filter((job) => {
|
||||
if (!context) return true; // startup/cron → all
|
||||
if (context.isAdmin) return true; // admin → all
|
||||
return context.userId ? job.userId === context.userId : false; // user → own
|
||||
})
|
||||
.forEach((job) => executeJob(job));
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a single job by id.
|
||||
* Manual runs are allowed even if the job is disabled, but never duplicated when already running.
|
||||
*
|
||||
* @param {string} jobId
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function runSingle(jobId) {
|
||||
if (settings.demoMode) return;
|
||||
const job = jobStorage.getJob(jobId);
|
||||
if (!job) return;
|
||||
// allow manual run even if disabled; keep guard to avoid duplicates
|
||||
await executeJob(job);
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes one job across all of its configured providers.
|
||||
* Emits SSE start/finish events via the bus and ensures the run-state guard is always cleared.
|
||||
* Provider errors are surfaced via logging but do not abort other providers.
|
||||
*
|
||||
* @param {Object} job
|
||||
* @param {string} job.id
|
||||
* @param {Array<{id:string}>} job.provider
|
||||
* @param {Array<string>} [job.blacklist]
|
||||
* @param {*} job.notificationAdapter
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function executeJob(job) {
|
||||
if (isRunning(job.id)) {
|
||||
logger.debug(`Job ${job.id} is already running. Skipping.`);
|
||||
return;
|
||||
}
|
||||
const acquired = markRunning(job.id);
|
||||
if (!acquired) return;
|
||||
// notify listeners (SSE) that the job started
|
||||
try {
|
||||
bus.emit('jobs:status', { jobId: job.id, running: true });
|
||||
} catch (err) {
|
||||
logger.warn('Failed to emit start status for job', job.id, err);
|
||||
}
|
||||
try {
|
||||
const jobProviders = job.provider.filter(
|
||||
(p) => providers.find((loaded) => loaded.metaInformation.id === p.id) != null,
|
||||
);
|
||||
const executions = jobProviders.map(async (prov) => {
|
||||
const matchedProvider = providers.find((loaded) => loaded.metaInformation.id === prov.id);
|
||||
matchedProvider.init(prov, job.blacklist);
|
||||
await new FredyPipelineExecutioner(
|
||||
matchedProvider.config,
|
||||
job.notificationAdapter,
|
||||
prov.id,
|
||||
job.id,
|
||||
similarityCache,
|
||||
).execute();
|
||||
});
|
||||
const results = await Promise.allSettled(executions);
|
||||
for (const r of results) {
|
||||
if (r.status === 'rejected') {
|
||||
logger.error(r.reason);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
markFinished(job.id);
|
||||
try {
|
||||
bus.emit('jobs:status', { jobId: job.id, running: false });
|
||||
} catch (err) {
|
||||
logger.warn('Failed to emit finish status for job', job.id, err);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
42
lib/services/jobs/run-state.js
Normal file
42
lib/services/jobs/run-state.js
Normal file
@@ -0,0 +1,42 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
/**
|
||||
* Simple in-memory running state registry for jobs.
|
||||
* Prevents concurrent execution of the same job within a single process.
|
||||
* This registry is reset on process restart.
|
||||
* @type {Set<string>}
|
||||
*/
|
||||
const running = new Set();
|
||||
|
||||
/**
|
||||
* Check if a job is currently marked as running.
|
||||
* @param {string} jobId
|
||||
* @returns {boolean}
|
||||
*/
|
||||
export function isRunning(jobId) {
|
||||
return running.has(jobId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Try to mark a job as running.
|
||||
* If it was already running, returns false and does not modify the set.
|
||||
* @param {string} jobId
|
||||
* @returns {boolean} true if the job was successfully marked as running
|
||||
*/
|
||||
export function markRunning(jobId) {
|
||||
if (running.has(jobId)) return false;
|
||||
running.add(jobId);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark a job as finished (remove from the running registry).
|
||||
* @param {string} jobId
|
||||
* @returns {void}
|
||||
*/
|
||||
export function markFinished(jobId) {
|
||||
running.delete(jobId);
|
||||
}
|
||||
35
lib/services/listings/distanceCalculator.js
Normal file
35
lib/services/listings/distanceCalculator.js
Normal file
@@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
const R = 6371000; // Earth radius in meters
|
||||
/**
|
||||
* Calculate the great-circle distance between two points on Earth using the Haversine formula.
|
||||
* This is to calculate the distance between the listing address & the address provided by the user. I know, it is only
|
||||
* a rough estimation as this calculates the distance as a straight line, but it's more convenient than using an external
|
||||
* service and still gives a good approximation for sorting purposes.
|
||||
* Returns distance in meters.
|
||||
*
|
||||
* @param {number} lat1
|
||||
* @param {number} lon1
|
||||
* @param {number} lat2
|
||||
* @param {number} lon2
|
||||
* @returns {number}
|
||||
*/
|
||||
export function distanceMeters(lat1, lon1, lat2, lon2) {
|
||||
const toRad = (deg) => (deg * Math.PI) / 180;
|
||||
|
||||
const phi1 = toRad(lat1);
|
||||
const phi2 = toRad(lat2);
|
||||
const dPhi = toRad(lat2 - lat1);
|
||||
const dLambda = toRad(lon2 - lon1);
|
||||
|
||||
const a =
|
||||
Math.sin(dPhi / 2) * Math.sin(dPhi / 2) +
|
||||
Math.cos(phi1) * Math.cos(phi2) * Math.sin(dLambda / 2) * Math.sin(dLambda / 2);
|
||||
|
||||
const c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
|
||||
|
||||
return Math.round(R * c * 10) / 10;
|
||||
}
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { deactivateListings, getActiveOrUnknownListings } from '../storage/listingsStorage.js';
|
||||
import { getProviders } from '../../utils.js';
|
||||
import logger from '../../services/logger.js';
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import fetch from 'node-fetch';
|
||||
import { randomBetween, sleep } from '../../utils.js';
|
||||
|
||||
@@ -21,6 +26,7 @@ export default async function checkIfListingIsActive(link) {
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
try {
|
||||
const res = await fetch(link, {
|
||||
redirect: 'manual',
|
||||
headers: {
|
||||
'User-Agent':
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36',
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
const COLORS = {
|
||||
debug: '\x1b[36m',
|
||||
info: '\x1b[32m',
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
export function markdown2Html(filePath) {
|
||||
return fs.readFileSync(filePath, 'utf8');
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import queryString from 'query-string';
|
||||
export default (_url, sortByDateParam) => {
|
||||
//if no mutation is necessary, just return the original url
|
||||
|
||||
@@ -1,2 +1,7 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import crypto from 'crypto';
|
||||
export const hash = (x) => crypto.createHash('sha256').update(x, 'utf8').digest('hex');
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
/**
|
||||
* Similarity cache
|
||||
*
|
||||
|
||||
108
lib/services/sse/sse-broker.js
Normal file
108
lib/services/sse/sse-broker.js
Normal file
@@ -0,0 +1,108 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
/**
|
||||
* In-memory SSE client registry.
|
||||
* Maps a userId to a Set of Node.js ServerResponse objects representing open streams.
|
||||
* @type {Map<string, Set<import('http').ServerResponse>>}
|
||||
*/
|
||||
const clients = new Map(); // Map<userId, Set<ServerResponse>>
|
||||
|
||||
/**
|
||||
* Write a single SSE event frame to a response.
|
||||
*
|
||||
* @param {import('http').ServerResponse} res - The open SSE HTTP response.
|
||||
* @param {string} [event] - Optional event name (sent as `event:`). If omitted, a generic message is sent.
|
||||
* @param {any} [data] - Optional payload. Objects are JSON.stringified.
|
||||
* @returns {void}
|
||||
*/
|
||||
function writeEvent(res, event, data) {
|
||||
try {
|
||||
if (event) {
|
||||
res.write(`event: ${event}\n`);
|
||||
}
|
||||
if (data !== undefined) {
|
||||
const payload = typeof data === 'string' ? data : JSON.stringify(data);
|
||||
res.write(`data: ${payload}\n`);
|
||||
}
|
||||
res.write('\n');
|
||||
} catch {
|
||||
// ignore write errors here; cleanup happens on close
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a new SSE client for the given user.
|
||||
*
|
||||
* @param {string} userId
|
||||
* @param {import('http').ServerResponse} res
|
||||
* @returns {void}
|
||||
*/
|
||||
export function addClient(userId, res) {
|
||||
let set = clients.get(userId);
|
||||
if (!set) {
|
||||
set = new Set();
|
||||
clients.set(userId, set);
|
||||
}
|
||||
set.add(res);
|
||||
// send a hello event
|
||||
writeEvent(res, 'hello', { ok: true });
|
||||
}
|
||||
|
||||
/**
|
||||
* Unregister a specific SSE client for a user. Removes the user entry when empty.
|
||||
*
|
||||
* @param {string} userId
|
||||
* @param {import('http').ServerResponse} res
|
||||
* @returns {void}
|
||||
*/
|
||||
export function removeClient(userId, res) {
|
||||
const set = clients.get(userId);
|
||||
if (!set) return;
|
||||
set.delete(res);
|
||||
if (set.size === 0) clients.delete(userId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Send an SSE event to all open connections of a user.
|
||||
*
|
||||
* @param {string} userId
|
||||
* @param {string} event
|
||||
* @param {any} data
|
||||
* @returns {void}
|
||||
*/
|
||||
export function sendToUser(userId, event, data) {
|
||||
const set = clients.get(userId);
|
||||
if (!set) return;
|
||||
for (const res of set) {
|
||||
writeEvent(res, event, data);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Broadcast an SSE event to multiple users (unique by id).
|
||||
*
|
||||
* @param {string[]} userIds
|
||||
* @param {string} event
|
||||
* @param {any} data
|
||||
* @returns {void}
|
||||
*/
|
||||
export function sendToUsers(userIds, event, data) {
|
||||
const unique = Array.from(new Set(userIds));
|
||||
unique.forEach((id) => sendToUser(id, event, data));
|
||||
}
|
||||
|
||||
// Heartbeat to keep connections alive on proxies (every 25s)
|
||||
setInterval(() => {
|
||||
for (const set of clients.values()) {
|
||||
for (const res of set) {
|
||||
try {
|
||||
res.write(`: ping ${Date.now()}\n\n`);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
}, 25000);
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import Database from 'better-sqlite3';
|
||||
@@ -150,3 +155,21 @@ class SqliteConnection {
|
||||
}
|
||||
|
||||
export default SqliteConnection;
|
||||
|
||||
// Centralized DB path computation to avoid duplication across modules
|
||||
// Returns: { dir, dbPath }
|
||||
/**
|
||||
* Compute the absolute SQLite database directory and file path based on configuration.
|
||||
* Ensures the directory exists on disk.
|
||||
* @returns {Promise<{dir:string, dbPath:string}>} Absolute directory and database file path.
|
||||
*/
|
||||
export async function computeDbPath() {
|
||||
const cfg = await readConfigFromStorage();
|
||||
const rawDir = cfg?.sqlitepath && cfg.sqlitepath.length > 0 ? cfg.sqlitepath : '/db';
|
||||
const relDir = rawDir.startsWith('/') ? rawDir.slice(1) : rawDir;
|
||||
const absDir = path.isAbsolute(relDir) ? relDir : path.join(process.cwd(), relDir);
|
||||
const dbPath = path.join(absDir, 'listings.db');
|
||||
const dir = path.dirname(dbPath);
|
||||
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
||||
return { dir: absDir, dbPath };
|
||||
}
|
||||
|
||||
320
lib/services/storage/backupRestoreService.js
Normal file
320
lib/services/storage/backupRestoreService.js
Normal file
@@ -0,0 +1,320 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import SqliteConnection, { computeDbPath } from './SqliteConnection.js';
|
||||
import logger from '../../services/logger.js';
|
||||
import { getPackageVersion } from '../../utils.js';
|
||||
import { runMigrations, listMigrationFiles } from './migrations/migrate.js';
|
||||
|
||||
/**
|
||||
* Lazily resolve and cache the AdmZip constructor via dynamic import.
|
||||
* This keeps startup costs low and avoids ESM/CJS interop pitfalls.
|
||||
* @returns {Promise<any>} AdmZip constructor (class)
|
||||
*/
|
||||
let _AdmZipSingleton = null;
|
||||
async function getAdmZip() {
|
||||
if (_AdmZipSingleton) return _AdmZipSingleton;
|
||||
// Allow tests to provide a mock constructor without ESM loader intricacies
|
||||
if (globalThis && globalThis.__TEST_ADM_ZIP__) {
|
||||
_AdmZipSingleton = globalThis.__TEST_ADM_ZIP__;
|
||||
return _AdmZipSingleton;
|
||||
}
|
||||
const mod = await import('adm-zip');
|
||||
_AdmZipSingleton = (mod && mod.default) || mod;
|
||||
return _AdmZipSingleton;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract numeric migration id from a migration file name like "12.add-users.js".
|
||||
* @param {string} name
|
||||
* @returns {number} Parsed id or 0 when not parsable
|
||||
*/
|
||||
function parseMigrationIdFromName(name) {
|
||||
if (typeof name !== 'string') return 0;
|
||||
const m = name.match(/^(\d+)\./);
|
||||
return m ? parseInt(m[1], 10) : 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the highest migration id from available migration files.
|
||||
* @returns {number} Highest migration id from files, or 0 when none.
|
||||
*/
|
||||
function getLatestMigrationIdFromFiles() {
|
||||
try {
|
||||
const files = listMigrationFiles();
|
||||
const ids = files.map((f) => f.id);
|
||||
return ids.length > 0 ? Math.max(...ids) : 0;
|
||||
} catch (e) {
|
||||
logger.warn('Failed to scan migrations directory:', e.message);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Inspect the current database and return the highest applied migration id.
|
||||
* @returns {number} Max id from schema_migrations, or 0 when table/rows are missing.
|
||||
*/
|
||||
function getCurrentDbMigration() {
|
||||
try {
|
||||
const exists = SqliteConnection.tableExists('schema_migrations');
|
||||
if (!exists) return 0;
|
||||
const rows = SqliteConnection.query('SELECT name FROM schema_migrations');
|
||||
if (!rows || rows.length === 0) return 0;
|
||||
return rows.reduce((max, r) => Math.max(max, parseMigrationIdFromName(r.name)), 0);
|
||||
} catch (e) {
|
||||
logger.warn('Failed to read current DB migration:', e.message);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a consistent SQLite snapshot using the native backup API into a temp folder.
|
||||
* @returns {Promise<{tempDir:string, backupPath:string}>}
|
||||
*/
|
||||
async function createTempBackupFile() {
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'fredy-db-'));
|
||||
const backupPath = path.join(tempDir, 'listings.db');
|
||||
// Ensure connection is open and create a consistent snapshot
|
||||
const db = SqliteConnection.getConnection();
|
||||
await db.backup(backupPath);
|
||||
return { tempDir, backupPath };
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a zip buffer that contains the DB snapshot and metadata marker.
|
||||
* Files:
|
||||
* - listings.db
|
||||
* - fredy-backup.json { formatVersion, createdAt, dbMigration, fredyVersion }
|
||||
* @returns {Promise<Buffer>}
|
||||
*/
|
||||
async function buildBackupZipBuffer() {
|
||||
const { backupPath, tempDir } = await createTempBackupFile();
|
||||
try {
|
||||
const AdmZip = await getAdmZip();
|
||||
const zip = new AdmZip();
|
||||
const meta = {
|
||||
formatVersion: 1,
|
||||
createdAt: new Date().toISOString(),
|
||||
dbMigration: getCurrentDbMigration(),
|
||||
fredyVersion: await getPackageVersion(),
|
||||
};
|
||||
// add files
|
||||
zip.addLocalFile(backupPath, '', 'listings.db');
|
||||
zip.addFile('fredy-backup.json', Buffer.from(JSON.stringify(meta, null, 2), 'utf-8'));
|
||||
return zip.toBuffer();
|
||||
} finally {
|
||||
// cleanup temp
|
||||
try {
|
||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
} catch (e) {
|
||||
logger.debug('Failed to cleanup temp backup dir:', e.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read and parse the metadata file from a backup zip buffer.
|
||||
* @param {Buffer} zipBuffer
|
||||
* @returns {Promise<any|null>} Parsed JSON or null when missing/invalid.
|
||||
*/
|
||||
async function readMetadataFromZip(zipBuffer) {
|
||||
const AdmZip = await getAdmZip();
|
||||
const zip = new AdmZip(zipBuffer);
|
||||
const entry = zip.getEntry('fredy-backup.json');
|
||||
if (!entry) return null;
|
||||
try {
|
||||
const txt = entry.getData().toString('utf-8');
|
||||
return JSON.parse(txt);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a backup zip contains a listings.db entry.
|
||||
* @param {Buffer} zipBuffer
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
async function hasListingsDbInZip(zipBuffer) {
|
||||
const AdmZip = await getAdmZip();
|
||||
const zip = new AdmZip(zipBuffer);
|
||||
return zip.getEntry('listings.db') != null || zip.getEntries().some((e) => /listings\.db$/i.test(e.entryName));
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the listings.db from a backup zip buffer to a temp directory.
|
||||
* @param {Buffer} zipBuffer
|
||||
* @returns {Promise<{tempDir:string, dbPath:string}>}
|
||||
*/
|
||||
async function extractListingsDbToTemp(zipBuffer) {
|
||||
const AdmZip = await getAdmZip();
|
||||
const zip = new AdmZip(zipBuffer);
|
||||
const entry = zip.getEntry('listings.db') || zip.getEntries().find((e) => /listings\.db$/i.test(e.entryName));
|
||||
if (!entry) throw new Error('Backup zip does not contain listings.db');
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'fredy-restore-'));
|
||||
const outPath = path.join(tempDir, 'listings.db');
|
||||
fs.writeFileSync(outPath, entry.getData());
|
||||
return { tempDir, dbPath: outPath };
|
||||
}
|
||||
|
||||
/**
|
||||
* Public: Create a backup zip buffer ready for download.
|
||||
* @returns {Promise<Buffer>}
|
||||
*/
|
||||
export async function createBackupZip() {
|
||||
return buildBackupZipBuffer();
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze a backup zip for compatibility with the current codebase.
|
||||
* - Missing DB yields danger.
|
||||
* - Newer backup migration than required yields danger.
|
||||
* - Older backup yields warning but is considered compatible (auto-migrate).
|
||||
* - Equal version yields info.
|
||||
* @param {Buffer} zipBuffer
|
||||
* @returns {Promise<{compatible:boolean,severity:'danger'|'warning'|'info',message:string,backupMigration:number|null,requiredMigration:number,fredyVersion?:string|null}>>}
|
||||
*/
|
||||
export async function precheckRestore(zipBuffer) {
|
||||
if (!zipBuffer || zipBuffer.length === 0) {
|
||||
return {
|
||||
compatible: false,
|
||||
severity: 'danger',
|
||||
message: 'Empty upload',
|
||||
backupMigration: null,
|
||||
requiredMigration: getLatestMigrationIdFromFiles(),
|
||||
};
|
||||
}
|
||||
if (!(await hasListingsDbInZip(zipBuffer))) {
|
||||
return {
|
||||
compatible: false,
|
||||
severity: 'danger',
|
||||
message: 'Zip file is missing the database file (listings.db).',
|
||||
backupMigration: null,
|
||||
requiredMigration: getLatestMigrationIdFromFiles(),
|
||||
};
|
||||
}
|
||||
const meta = await readMetadataFromZip(zipBuffer);
|
||||
const requiredMigration = getLatestMigrationIdFromFiles();
|
||||
const backupMigration = meta?.dbMigration ?? null;
|
||||
const fredyVersion = meta?.fredyVersion ?? null;
|
||||
|
||||
if (backupMigration == null) {
|
||||
return {
|
||||
compatible: false,
|
||||
severity: 'danger',
|
||||
message:
|
||||
'Backup metadata is missing the migration marker. Cannot validate compatibility. It is NOT advised to continue!',
|
||||
backupMigration,
|
||||
requiredMigration,
|
||||
fredyVersion,
|
||||
};
|
||||
}
|
||||
|
||||
if (backupMigration > requiredMigration) {
|
||||
return {
|
||||
compatible: false,
|
||||
severity: 'danger',
|
||||
message:
|
||||
'Backup schema is newer than this Fredy version. Please upgrade Fredy to a version that supports this backup or proceed at your own risk.',
|
||||
backupMigration,
|
||||
requiredMigration,
|
||||
fredyVersion,
|
||||
};
|
||||
}
|
||||
|
||||
if (backupMigration < requiredMigration) {
|
||||
return {
|
||||
compatible: true,
|
||||
severity: 'warning',
|
||||
message:
|
||||
'Backup contains an older database schema than this Fredy version requires. We will apply automatic migrations right after the restore to upgrade the database.',
|
||||
backupMigration,
|
||||
requiredMigration,
|
||||
fredyVersion,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
compatible: true,
|
||||
severity: 'info',
|
||||
message: 'Backup is compatible with the current Fredy version.',
|
||||
backupMigration,
|
||||
requiredMigration,
|
||||
fredyVersion,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform a restore from a validated backup zip.
|
||||
* - Optionally forces restore when incompatible.
|
||||
* - Replaces the on-disk DB and runs migrations when needed.
|
||||
* @param {Buffer} zipBuffer
|
||||
* @param {{force?:boolean}} [opts]
|
||||
* @returns {Promise<{restored:true,warning:string|null,details:any}>}
|
||||
* @throws Error with code 'INCOMPATIBLE' when not forced and incompatible
|
||||
*/
|
||||
export async function restoreFromZip(zipBuffer, { force = false } = {}) {
|
||||
const check = await precheckRestore(zipBuffer);
|
||||
if (!check.compatible && !force) {
|
||||
const err = new Error(check.message || 'Backup is incompatible');
|
||||
err.code = 'INCOMPATIBLE';
|
||||
err.payload = check;
|
||||
throw err;
|
||||
}
|
||||
|
||||
const { dbPath } = await computeDbPath();
|
||||
const { tempDir, dbPath: tempDbPath } = await extractListingsDbToTemp(zipBuffer);
|
||||
|
||||
try {
|
||||
// Close existing connection to allow file replacement
|
||||
SqliteConnection.close();
|
||||
|
||||
// Backup existing DB file
|
||||
try {
|
||||
if (fs.existsSync(dbPath)) {
|
||||
const backupName = `${dbPath}.bak-${Date.now()}`;
|
||||
fs.copyFileSync(dbPath, backupName);
|
||||
}
|
||||
} catch (e) {
|
||||
logger.warn('Failed to create on-disk backup copy of current DB:', e.message);
|
||||
}
|
||||
|
||||
// Replace DB with the one from the zip
|
||||
fs.copyFileSync(tempDbPath, dbPath);
|
||||
|
||||
// Re-run migrations when needed
|
||||
if (check.backupMigration < check.requiredMigration) {
|
||||
await runMigrations();
|
||||
} else {
|
||||
// Ensure we can re-open the DB
|
||||
SqliteConnection.getConnection();
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
} catch (e) {
|
||||
logger.debug('Failed to cleanup temp restore dir:', e.message);
|
||||
}
|
||||
}
|
||||
|
||||
return { restored: true, warning: check.severity !== 'info' ? check.message : null, details: check };
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the backup file name with current date and Fredy version.
|
||||
* Pattern: YYYY-MM-DD-FredyBackup-{version}.zip
|
||||
* @returns {Promise<string>}
|
||||
*/
|
||||
export async function buildBackupFileName() {
|
||||
const dt = new Date();
|
||||
const yyyy = dt.getFullYear();
|
||||
const mm = String(dt.getMonth() + 1).padStart(2, '0');
|
||||
const dd = String(dt.getDate()).padStart(2, '0');
|
||||
const version = await getPackageVersion();
|
||||
return `${yyyy}-${mm}-${dd}-FredyBackup-${version}.zip`.replaceAll(' ', '');
|
||||
}
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { nanoid } from 'nanoid';
|
||||
import SqliteConnection from './SqliteConnection.js';
|
||||
import logger from '../logger.js';
|
||||
@@ -80,6 +85,7 @@ export const getJob = (jobId) => {
|
||||
j.name,
|
||||
j.blacklist,
|
||||
j.provider,
|
||||
j.shared_with_user,
|
||||
j.notification_adapter AS notificationAdapter,
|
||||
(SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id) AS numberOfFoundListings
|
||||
FROM jobs j
|
||||
@@ -93,6 +99,7 @@ export const getJob = (jobId) => {
|
||||
enabled: !!row.enabled,
|
||||
blacklist: fromJson(row.blacklist, []),
|
||||
provider: fromJson(row.provider, []),
|
||||
shared_with_user: fromJson(row.shared_with_user, []),
|
||||
notificationAdapter: fromJson(row.notificationAdapter, []),
|
||||
};
|
||||
};
|
||||
@@ -156,3 +163,109 @@ export const getJobs = () => {
|
||||
notificationAdapter: fromJson(row.notificationAdapter, []),
|
||||
}));
|
||||
};
|
||||
|
||||
/**
|
||||
* Query jobs with pagination, filtering and sorting.
|
||||
*
|
||||
* @param {Object} params
|
||||
* @param {number} [params.pageSize=50]
|
||||
* @param {number} [params.page=1]
|
||||
* @param {string} [params.freeTextFilter]
|
||||
* @param {object} [params.activityFilter]
|
||||
* @param {string|null} [params.sortField=null]
|
||||
* @param {('asc'|'desc')} [params.sortDir='asc']
|
||||
* @param {string} [params.userId] - Current user id used to scope jobs (ignored for admins).
|
||||
* @param {boolean} [params.isAdmin=false] - When true, returns all jobs.
|
||||
* @returns {{ totalNumber:number, page:number, result:Object[] }}
|
||||
*/
|
||||
export const queryJobs = ({
|
||||
pageSize = 50,
|
||||
page = 1,
|
||||
activityFilter,
|
||||
freeTextFilter,
|
||||
sortField = null,
|
||||
sortDir = 'asc',
|
||||
userId = null,
|
||||
isAdmin = false,
|
||||
} = {}) => {
|
||||
// sanitize inputs
|
||||
const safePageSize = Number.isFinite(pageSize) && pageSize > 0 ? Math.min(500, Math.floor(pageSize)) : 50;
|
||||
const safePage = Number.isFinite(page) && page > 0 ? Math.floor(page) : 1;
|
||||
const offset = (safePage - 1) * safePageSize;
|
||||
|
||||
// build WHERE filter
|
||||
const whereParts = [];
|
||||
const params = { limit: safePageSize, offset };
|
||||
params.userId = userId || '__NO_USER__';
|
||||
|
||||
if (!isAdmin) {
|
||||
whereParts.push(
|
||||
`(j.user_id = @userId OR EXISTS (SELECT 1 FROM json_each(j.shared_with_user) AS sw WHERE sw.value = @userId))`,
|
||||
);
|
||||
}
|
||||
|
||||
if (freeTextFilter && String(freeTextFilter).trim().length > 0) {
|
||||
params.filter = `%${String(freeTextFilter).trim()}%`;
|
||||
whereParts.push(`(j.name LIKE @filter)`);
|
||||
}
|
||||
|
||||
if (activityFilter === true) {
|
||||
whereParts.push('(j.enabled = 1)');
|
||||
} else if (activityFilter === false) {
|
||||
whereParts.push('(j.enabled = 0)');
|
||||
}
|
||||
|
||||
const whereSql = whereParts.length ? `WHERE ${whereParts.join(' AND ')}` : '';
|
||||
|
||||
// whitelist sortable fields
|
||||
const sortable = new Set(['name', 'numberOfFoundListings', 'enabled']);
|
||||
const safeSortField = sortField && sortable.has(sortField) ? sortField : null;
|
||||
const safeSortDir = String(sortDir).toLowerCase() === 'desc' ? 'DESC' : 'ASC';
|
||||
|
||||
let orderSql = 'ORDER BY j.name IS NULL, j.name ASC';
|
||||
if (safeSortField) {
|
||||
if (safeSortField === 'numberOfFoundListings') {
|
||||
orderSql = `ORDER BY numberOfFoundListings ${safeSortDir}`;
|
||||
} else {
|
||||
orderSql = `ORDER BY j.${safeSortField} ${safeSortDir}`;
|
||||
}
|
||||
}
|
||||
|
||||
// count total
|
||||
const countRow = SqliteConnection.query(
|
||||
`SELECT COUNT(1) as cnt
|
||||
FROM jobs j
|
||||
${whereSql}`,
|
||||
params,
|
||||
);
|
||||
const totalNumber = countRow?.[0]?.cnt ?? 0;
|
||||
|
||||
// fetch page
|
||||
const rows = SqliteConnection.query(
|
||||
`SELECT j.id,
|
||||
j.user_id AS userId,
|
||||
j.enabled,
|
||||
j.name,
|
||||
j.blacklist,
|
||||
j.provider,
|
||||
j.shared_with_user,
|
||||
j.notification_adapter AS notificationAdapter,
|
||||
(SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id) AS numberOfFoundListings
|
||||
FROM jobs j
|
||||
${whereSql}
|
||||
${orderSql}
|
||||
LIMIT @limit OFFSET @offset`,
|
||||
params,
|
||||
);
|
||||
|
||||
const result = rows.map((row) => ({
|
||||
...row,
|
||||
enabled: !!row.enabled,
|
||||
blacklist: fromJson(row.blacklist, []),
|
||||
provider: fromJson(row.provider, []),
|
||||
shared_with_user: fromJson(row.shared_with_user, []),
|
||||
notificationAdapter: fromJson(row.notificationAdapter, []),
|
||||
}));
|
||||
|
||||
return { totalNumber, page: safePage, result };
|
||||
};
|
||||
|
||||
@@ -1,41 +1,12 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { nullOrEmpty } from '../../utils.js';
|
||||
import SqliteConnection from './SqliteConnection.js';
|
||||
import { nanoid } from 'nanoid';
|
||||
|
||||
/**
|
||||
* Build analytics data for a given job by grouping all listings by provider and
|
||||
* mapping each listing hash to its creation timestamp.
|
||||
*
|
||||
* SQL shape:
|
||||
* SELECT json_group_object(provider, json_object(hash, created_at)) AS result
|
||||
* FROM listings WHERE job_id = @jobId;
|
||||
*
|
||||
* The resulting object has the shape:
|
||||
* {
|
||||
* providerA: { "<hash1>": <created_at_ms>, "<hash2>": <created_at_ms>, ... },
|
||||
* providerB: { ... }
|
||||
* }
|
||||
*
|
||||
* @param {string} jobId - ID of the job whose listings should be aggregated.
|
||||
* @returns {Record<string, Record<string, number>>} Object grouped by provider mapping listing-hash -> created_at epoch ms.
|
||||
*/
|
||||
export const getListingProviderDataForAnalytics = (jobId) => {
|
||||
const row = SqliteConnection.query(
|
||||
`SELECT COALESCE(
|
||||
json_group_object(provider, json(provider_map)),
|
||||
json('{}')
|
||||
) AS result
|
||||
FROM (SELECT provider,
|
||||
json_group_object(hash, created_at) AS provider_map
|
||||
FROM listings
|
||||
WHERE job_id = @jobId
|
||||
GROUP BY provider);`,
|
||||
{ jobId },
|
||||
);
|
||||
|
||||
return row?.length > 0 ? JSON.parse(row[0].result) : {};
|
||||
};
|
||||
|
||||
/**
|
||||
* Return a list of known listing hashes for a given job and provider.
|
||||
* Useful to de-duplicate before inserting new listings.
|
||||
@@ -54,6 +25,89 @@ export const getKnownListingHashesForJobAndProvider = (jobId, providerId) => {
|
||||
).map((r) => r.hash);
|
||||
};
|
||||
|
||||
/**
|
||||
* Compute KPI aggregates for a given set of job IDs from the listings table.
|
||||
*
|
||||
* - numberOfActiveListings: count of listings where is_active = 1
|
||||
* - avgPriceOfListings: average of numeric price, rounded to nearest integer
|
||||
*
|
||||
* When no jobIds are provided, returns zeros.
|
||||
*
|
||||
* @param {string[]} jobIds
|
||||
* @returns {{ numberOfActiveListings: number, avgPriceOfListings: number }}
|
||||
*/
|
||||
export const getListingsKpisForJobIds = (jobIds = []) => {
|
||||
if (!Array.isArray(jobIds) || jobIds.length === 0) {
|
||||
return { numberOfActiveListings: 0, avgPriceOfListings: 0 };
|
||||
}
|
||||
|
||||
const placeholders = jobIds.map(() => '?').join(',');
|
||||
const row =
|
||||
SqliteConnection.query(
|
||||
`SELECT
|
||||
SUM(CASE WHEN is_active = 1 THEN 1 ELSE 0 END) AS activeCount,
|
||||
AVG(price) AS avgPrice
|
||||
FROM listings
|
||||
WHERE job_id IN (${placeholders})`,
|
||||
jobIds,
|
||||
)[0] || {};
|
||||
|
||||
return {
|
||||
numberOfActiveListings: Number(row.activeCount || 0),
|
||||
avgPriceOfListings: row?.avgPrice == null ? 0 : Math.round(Number(row.avgPrice)),
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Compute distribution of listings by provider for the given set of job IDs.
|
||||
* Returns data ready for the pie chart component with fields `type` and `value` (percentage).
|
||||
*
|
||||
* Example return:
|
||||
* [ { type: 'immoscout', value: 62 }, { type: 'immowelt', value: 38 } ]
|
||||
*
|
||||
* When no jobIds are provided or no listings exist, returns empty array.
|
||||
*
|
||||
* @param {string[]} jobIds
|
||||
* @returns {{ type: string, value: number }[]}
|
||||
*/
|
||||
export const getProviderDistributionForJobIds = (jobIds = []) => {
|
||||
if (!Array.isArray(jobIds) || jobIds.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const placeholders = jobIds.map(() => '?').join(',');
|
||||
const rows = SqliteConnection.query(
|
||||
`SELECT provider, COUNT(*) AS cnt
|
||||
FROM listings
|
||||
WHERE job_id IN (${placeholders})
|
||||
GROUP BY provider
|
||||
ORDER BY cnt DESC`,
|
||||
jobIds,
|
||||
);
|
||||
|
||||
const total = rows.reduce((acc, r) => acc + Number(r.cnt || 0), 0);
|
||||
if (total === 0) return [];
|
||||
|
||||
// Map counts to integer percentage values (0-100). Ensure sum is ~100 by rounding.
|
||||
const percentages = rows.map((r) => ({
|
||||
type: r.provider,
|
||||
value: Math.round((Number(r.cnt) / total) * 100),
|
||||
}));
|
||||
|
||||
// Adjust rounding drift to keep sum at 100 (optional minor correction)
|
||||
const drift = 100 - percentages.reduce((s, p) => s + p.value, 0);
|
||||
if (drift !== 0 && percentages.length > 0) {
|
||||
// apply drift to the largest slice to keep UX simple
|
||||
let maxIdx = 0;
|
||||
for (let i = 1; i < percentages.length; i++) {
|
||||
if (percentages[i].value > percentages[maxIdx].value) maxIdx = i;
|
||||
}
|
||||
percentages[maxIdx].value = Math.max(0, percentages[maxIdx].value + drift);
|
||||
}
|
||||
|
||||
return percentages;
|
||||
};
|
||||
|
||||
/**
|
||||
* Return a list of listing that either are active or have an unknown status
|
||||
* to constantly check if they are still online
|
||||
@@ -119,9 +173,9 @@ export const storeListings = (jobId, providerId, listings) => {
|
||||
SqliteConnection.withTransaction((db) => {
|
||||
const stmt = db.prepare(
|
||||
`INSERT INTO listings (id, hash, provider, job_id, price, size, title, image_url, description, address,
|
||||
link, created_at, is_active)
|
||||
link, created_at, is_active, latitude, longitude)
|
||||
VALUES (@id, @hash, @provider, @job_id, @price, @size, @title, @image_url, @description, @address, @link,
|
||||
@created_at, 1)
|
||||
@created_at, 1, @latitude, @longitude)
|
||||
ON CONFLICT(job_id, hash) DO NOTHING`,
|
||||
);
|
||||
|
||||
@@ -139,6 +193,8 @@ export const storeListings = (jobId, providerId, listings) => {
|
||||
address: removeParentheses(item.address),
|
||||
link: item.link,
|
||||
created_at: Date.now(),
|
||||
latitude: item.latitude || null,
|
||||
longitude: item.longitude || null,
|
||||
};
|
||||
stmt.run(params);
|
||||
}
|
||||
@@ -223,9 +279,11 @@ export const queryListings = ({
|
||||
params.filter = `%${String(freeTextFilter).trim()}%`;
|
||||
whereParts.push(`(title LIKE @filter OR address LIKE @filter OR provider LIKE @filter OR link LIKE @filter)`);
|
||||
}
|
||||
// activityFilter: when true -> only active listings (is_active = 1)
|
||||
// activityFilter: when true -> only active listings (is_active = 1), false -> only inactive
|
||||
if (activityFilter === true) {
|
||||
whereParts.push('(is_active = 1)');
|
||||
} else if (activityFilter === false) {
|
||||
whereParts.push('(is_active = 0)');
|
||||
}
|
||||
// Prefer filtering by job id when provided (unambiguous and robust)
|
||||
if (jobIdFilter && String(jobIdFilter).trim().length > 0) {
|
||||
@@ -241,9 +299,11 @@ export const queryListings = ({
|
||||
params.providerName = String(providerFilter).trim();
|
||||
whereParts.push('(provider = @providerName)');
|
||||
}
|
||||
// watchListFilter: when true -> only watched listings
|
||||
// watchListFilter: when true -> only watched listings, false -> only unwatched
|
||||
if (watchListFilter === true) {
|
||||
whereParts.push('(wl.id IS NOT NULL)');
|
||||
} else if (watchListFilter === false) {
|
||||
whereParts.push('(wl.id IS NULL)');
|
||||
}
|
||||
|
||||
const whereSql = whereParts.length ? `WHERE ${whereParts.join(' AND ')}` : '';
|
||||
@@ -334,6 +394,84 @@ export const deleteListingsById = (ids) => {
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Return all listings that are active, have an address, and do not yet have geocoordinates.
|
||||
*
|
||||
* @returns {Object[]} Array of listing objects {id, address}.
|
||||
*/
|
||||
export const getListingsToGeocode = () => {
|
||||
return SqliteConnection.query(
|
||||
`SELECT id, address
|
||||
FROM listings
|
||||
WHERE is_active = 1
|
||||
AND address IS NOT NULL
|
||||
AND (latitude IS NULL OR longitude IS NULL)`,
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Update the geocoordinates for a listing.
|
||||
*
|
||||
* @param {string} id - The listing ID.
|
||||
* @param {number} latitude
|
||||
* @param {number} longitude
|
||||
* @returns {void}
|
||||
*/
|
||||
export const updateListingGeocoordinates = (id, latitude, longitude) => {
|
||||
SqliteConnection.execute(
|
||||
`UPDATE listings
|
||||
SET latitude = @latitude,
|
||||
longitude = @longitude
|
||||
WHERE id = @id`,
|
||||
{ id, latitude, longitude },
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Return listings with geocoordinates for the map view, with optional filtering.
|
||||
*
|
||||
* @param {Object} params
|
||||
* @param {string} [params.jobId]
|
||||
* @param {string} [params.userId]
|
||||
* @param {boolean} [params.isAdmin=false]
|
||||
* @returns {{listings: Object[], maxPrice: number}} Object containing listings and maxPrice.
|
||||
*/
|
||||
export const getListingsForMap = ({ jobId, userId = null, isAdmin = false } = {}) => {
|
||||
const baseWhereParts = [
|
||||
'l.latitude IS NOT NULL',
|
||||
'l.longitude IS NOT NULL',
|
||||
'l.latitude != -1',
|
||||
'l.longitude != -1',
|
||||
'l.is_active = 1',
|
||||
];
|
||||
const params = { userId: userId || '__NO_USER__' };
|
||||
|
||||
if (!isAdmin) {
|
||||
baseWhereParts.push(
|
||||
`(j.user_id = @userId OR EXISTS (SELECT 1 FROM json_each(j.shared_with_user) AS sw WHERE sw.value = @userId))`,
|
||||
);
|
||||
}
|
||||
|
||||
if (jobId) {
|
||||
params.jobId = jobId;
|
||||
baseWhereParts.push('l.job_id = @jobId');
|
||||
}
|
||||
|
||||
const wherePartsForListings = [...baseWhereParts];
|
||||
|
||||
const listings = SqliteConnection.query(
|
||||
`SELECT l.*, j.name AS job_name
|
||||
FROM listings l
|
||||
LEFT JOIN jobs j ON j.id = l.job_id
|
||||
WHERE ${wherePartsForListings.join(' AND ')}`,
|
||||
params,
|
||||
);
|
||||
|
||||
return {
|
||||
listings,
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Return all listings with only the fields: title, address, and price.
|
||||
* This is the single helper requested for simple consumers.
|
||||
@@ -343,3 +481,78 @@ export const deleteListingsById = (ids) => {
|
||||
export const getAllEntriesFromListings = () => {
|
||||
return SqliteConnection.query(`SELECT title, address, price FROM listings`);
|
||||
};
|
||||
|
||||
/**
|
||||
* Return geocoordinates for a given address if it has been geocoded before.
|
||||
*
|
||||
* @param {string} address
|
||||
* @returns {{lat: number, lng: number}|null}
|
||||
*/
|
||||
export const getGeocoordinatesByAddress = (address) => {
|
||||
const row = SqliteConnection.query(
|
||||
`SELECT latitude, longitude
|
||||
FROM listings
|
||||
WHERE address = @address
|
||||
AND latitude IS NOT NULL
|
||||
AND longitude IS NOT NULL
|
||||
AND latitude != -1
|
||||
AND longitude != -1
|
||||
LIMIT 1`,
|
||||
{ address },
|
||||
)[0];
|
||||
return row ? { lat: row.latitude, lng: row.longitude } : null;
|
||||
};
|
||||
|
||||
/**
|
||||
* Return all active listings for a given job that have geocoordinates but no distance set.
|
||||
*
|
||||
* @param {string} jobId
|
||||
* @returns {Object[]}
|
||||
*/
|
||||
export const getListingsToCalculateDistance = (jobId) => {
|
||||
return SqliteConnection.query(
|
||||
`SELECT id, latitude, longitude
|
||||
FROM listings
|
||||
WHERE job_id = @jobId
|
||||
AND is_active = 1
|
||||
AND latitude IS NOT NULL
|
||||
AND longitude IS NOT NULL
|
||||
AND distance_to_destination IS NULL`,
|
||||
{ jobId },
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Return all active listings for a given user (across all jobs) that have geocoordinates.
|
||||
*
|
||||
* @param {string} userId
|
||||
* @returns {Object[]}
|
||||
*/
|
||||
export const getListingsForUserToCalculateDistance = (userId) => {
|
||||
return SqliteConnection.query(
|
||||
`SELECT l.id, l.latitude, l.longitude
|
||||
FROM listings l
|
||||
JOIN jobs j ON l.job_id = j.id
|
||||
WHERE j.user_id = @userId
|
||||
AND l.is_active = 1
|
||||
AND l.latitude IS NOT NULL
|
||||
AND l.longitude IS NOT NULL`,
|
||||
{ userId },
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Update the distance to destination for a listing.
|
||||
*
|
||||
* @param {string} id
|
||||
* @param {number} distance
|
||||
* @returns {void}
|
||||
*/
|
||||
export const updateListingDistance = (id, distance) => {
|
||||
SqliteConnection.execute(
|
||||
`UPDATE listings
|
||||
SET distance_to_destination = @distance
|
||||
WHERE id = @id`,
|
||||
{ id, distance },
|
||||
);
|
||||
};
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
/**
|
||||
* Migration Runner for better-sqlite3
|
||||
* I know there are external libs out there, but
|
||||
@@ -30,7 +35,11 @@ import SqliteConnection from '../SqliteConnection.js';
|
||||
import logger from '../../logger.js';
|
||||
|
||||
const ROOT = path.resolve('.');
|
||||
const MIGRATIONS_DIR = path.join(ROOT, 'lib', 'services', 'storage', 'migrations', 'sql');
|
||||
/**
|
||||
* Absolute path to the migrations directory (lib/services/storage/migrations/sql).
|
||||
* @type {string}
|
||||
*/
|
||||
export const MIGRATIONS_DIR = path.join(ROOT, 'lib', 'services', 'storage', 'migrations', 'sql');
|
||||
|
||||
/**
|
||||
* Ensures that the given directory exists, creating it recursively if needed.
|
||||
@@ -45,7 +54,7 @@ function ensureDir(p) {
|
||||
* Migration files must follow the format: <number>.<label>.js
|
||||
* @returns {Array<{id:number, name:string, label:string, path:string}>}
|
||||
*/
|
||||
function listMigrationFiles() {
|
||||
export function listMigrationFiles() {
|
||||
ensureDir(MIGRATIONS_DIR);
|
||||
return fs
|
||||
.readdirSync(MIGRATIONS_DIR)
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
// Initial migration: creates schema_migrations table used by the migration runner.
|
||||
//
|
||||
export function up(db) {
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
// Migration: Create fredy's base structure (users, jobs and listings) import initial
|
||||
// data from JSON files if present. (This applies only for jobs and users, for the old jobListingData,
|
||||
// I cannot migrate the data as the new format is totally different.
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
// Migration: there needs to be a unique index on job_id and hash as only
|
||||
// this makes the listing indeed unique
|
||||
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
// Migration: there needs to be a unique index on job_id and hash as only
|
||||
// this makes the listing indeed unique
|
||||
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
// Migration: Adding a changeset field to the listings table in preparation for
|
||||
// a price watch feature
|
||||
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
// Migration: Adding a new table to store if somebody "watches" (a.k.a favorite) a listing
|
||||
|
||||
export function up(db) {
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
// Migration: Adding a new table to store if somebody shared a job with someone
|
||||
|
||||
export function up(db) {
|
||||
|
||||
@@ -1,8 +1,15 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
// Migration: Adding a settings table to store important (config) settings instead of using config file
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { nanoid } from 'nanoid';
|
||||
import logger from '../../../logger.js';
|
||||
import { DEFAULT_CONFIG } from '../../../../defaultConfig.js';
|
||||
import { getDirName } from '../../../../utils.js';
|
||||
|
||||
export function up(db) {
|
||||
db.exec(`
|
||||
@@ -67,6 +74,10 @@ export function up(db) {
|
||||
'analyticsEnabled',
|
||||
config.analyticsEnabled != null ? config.analyticsEnabled : defaults.analyticsEnabled,
|
||||
);
|
||||
|
||||
//now making sure only sqlite path remains in the config
|
||||
const sqlitepath = config.sqlitepath || DEFAULT_CONFIG.sqlitepath;
|
||||
fs.writeFileSync(`${getDirName()}/../conf/config.json`, JSON.stringify({ sqlitepath }));
|
||||
} catch (e) {
|
||||
logger.error(e);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
// Migration: Add geocoordinates to listings for map display
|
||||
|
||||
export function up(db) {
|
||||
db.exec(`
|
||||
ALTER TABLE listings ADD COLUMN latitude REAL;
|
||||
ALTER TABLE listings ADD COLUMN longitude REAL;
|
||||
`);
|
||||
}
|
||||
12
lib/services/storage/migrations/sql/8.distances.js
Normal file
12
lib/services/storage/migrations/sql/8.distances.js
Normal file
@@ -0,0 +1,12 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
// Migration: Removing city field and adding distance field
|
||||
|
||||
export function up(db) {
|
||||
db.exec(`
|
||||
ALTER TABLE listings ADD COLUMN distance_to_destination INTEGER;
|
||||
`);
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user