Compare commits

...

5 Commits
7.3.2 ... 7.4.1

Author SHA1 Message Date
Christian Kellner
37948be0d3 next build version 2023-10-26 12:47:14 +02:00
Christian Kellner
cc7bbb77c4 removing sqlite as it only generates build errors 2023-10-26 12:46:42 +02:00
Christian Kellner
96da0b7892 Update LICENSE 2023-10-05 18:39:16 +02:00
jstnw
72993312c7 fix: kleinanzeigen price (#82) 2023-10-05 18:33:55 +02:00
weakmap@gmail.com
17b4bad2e4 fixing notification provider 2023-09-27 17:45:38 +02:00
12 changed files with 1127 additions and 1400 deletions

View File

@@ -1,6 +1,6 @@
MIT License MIT License
Copyright (c) 2021 Christian Kellner Copyright (c) 2023 Christian Kellner
Permission is hereby granted, free of charge, to any person obtaining a copy Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal of this software and associated documentation files (the "Software"), to deal

View File

@@ -9,7 +9,7 @@ const template = fs.readFileSync(path.resolve(__dirname + '/notification/emailTe
const emailTemplate = Handlebars.compile(template); const emailTemplate = Handlebars.compile(template);
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => { export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
const { apiPublicKey, apiPrivateKey, receiver, from } = notificationConfig.find( const { apiPublicKey, apiPrivateKey, receiver, from } = notificationConfig.find(
(adapter) => adapter.id === 'mailjet' (adapter) => adapter.id === config.id,
).fields; ).fields;
const to = receiver const to = receiver
.trim() .trim()

View File

@@ -2,13 +2,13 @@ import { markdown2Html } from '../../services/markdown.js';
import { getJob } from '../../services/storage/jobStorage.js'; import { getJob } from '../../services/storage/jobStorage.js';
import fetch from 'node-fetch'; import fetch from 'node-fetch';
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => { export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
const { webhook, channel } = notificationConfig.find((adapter) => adapter.id === 'mattermost').fields; const { webhook, channel } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
const job = getJob(jobKey); const job = getJob(jobKey);
const jobName = job == null ? jobKey : job.name; const jobName = job == null ? jobKey : job.name;
let message = `### *${jobName}* (${serviceName}) found **${newListings.length}** new listings:\n\n`; let message = `### *${jobName}* (${serviceName}) found **${newListings.length}** new listings:\n\n`;
message += `| Title | Address | Size | Price |\n|:----|:----|:----|:----|\n`; message += `| Title | Address | Size | Price |\n|:----|:----|:----|:----|\n`;
message += newListings.map( message += newListings.map(
(o) => `| [${o.title}](${o.link}) | ` + [o.address, o.size.replace(/2m/g, '$m^2$'), o.price].join(' | ') + ' |\n' (o) => `| [${o.title}](${o.link}) | ` + [o.address, o.size.replace(/2m/g, '$m^2$'), o.price].join(' | ') + ' |\n',
); );
return fetch(webhook, { return fetch(webhook, {
method: 'POST', method: 'POST',

View File

@@ -3,7 +3,7 @@ import { getJob } from '../../services/storage/jobStorage.js';
import fetch from 'node-fetch'; import fetch from 'node-fetch';
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => { export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
const { priority, server, topic } = notificationConfig.find((adapter) => adapter.id === 'ntfy').fields; const { priority, server, topic } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
const job = getJob(jobKey); const job = getJob(jobKey);
const jobName = job == null ? jobKey : job.name; const jobName = job == null ? jobKey : job.name;
const promises = newListings.map((newListing) => { const promises = newListings.map((newListing) => {

View File

@@ -1,7 +1,7 @@
import sgMail from '@sendgrid/mail'; import sgMail from '@sendgrid/mail';
import { markdown2Html } from '../../services/markdown.js'; import { markdown2Html } from '../../services/markdown.js';
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => { export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
const { apiKey, receiver, from, templateId } = notificationConfig.find((adapter) => adapter.id === 'sendGrid').fields; const { apiKey, receiver, from, templateId } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
sgMail.setApiKey(apiKey); sgMail.setApiKey(apiKey);
const msg = { const msg = {
templateId, templateId,

View File

@@ -2,7 +2,7 @@ import Slack from 'slack';
import { markdown2Html } from '../../services/markdown.js'; import { markdown2Html } from '../../services/markdown.js';
const msg = Slack.chat.postMessage; const msg = Slack.chat.postMessage;
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => { export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
const { token, channel } = notificationConfig.find((adapter) => adapter.id === 'slack').fields; const { token, channel } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
return newListings.map((payload) => return newListings.map((payload) =>
msg({ msg({
token, token,
@@ -35,7 +35,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
ts: new Date().getTime() / 1000, ts: new Date().getTime() / 1000,
}, },
], ],
}) }),
); );
}; };
export const config = { export const config = {

View File

@@ -1,25 +0,0 @@
import { markdown2Html } from '../../services/markdown.js';
import Database from 'better-sqlite3';
export const send = ({ serviceName, newListings, jobKey }) => {
const db = new Database('db/listings.db');
const fields = ['serviceName', 'jobKey', 'id', 'size', 'rooms', 'price', 'address', 'title', 'link', 'description'];
db.prepare(`CREATE TABLE IF NOT EXISTS listing (${fields.join(' TEXT, ')} TEXT);`).run();
const insert = db.prepare(`INSERT INTO listing (${fields.join(', ')}) VALUES (@${fields.join(', @')})`);
newListings.map((listing) => {
let insertListing = {};
fields.map((field) => {
insertListing[field] = listing[field];
});
insertListing.serviceName = serviceName;
insertListing.jobKey = jobKey;
insert.run(insertListing);
});
return Promise.resolve();
};
export const config = {
id: 'sqlite',
name: 'Sqlite',
description: 'This adapter stores listings in a local sqlite3 database.',
config: {},
readme: markdown2Html('lib/notification/adapter/sqlite.md'),
};

View File

@@ -1,3 +0,0 @@
### Sqlite Adapter
This adapter stores search results in an sqlite database in db/listings.db

View File

@@ -19,7 +19,7 @@ function shorten(str, len = 30) {
return str.length > len ? str.substring(0, len) + '...' : str; return str.length > len ? str.substring(0, len) + '...' : str;
} }
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => { export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
const { token, chatId } = notificationConfig.find((adapter) => adapter.id === 'telegram').fields; const { token, chatId } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
const job = getJob(jobKey); const job = getJob(jobKey);
const jobName = job == null ? jobKey : job.name; const jobName = job == null ? jobKey : job.name;
//we have to split messages into chunk, because otherwise messages are going to become too big and will fail //we have to split messages into chunk, because otherwise messages are going to become too big and will fail
@@ -30,7 +30,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
(o) => (o) =>
`<a href='${o.link}'><b>${shorten(o.title.replace(/\*/g, ''), 45).trim()}</b></a>\n` + `<a href='${o.link}'><b>${shorten(o.title.replace(/\*/g, ''), 45).trim()}</b></a>\n` +
[o.address, o.price, o.size].join(' | ') + [o.address, o.price, o.size].join(' | ') +
'\n\n' '\n\n',
); );
/** /**
* This is to not break the rate limit. It is to only send 1 message per second * This is to not break the rate limit. It is to only send 1 message per second

View File

@@ -19,7 +19,7 @@ const config = {
sortByDateParam: null, sortByDateParam: null,
crawlFields: { crawlFields: {
id: '.aditem@data-adid | int', id: '.aditem@data-adid | int',
price: '.aditem-main--middle--price | removeNewline | trim', price: '.aditem-main--middle--price-shipping--price | removeNewline | trim',
size: '.aditem-main .text-module-end span:nth-child(2) | removeNewline | trim', size: '.aditem-main .text-module-end span:nth-child(2) | removeNewline | trim',
title: '.aditem-main .text-module-begin a | removeNewline | trim', title: '.aditem-main .text-module-begin a | removeNewline | trim',
link: '.aditem-main .text-module-begin a@href | removeNewline | trim', link: '.aditem-main .text-module-begin a@href | removeNewline | trim',

View File

@@ -1,6 +1,6 @@
{ {
"name": "fredy", "name": "fredy",
"version": "7.3.2", "version": "7.4.1",
"description": "[F]ind [R]eal [E]states [d]amn eas[y].", "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
"scripts": { "scripts": {
"start": "node index.js", "start": "node index.js",
@@ -55,12 +55,11 @@
"Firefox ESR" "Firefox ESR"
], ],
"dependencies": { "dependencies": {
"@douyinfe/semi-ui": "2.42.4", "@douyinfe/semi-ui": "2.45.2",
"@rematch/core": "2.2.0", "@rematch/core": "2.2.0",
"@rematch/loading": "2.1.2", "@rematch/loading": "2.1.2",
"@sendgrid/mail": "7.7.0", "@sendgrid/mail": "7.7.0",
"@vitejs/plugin-react": "4.0.4", "@vitejs/plugin-react": "4.1.0",
"better-sqlite3": "8.6.0",
"body-parser": "1.20.2", "body-parser": "1.20.2",
"cookie-session": "2.0.0", "cookie-session": "2.0.0",
"handlebars": "4.7.8", "handlebars": "4.7.8",
@@ -75,7 +74,7 @@
"query-string": "8.1.0", "query-string": "8.1.0",
"react": "18.2.0", "react": "18.2.0",
"react-dom": "18.2.0", "react-dom": "18.2.0",
"react-redux": "8.1.2", "react-redux": "8.1.3",
"react-router": "5.2.1", "react-router": "5.2.1",
"react-router-dom": "5.3.0", "react-router-dom": "5.3.0",
"redux": "4.2.1", "redux": "4.2.1",
@@ -84,25 +83,25 @@
"serve-static": "1.15.0", "serve-static": "1.15.0",
"slack": "11.0.2", "slack": "11.0.2",
"string-similarity": "^4.0.4", "string-similarity": "^4.0.4",
"vite": "4.4.9", "vite": "4.5.0",
"x-ray": "2.3.4" "x-ray": "2.3.4"
}, },
"devDependencies": { "devDependencies": {
"@babel/core": "7.22.15", "@babel/core": "7.23.2",
"@babel/eslint-parser": "7.22.15", "@babel/eslint-parser": "7.22.15",
"@babel/preset-env": "7.22.15", "@babel/preset-env": "7.23.2",
"@babel/preset-react": "7.22.15", "@babel/preset-react": "7.22.15",
"chai": "4.3.8", "chai": "4.3.10",
"eslint": "8.48.0", "eslint": "8.52.0",
"eslint-config-prettier": "8.8.0", "eslint-config-prettier": "8.8.0",
"eslint-plugin-react": "7.33.2", "eslint-plugin-react": "7.33.2",
"esmock": "2.4.0", "esmock": "2.5.8",
"history": "5.3.0", "history": "5.3.0",
"husky": "4.3.8", "husky": "4.3.8",
"less": "4.2.0", "less": "4.2.0",
"lint-staged": "13.2.2", "lint-staged": "13.2.2",
"mocha": "10.2.0", "mocha": "10.2.0",
"prettier": "2.8.8", "prettier": "3.0.3",
"redux-logger": "3.0.6" "redux-logger": "3.0.6"
} }
} }

2454
yarn.lock

File diff suppressed because it is too large Load Diff