Compare commits

..

5 Commits
7.3.2 ... 7.4.1

Author SHA1 Message Date
Christian Kellner
37948be0d3 next build version 2023-10-26 12:47:14 +02:00
Christian Kellner
cc7bbb77c4 removing sqlite as it only generates build errors 2023-10-26 12:46:42 +02:00
Christian Kellner
96da0b7892 Update LICENSE 2023-10-05 18:39:16 +02:00
jstnw
72993312c7 fix: kleinanzeigen price (#82) 2023-10-05 18:33:55 +02:00
weakmap@gmail.com
17b4bad2e4 fixing notification provider 2023-09-27 17:45:38 +02:00
12 changed files with 1127 additions and 1400 deletions

View File

@@ -1,6 +1,6 @@
MIT License
Copyright (c) 2021 Christian Kellner
Copyright (c) 2023 Christian Kellner
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal

View File

@@ -9,7 +9,7 @@ const template = fs.readFileSync(path.resolve(__dirname + '/notification/emailTe
const emailTemplate = Handlebars.compile(template);
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
const { apiPublicKey, apiPrivateKey, receiver, from } = notificationConfig.find(
(adapter) => adapter.id === 'mailjet'
(adapter) => adapter.id === config.id,
).fields;
const to = receiver
.trim()

View File

@@ -2,13 +2,13 @@ import { markdown2Html } from '../../services/markdown.js';
import { getJob } from '../../services/storage/jobStorage.js';
import fetch from 'node-fetch';
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
const { webhook, channel } = notificationConfig.find((adapter) => adapter.id === 'mattermost').fields;
const { webhook, channel } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
const job = getJob(jobKey);
const jobName = job == null ? jobKey : job.name;
let message = `### *${jobName}* (${serviceName}) found **${newListings.length}** new listings:\n\n`;
message += `| Title | Address | Size | Price |\n|:----|:----|:----|:----|\n`;
message += newListings.map(
(o) => `| [${o.title}](${o.link}) | ` + [o.address, o.size.replace(/2m/g, '$m^2$'), o.price].join(' | ') + ' |\n'
(o) => `| [${o.title}](${o.link}) | ` + [o.address, o.size.replace(/2m/g, '$m^2$'), o.price].join(' | ') + ' |\n',
);
return fetch(webhook, {
method: 'POST',

View File

@@ -3,7 +3,7 @@ import { getJob } from '../../services/storage/jobStorage.js';
import fetch from 'node-fetch';
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
const { priority, server, topic } = notificationConfig.find((adapter) => adapter.id === 'ntfy').fields;
const { priority, server, topic } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
const job = getJob(jobKey);
const jobName = job == null ? jobKey : job.name;
const promises = newListings.map((newListing) => {

View File

@@ -1,7 +1,7 @@
import sgMail from '@sendgrid/mail';
import { markdown2Html } from '../../services/markdown.js';
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
const { apiKey, receiver, from, templateId } = notificationConfig.find((adapter) => adapter.id === 'sendGrid').fields;
const { apiKey, receiver, from, templateId } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
sgMail.setApiKey(apiKey);
const msg = {
templateId,

View File

@@ -2,7 +2,7 @@ import Slack from 'slack';
import { markdown2Html } from '../../services/markdown.js';
const msg = Slack.chat.postMessage;
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
const { token, channel } = notificationConfig.find((adapter) => adapter.id === 'slack').fields;
const { token, channel } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
return newListings.map((payload) =>
msg({
token,
@@ -35,7 +35,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
ts: new Date().getTime() / 1000,
},
],
})
}),
);
};
export const config = {

View File

@@ -1,25 +0,0 @@
import { markdown2Html } from '../../services/markdown.js';
import Database from 'better-sqlite3';
export const send = ({ serviceName, newListings, jobKey }) => {
const db = new Database('db/listings.db');
const fields = ['serviceName', 'jobKey', 'id', 'size', 'rooms', 'price', 'address', 'title', 'link', 'description'];
db.prepare(`CREATE TABLE IF NOT EXISTS listing (${fields.join(' TEXT, ')} TEXT);`).run();
const insert = db.prepare(`INSERT INTO listing (${fields.join(', ')}) VALUES (@${fields.join(', @')})`);
newListings.map((listing) => {
let insertListing = {};
fields.map((field) => {
insertListing[field] = listing[field];
});
insertListing.serviceName = serviceName;
insertListing.jobKey = jobKey;
insert.run(insertListing);
});
return Promise.resolve();
};
export const config = {
id: 'sqlite',
name: 'Sqlite',
description: 'This adapter stores listings in a local sqlite3 database.',
config: {},
readme: markdown2Html('lib/notification/adapter/sqlite.md'),
};

View File

@@ -1,3 +0,0 @@
### Sqlite Adapter
This adapter stores search results in an sqlite database in db/listings.db

View File

@@ -19,7 +19,7 @@ function shorten(str, len = 30) {
return str.length > len ? str.substring(0, len) + '...' : str;
}
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
const { token, chatId } = notificationConfig.find((adapter) => adapter.id === 'telegram').fields;
const { token, chatId } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
const job = getJob(jobKey);
const jobName = job == null ? jobKey : job.name;
//we have to split messages into chunk, because otherwise messages are going to become too big and will fail
@@ -30,7 +30,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
(o) =>
`<a href='${o.link}'><b>${shorten(o.title.replace(/\*/g, ''), 45).trim()}</b></a>\n` +
[o.address, o.price, o.size].join(' | ') +
'\n\n'
'\n\n',
);
/**
* This is to not break the rate limit. It is to only send 1 message per second

View File

@@ -19,7 +19,7 @@ const config = {
sortByDateParam: null,
crawlFields: {
id: '.aditem@data-adid | int',
price: '.aditem-main--middle--price | removeNewline | trim',
price: '.aditem-main--middle--price-shipping--price | removeNewline | trim',
size: '.aditem-main .text-module-end span:nth-child(2) | removeNewline | trim',
title: '.aditem-main .text-module-begin a | removeNewline | trim',
link: '.aditem-main .text-module-begin a@href | removeNewline | trim',

View File

@@ -1,6 +1,6 @@
{
"name": "fredy",
"version": "7.3.2",
"version": "7.4.1",
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
"scripts": {
"start": "node index.js",
@@ -55,12 +55,11 @@
"Firefox ESR"
],
"dependencies": {
"@douyinfe/semi-ui": "2.42.4",
"@douyinfe/semi-ui": "2.45.2",
"@rematch/core": "2.2.0",
"@rematch/loading": "2.1.2",
"@sendgrid/mail": "7.7.0",
"@vitejs/plugin-react": "4.0.4",
"better-sqlite3": "8.6.0",
"@vitejs/plugin-react": "4.1.0",
"body-parser": "1.20.2",
"cookie-session": "2.0.0",
"handlebars": "4.7.8",
@@ -75,7 +74,7 @@
"query-string": "8.1.0",
"react": "18.2.0",
"react-dom": "18.2.0",
"react-redux": "8.1.2",
"react-redux": "8.1.3",
"react-router": "5.2.1",
"react-router-dom": "5.3.0",
"redux": "4.2.1",
@@ -84,25 +83,25 @@
"serve-static": "1.15.0",
"slack": "11.0.2",
"string-similarity": "^4.0.4",
"vite": "4.4.9",
"vite": "4.5.0",
"x-ray": "2.3.4"
},
"devDependencies": {
"@babel/core": "7.22.15",
"@babel/core": "7.23.2",
"@babel/eslint-parser": "7.22.15",
"@babel/preset-env": "7.22.15",
"@babel/preset-env": "7.23.2",
"@babel/preset-react": "7.22.15",
"chai": "4.3.8",
"eslint": "8.48.0",
"chai": "4.3.10",
"eslint": "8.52.0",
"eslint-config-prettier": "8.8.0",
"eslint-plugin-react": "7.33.2",
"esmock": "2.4.0",
"esmock": "2.5.8",
"history": "5.3.0",
"husky": "4.3.8",
"less": "4.2.0",
"lint-staged": "13.2.2",
"mocha": "10.2.0",
"prettier": "2.8.8",
"prettier": "3.0.3",
"redux-logger": "3.0.6"
}
}

2454
yarn.lock

File diff suppressed because it is too large Load Diff