Adding images to scraping data (#157)

* Fredy now supports pulling the main Image from the listing and send it together with the usual information
This commit is contained in:
Christian Kellner
2025-08-30 21:21:34 +02:00
committed by GitHub
parent da743c8279
commit f0b146fd7f
26 changed files with 1141 additions and 1135 deletions

View File

@@ -2,50 +2,55 @@ import { markdown2Html } from '../../services/markdown.js';
import { getJob } from '../../services/storage/jobStorage.js';
import fetch from 'node-fetch';
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
export const send = async ({ serviceName, newListings, notificationConfig, jobKey }) => {
const { token, user, device } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
const job = getJob(jobKey);
const jobName = job == null ? jobKey : job.name;
const promises = newListings.map((newListing) => {
const title = `${jobName} at ${serviceName}: ${newListing.title}`;
const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nLink: ${newListing.link}`;
return fetch('https://api.pushover.net/1/messages.json', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
token: token,
user: user,
message: message,
device: device,
title: title,
}),
});
});
return Promise.all(promises)
.then((responses) => {
// Convert all responses to JSON
return Promise.all(responses.map((response) => response.json()));
})
.then((data) => {
// Check for errors in the data
const error = data
.map((item) => (item.errors != null && item.errors.length > 0 ? item.errors.join(', ') : null))
.filter((err) => err !== null);
const results = await Promise.all(
newListings.map(async (newListing) => {
const title = `${jobName} at ${serviceName}: ${newListing.title}`;
const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nLink: ${newListing.link}`;
if (error.length > 0) {
// Reject with the combined error messages
return Promise.reject(error.join('; '));
const form = new FormData();
form.append('token', token);
form.append('user', user);
form.append('title', title);
form.append('message', message);
if (device) form.append('device', device);
// Try to attach image if available
if (newListing.image && typeof newListing.image === 'string') {
try {
const imgRes = await fetch(newListing.image);
if (imgRes.ok) {
const ab = await imgRes.arrayBuffer();
form.append('attachment', new Blob([ab]), 'image.jpg');
}
} catch {
// fail silently, just skip the image
}
}
return data;
})
.then(() => {
return Promise.resolve();
})
.catch((error) => {
return Promise.reject(error);
});
const res = await fetch('https://api.pushover.net/1/messages.json', {
method: 'POST',
body: form,
});
return res.json();
}),
);
// Collect errors
const errors = results
.map((r) => (r.errors && r.errors.length > 0 ? r.errors.join(', ') : null))
.filter((e) => e !== null);
if (errors.length > 0) {
return Promise.reject(errors.join('; '));
}
return results;
};
export const config = {