Migrate to SQLite (#174)

* Migrating Fredy from LowDb to SqLite 🎉

* adding new sql migration system for future sql migrations

* adding setting to change  sqlite path for db files

* create migration plan for graceful migration lowdb -> sqlite

* Improving Documentation

* adding test for sqliteconnection

* upgrading dependencies

* making nodejs 22 as min version

* improve scraper

* adding overwrite ability for db migra

lib/provider/einsAImmobilien.js

@@ -7,7 +7,8 @@ function normalize(o) {
   const price = normalizePrice(o.price);
   const id = buildHash(o.id, price);
   const image = baseUrl + o.image;
-  return Object.assign(o, { id, price, link, image });
+  const address = o.address == null ? null : o.address.trim().replaceAll('/', ',');
+  return Object.assign(o, { id, price, link, image, address });
 }
 
 /**
@@ -44,6 +45,7 @@ const config = {
     size: '.tabelle .tabelle_inhalt_infos .single_data_box  | removeNewline | trim',
     title: '.inner_object_data .tabelle_inhalt_titel_black | removeNewline | trim',
     image: '.inner_object_pic img@src',
+    address: '.tabelle .tabelle_inhalt_infos .left_information > div:nth-child(2) | removeNewline | trim',
   },
   normalize: normalize,
   filter: applyBlacklist,

lib/provider/immobilienDe.js

@@ -12,10 +12,10 @@ function parseId(shortenedLink) {
 
 function normalize(o) {
   const baseUrl = 'https://www.immobilien.de';
-  const size = o.size || 'N/A m²';
-  const price = o.price || 'N/A €';
+  const size = o.size || null;
+  const price = o.price || null;
   const title = o.title || 'No title available';
-  const address = o.address || 'No address available';
+  const address = o.address || null;
   const shortLink = shortenLink(o.link);
   const link = `${baseUrl}/${shortLink}`;
   const image = baseUrl + o.image;

lib/provider/immonet.js

@@ -1,19 +1,12 @@
 import utils, { buildHash } from '../utils.js';
 let appliedBlackList = [];
 
-/**
- * Note, Immonet is rly a piece of sh*t. It is using a weird combination of React and some buttons (instead of links),
- * so that if somebody clicks the listing, a new page will open with the actual link to the listing. Of course, a scraper
- * cannot do this (which is why I always just return the link to the whole list of listings).
- * This is not only bad for us, but also bad for ppl with disabilities...
- */
-
 function normalize(o) {
   const size = o.size != null ? o.size.replace('Wohnfläche ', '') : 'N/A m²';
   const price = o.price.replace('Kaufpreis ', '');
   const address = o.address?.split(' • ')?.pop() ?? null;
   const title = o.title || 'No title available';
-  const link = config.url;
+  const link = o.link != null ? decodeURIComponent(o.link) : config.url;
   const id = buildHash(title, price);
   return Object.assign(o, { id, address, price, size, title, link });
 }
@@ -28,12 +21,13 @@ const config = {
   sortByDateParam: 'sortby=19',
   waitForSelector: 'div[data-testid="serp-gridcontainer-testid"]',
   crawlFields: {
-    id: 'button@title |trim', // immonet is a piece of sh*t. See comment above
+    id: 'button@title |trim',
     title: 'button@title |trim',
     price: 'div[data-testid="cardmfe-price-testid"] | trim',
     size: 'div[data-testid="cardmfe-keyfacts-testid"] | trim',
     address: 'div[data-testid="cardmfe-description-box-address"] | trim',
     image: 'div[data-testid="cardmfe-picture-box-test-id"] img@src',
+    link: 'button@data-base',
   },
   normalize: normalize,
   filter: applyBlacklist,

lib/provider/immoscout.js

@@ -69,6 +69,7 @@ async function getListings(url) {
         price: price?.value,
         size: size?.value,
         title: item.title,
+        description: item.description,
         link: `${metaInformation.baseUrl}expose/${item.id}`,
         address: item.address?.line,
         image,