fixing immoswp

Dockerfile

@@ -1,19 +1,20 @@
-# syntax=docker/dockerfile:1.3
+FROM node:18
-FROM node:18-alpine AS builder
+
-COPY --chown=1000:1000 . /fredy
 WORKDIR  /fredy
-USER 1000
+
+COPY . /fredy
+
 RUN yarn install
+
+RUN yarn global add pm2
+
 RUN yarn run prod
 
-FROM node:16-alpine
-COPY --from=builder --chown=1000:1000 /fredy /fredy
 RUN mkdir /db /conf && \
   chown 1000:1000 /db /conf && \
   chmod 777 -R /db/ && \
   ln -s /db /fredy/db && ln -s /conf /fredy/conf
+
 EXPOSE 9998
-USER 1000
+
-VOLUME [ "/conf", "/db" ]
+CMD pm2-runtime index.js
-WORKDIR  /fredy
-CMD node index.js --no-daemon

docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3.3'
+version: '3.8'
 services:
   fredy:
     container_name: fredy
@@ -13,3 +13,4 @@ services:
       - ./db:/db
     ports:
         - 9998:9998
+    restart: unless-stopped

lib/provider/immoswp.js

@@ -1,44 +1,47 @@
 import utils from '../utils.js';
+
 let appliedBlackList = [];
+
 function normalize(o) {
-  const id = o.id.substring(o.id.indexOf('-') + 1, o.id.length);
+    const id = o.id.substring(o.id.indexOf('-') + 1, o.id.length);
-  const size = o.size || 'N/A m²';
+    const size = o.size || 'N/A m²';
-  const price = (o.price || '--- €').replace('Preis auf Anfrage', '--- €');
+    const price = (o.price || '--- €').replace('Preis auf Anfrage', '--- €');
-  const address = o.address || 'No address available';
+    const title = o.title || 'No title available';
-  const title = o.title || 'No title available';
+    const link = `https://immo.swp.de/immobilien/${id}`;
-  const link = `https://immo.swp.de/immobilien/${id}`;
+    const description = o.description;
-  const description = o.description;
+    return Object.assign(o, {id, price, size, title, link, description});
-  return Object.assign(o, { id, address, price, size, title, link, description });
 }
+
 function applyBlacklist(o) {
-  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
+    const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
-  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
+    const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-  return titleNotBlacklisted && descNotBlacklisted;
+    return titleNotBlacklisted && descNotBlacklisted;
 }
+
 const config = {
-  url: null,
+    url: null,
-  crawlContainer: '.js-serp-item',
+    crawlContainer: '.js-serp-item',
-  sortByDateParam: 's=most_recently_updated_first',
+    sortByDateParam: 's=most_recently_updated_first',
-  crawlFields: {
+    crawlFields: {
-    id: '@id',
+        id: '.js-bookmark-btn@data-id',
-    price: 'div.item__spec.item-spec-price | trim',
+        price: 'div.align-items-start div:first-child | trim',
-    size: 'div.item__spec.item-spec-area | trim',
+        size: 'div.align-items-start div:nth-child(3) | trim',
-    title: 'a.js-item-title-link@title',
+        title: '.card-title h2 | trim',
-    address: 'div.item__locality | removeNewline | trim',
+        link: '.ci-search-result__link@href',
-    description: 'div.item__main-info-points.clearfix p small | removeNewline | trim',
+        description: '.js-show-more-item-sm | removeNewline | trim',
-  },
+    },
-  paginate: 'li.page-item.pagination__item a.page-link@href',
+    paginate: 'li.page-item.pagination__item a.page-link@href',
-  normalize: normalize,
+    normalize: normalize,
-  filter: applyBlacklist,
+    filter: applyBlacklist,
 };
 export const init = (sourceConfig, blacklist) => {
-  config.enabled = sourceConfig.enabled;
+    config.enabled = sourceConfig.enabled;
-  config.url = sourceConfig.url;
+    config.url = sourceConfig.url;
-  appliedBlackList = blacklist || [];
+    appliedBlackList = blacklist || [];
 };
 export const metaInformation = {
-  name: 'Immo Südwest Presse',
+    name: 'Immo Südwest Presse',
-  baseUrl: 'https://immo.swp.de/',
+    baseUrl: 'https://immo.swp.de/',
-  id: 'immoswp',
+    id: 'immoswp',
 };
-export { config };
+export {config};

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "8.0.5",
+  "version": "8.0.6",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "start": "node index.js",
@@ -55,54 +55,54 @@
     "Firefox ESR"
   ],
   "dependencies": {
-    "@douyinfe/semi-ui": "2.52.0",
+    "@douyinfe/semi-ui": "2.60.0",
     "@rematch/core": "2.2.0",
     "@rematch/loading": "2.1.2",
-    "@sendgrid/mail": "8.1.0",
+    "@sendgrid/mail": "8.1.3",
-    "@vitejs/plugin-react": "4.2.1",
+    "@vitejs/plugin-react": "4.3.1",
     "better-sqlite3": "8.6.0",
     "body-parser": "1.20.2",
     "cookie-session": "2.1.0",
     "handlebars": "4.7.8",
-    "highcharts": "11.3.0",
+    "highcharts": "11.4.3",
     "highcharts-react-official": "3.2.1",
     "lodash": "4.17.21",
     "lowdb": "6.0.1",
     "markdown": "^0.5.0",
-    "nanoid": "5.0.5",
+    "nanoid": "5.0.7",
     "node-fetch": "3.3.2",
     "node-mailjet": "6.0.5",
     "query-string": "8.2.0",
-    "react": "18.2.0",
+    "react": "18.3.1",
-    "react-dom": "18.2.0",
+    "react-dom": "18.3.1",
-    "react-redux": "9.1.0",
+    "react-redux": "9.1.2",
     "react-router": "5.2.1",
     "react-router-dom": "5.3.0",
     "redux": "5.0.1",
     "redux-thunk": "3.1.0",
-    "restana": "4.9.7",
+    "restana": "4.9.9",
     "serve-static": "1.15.0",
     "slack": "11.0.2",
     "string-similarity": "^4.0.4",
-    "vite": "5.0.12",
+    "vite": "5.2.13",
     "x-ray": "2.3.4"
   },
   "devDependencies": {
-    "@babel/core": "7.23.9",
+    "@babel/core": "7.24.7",
-    "@babel/eslint-parser": "7.23.10",
+    "@babel/eslint-parser": "7.24.7",
-    "@babel/preset-env": "7.23.9",
+    "@babel/preset-env": "7.24.7",
-    "@babel/preset-react": "7.23.3",
+    "@babel/preset-react": "7.24.7",
-    "chai": "5.0.3",
+    "chai": "5.1.1",
     "eslint": "8.56.0",
     "eslint-config-prettier": "8.8.0",
-    "eslint-plugin-react": "7.33.2",
+    "eslint-plugin-react": "7.34.2",
-    "esmock": "2.6.3",
+    "esmock": "2.6.5",
     "history": "5.3.0",
     "husky": "4.3.8",
     "less": "4.2.0",
     "lint-staged": "13.2.2",
-    "mocha": "10.2.0",
+    "mocha": "10.4.0",
-    "prettier": "3.2.5",
+    "prettier": "3.3.2",
     "redux-logger": "3.0.6"
   }
 }

test/provider/immoswp.test.js

@@ -25,12 +25,10 @@ describe('#immoswp testsuite()', () => {
           expect(notify.size).to.be.a('string');
           expect(notify.title).to.be.a('string');
           expect(notify.link).to.be.a('string');
-          expect(notify.address).to.be.a('string');
           /** check the values if possible **/
           expect(notify.price).that.does.include('€');
           expect(notify.title).to.be.not.empty;
           expect(notify.link).that.does.include('https://immo.swp.de');
-          expect(notify.address).to.be.not.empty;
         });
         resolve();
       });

ui/src/views/generalSettings/GeneralSettings.jsx

@@ -184,8 +184,7 @@ const GeneralSettings = function GeneralSettings() {
                     more likely to fail, but they are cheaper. A call with a datacenter proxy cost 10 credits.
                     <h4>Residential-Proxy</h4>
                     High-quality proxy server located in one of the real people houses across the world. Datacenter
-                    proxies are faster and more likely to success, but they are more expensive. A call with a datacenter
+                    proxies are faster and more likely to success, but they are more expensive.
-                    proxy cost 250 credits.
                     <br />
                     <br />
                     <b>

ui/src/views/jobs/ProcessingTimes.jsx

@@ -45,7 +45,7 @@ export default function ProcessingTimes({ processingTimes }) {
               {format(new Date(processingTimes.scrapingAntData.end_date))}
               <br />
               Credits: {processingTimes.scrapingAntData.remained_credits}/
-              {processingTimes.scrapingAntData.plan_total_credits} (250 credits per call)
+              {processingTimes.scrapingAntData.plan_total_credits}
             </p>
             If you want to scrape Immoscout or Immonet more often, you have to purchase a premium account of{' '}
             <a href="https://scrapingant.com/" target="_blank" rel="noreferrer">