new usersetting to blacklist (filter) also on description

.gitignore

@@ -7,3 +7,4 @@ npm-debug.log
 .idea
 .vscode
 tools/release/config.json
+.agents

lib/FredyPipelineExecutioner.js

@@ -38,11 +38,15 @@ import { formatListing } from './utils/formatListing.js';
  * 3) Normalize listings to the provider schema
  * 4) Filter out incomplete/blacklisted listings
  * 5) Identify new listings (vs. previously stored hashes)
- * 6) Persist new listings
+ * 6) Optionally enrich new listings via provider.fetchDetails
- * 7) Filter out entries similar to already seen ones
+ * 7) Optionally re-apply the provider blacklist using the (now enriched)
- * 8) Filter out entries that do not match the job's specFilter
+ *    description — only when the user opted in via
- * 9) Filter out entries that do not match the job's spatialFilter
+ *    `blacklist_filter_on_provider_details`
- * 10) Dispatch notifications
+ * 8) Persist new listings
+ * 9) Filter out entries similar to already seen ones
+ * 10) Filter out entries that do not match the job's specFilter
+ * 11) Filter out entries that do not match the job's spatialFilter
+ * 12) Dispatch notifications
  */
 class FredyPipelineExecutioner {
   /**
@@ -86,6 +90,7 @@ class FredyPipelineExecutioner {
       .then(this._filter.bind(this))
       .then(this._findNew.bind(this))
       .then(this._fetchDetails.bind(this))
+      .then(this._filterAfterDetails.bind(this))
       .then(this._geocode.bind(this))
       .then(this._save.bind(this))
       .then(this._calculateDistance.bind(this))
@@ -266,6 +271,48 @@ class FredyPipelineExecutioner {
     );
   }
 
+  /**
+   * Re-apply the provider's blacklist filter after `_fetchDetails` has had a
+   * chance to enrich the listings (e.g., load the full description from the
+   * detail page). The initial `_filter` step only sees the truncated snippet
+   * exposed on the search results page, so a blacklisted term that lives
+   * deeper in the listing's full description would otherwise slip through.
+   *
+   * Opt-in: gated by the user setting `blacklist_filter_on_provider_details`.
+   * The full detail description tends to contain a lot of boilerplate (legal,
+   * exposé contact info, generic marketing copy) which can accidentally match
+   * a blacklist term and remove otherwise relevant listings. Users who want
+   * the stricter behavior must enable the setting explicitly.
+   *
+   * Throws {@link NoNewListingsWarning} when all listings are filtered out
+   * so the rest of the pipeline (save + notify) is short-circuited.
+   *
+   * @param {ParsedListing[]} listings Enriched listings to re-filter.
+   * @returns {ParsedListing[]} Listings that still pass the provider's filter.
+   * @throws {NoNewListingsWarning} When every listing is filtered out.
+   */
+  _filterAfterDetails(listings) {
+    if (typeof this._providerConfig.filter !== 'function') {
+      return listings;
+    }
+    const userId = getJob(this._jobKey)?.userId;
+    const enabled = getUserSettings(userId)?.blacklist_filter_on_provider_details === true;
+    if (!enabled) {
+      return listings;
+    }
+    const kept = listings.filter(this._providerConfig.filter);
+    const removed = listings.length - kept.length;
+    if (removed > 0) {
+      logger.debug(
+        `Re-filter after detail enrichment removed ${removed} listing(s) by blacklist (Provider: '${this._providerId}')`,
+      );
+    }
+    if (kept.length === 0) {
+      throw new NoNewListingsWarning();
+    }
+    return kept;
+  }
+
   /**
    * Determine which listings are new by comparing their IDs against stored hashes.
    *

lib/api/routes/userSettingsRoute.js

@@ -103,6 +103,28 @@ export default async function userSettingsPlugin(fastify) {
     }
   });
 
+  fastify.post('/blacklist-filter-on-details', async (request, reply) => {
+    const userId = request.session.currentUser;
+    const { blacklist_filter_on_provider_details } = request.body;
+
+    const globalSettings = await getSettings();
+    if (globalSettings.demoMode && !isAdmin(request)) {
+      return reply.code(403).send({ error: 'In demo mode, it is not allowed to change settings.' });
+    }
+
+    if (typeof blacklist_filter_on_provider_details !== 'boolean') {
+      return reply.code(400).send({ error: 'blacklist_filter_on_provider_details must be a boolean.' });
+    }
+
+    try {
+      upsertSettings({ blacklist_filter_on_provider_details }, userId);
+      return { success: true };
+    } catch (error) {
+      logger.error('Error updating blacklist-filter-on-details setting', error);
+      return reply.code(500).send({ error: error.message });
+    }
+  });
+
   fastify.post('/listings-view-mode', async (request, reply) => {
     const userId = request.session.currentUser;
     const { listings_view_mode } = request.body;

lib/provider/immoscout.js

@@ -198,7 +198,9 @@ function normalize(o) {
  * @returns {boolean}
  */
 function applyBlacklist(o) {
-  return !isOneOf(o.title, appliedBlackList);
+  const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
+  const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
+  return titleNotBlacklisted && descNotBlacklisted;
 }
 /** @type {ProviderConfig} */
 const config = {

lib/provider/neubauKompass.js

@@ -42,7 +42,9 @@ function normalize(o) {
  * @returns {boolean}
  */
 function applyBlacklist(o) {
-  return !isOneOf(o.title, appliedBlackList);
+  const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
+  const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
+  return titleNotBlacklisted && descNotBlacklisted;
 }
 
 /** @type {ProviderConfig} */

test/mocks/mockStore.js

@@ -17,8 +17,12 @@ export const getGeocoordinatesByAddress = (any) => {
   return null;
 };
 
+let userSettings = null;
+export function setUserSettings(settings) {
+  userSettings = settings;
+}
 export function getUserSettings(userId) {
-  return null;
+  return userSettings;
 }
 
 export async function getSettings() {

test/pipeline_filtering.test.js

@@ -3,9 +3,10 @@
  * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
  */
 
-import { expect } from 'vitest';
+import { afterEach, expect } from 'vitest';
 import { mockFredy } from './utils.js';
 import * as mockStore from './mocks/mockStore.js';
+import { get as getLastNotification } from './mocks/mockNotification.js';
 
 describe('Issue reproduction: listings filtered by similarity or area should be marked as manually deleted', () => {
   it('should call deleteListingsById when listings are filtered by similarity', async () => {
@@ -113,3 +114,223 @@ describe('Issue reproduction: listings filtered by similarity or area should be
     expect(mockStore.deletedIds).toContain('2');
   });
 });
+
+describe('Blacklist is re-applied after detail enrichment', () => {
+  afterEach(() => {
+    mockStore.setUserSettings(null);
+  });
+
+  it('filters out a listing whose blacklisted term only appears in the enriched description', async () => {
+    const Fredy = await mockFredy();
+    const providerId = 'test-provider';
+
+    mockStore.setUserSettings({
+      provider_details: [providerId],
+      blacklist_filter_on_provider_details: true,
+    });
+
+    const mockSimilarityCache = {
+      checkAndAddEntry: () => false,
+    };
+
+    const blacklist = ['allkauf'];
+
+    // The search results page returns a clean snippet (no blacklisted term).
+    // fetchDetails simulates loading the full detail page and discovers the
+    // blacklisted term hidden deep in the description.
+    const providerConfig = {
+      url: 'http://example.com',
+      getListings: () =>
+        Promise.resolve([
+          {
+            id: 'kept',
+            title: 'Nice house',
+            address: 'Some street',
+            price: '500000',
+            link: 'http://example.com/kept',
+            description: 'Cozy home with garden',
+          },
+          {
+            id: 'blacklisted',
+            title: 'Eleganz trifft Raumkomfort',
+            address: 'Other street',
+            price: '600000',
+            link: 'http://example.com/blacklisted',
+            description: 'Eleganz trifft Raumkomfort',
+          },
+        ]),
+      normalize: (l) => l,
+      filter: (l) => {
+        const text = `${l.title ?? ''} ${l.description ?? ''}`.toLowerCase();
+        return !blacklist.some((term) => text.includes(term));
+      },
+      fetchDetails: (listing) => {
+        if (listing.id === 'blacklisted') {
+          return Promise.resolve({
+            ...listing,
+            description: 'Mit allkauf Haus wird dein Traum vom Eigenheim wahr.',
+          });
+        }
+        return Promise.resolve(listing);
+      },
+      crawlFields: {
+        id: 'id',
+        title: 'title',
+        address: 'address',
+        price: 'price',
+        link: 'link',
+        description: 'description',
+      },
+      requiredFieldNames: ['id', 'title', 'address', 'price', 'link', 'description'],
+    };
+
+    const mockedJob = {
+      id: 'blacklist-test-job',
+      notificationAdapter: null,
+      specFilter: null,
+      spatialFilter: null,
+    };
+
+    const fredy = new Fredy(providerConfig, mockedJob, providerId, mockSimilarityCache, undefined);
+
+    const result = await fredy.execute();
+
+    expect(result).toBeInstanceOf(Array);
+    const ids = result.map((l) => l.id);
+    expect(ids).toContain('kept');
+    expect(ids).not.toContain('blacklisted');
+
+    const notification = getLastNotification();
+    const notifiedIds = (notification?.payload ?? []).map((p) => p.id);
+    expect(notifiedIds).not.toContain('blacklisted');
+  });
+
+  it('short-circuits the pipeline when all listings get blacklisted after enrichment', async () => {
+    const Fredy = await mockFredy();
+    const providerId = 'all-blacklisted-provider';
+
+    mockStore.setUserSettings({
+      provider_details: [providerId],
+      blacklist_filter_on_provider_details: true,
+    });
+
+    const mockSimilarityCache = {
+      checkAndAddEntry: () => false,
+    };
+
+    const blacklist = ['allkauf'];
+
+    const providerConfig = {
+      url: 'http://example.com',
+      getListings: () =>
+        Promise.resolve([
+          {
+            id: 'only',
+            title: 'Eleganz trifft Raumkomfort',
+            address: 'Some street',
+            price: '700000',
+            link: 'http://example.com/only',
+            description: 'Eleganz trifft Raumkomfort',
+          },
+        ]),
+      normalize: (l) => l,
+      filter: (l) => {
+        const text = `${l.title ?? ''} ${l.description ?? ''}`.toLowerCase();
+        return !blacklist.some((term) => text.includes(term));
+      },
+      fetchDetails: (listing) =>
+        Promise.resolve({
+          ...listing,
+          description: 'Mit allkauf Haus wird dein Traum vom Eigenheim wahr.',
+        }),
+      crawlFields: {
+        id: 'id',
+        title: 'title',
+        address: 'address',
+        price: 'price',
+        link: 'link',
+        description: 'description',
+      },
+      requiredFieldNames: ['id', 'title', 'address', 'price', 'link', 'description'],
+    };
+
+    const mockedJob = {
+      id: 'all-blacklisted-job',
+      notificationAdapter: null,
+      specFilter: null,
+      spatialFilter: null,
+    };
+
+    const fredy = new Fredy(providerConfig, mockedJob, providerId, mockSimilarityCache, undefined);
+
+    // Should resolve to undefined (NoNewListingsWarning is caught in _handleError).
+    const result = await fredy.execute();
+    expect(result).toBeUndefined();
+  });
+
+  it('does NOT re-filter when blacklist_filter_on_provider_details is disabled', async () => {
+    const Fredy = await mockFredy();
+    const providerId = 'opt-out-provider';
+
+    // provider_details enabled (so fetchDetails runs) but blacklist re-filter NOT enabled.
+    mockStore.setUserSettings({
+      provider_details: [providerId],
+      blacklist_filter_on_provider_details: false,
+    });
+
+    const mockSimilarityCache = {
+      checkAndAddEntry: () => false,
+    };
+
+    const blacklist = ['allkauf'];
+
+    const providerConfig = {
+      url: 'http://example.com',
+      getListings: () =>
+        Promise.resolve([
+          {
+            id: 'leaks-through',
+            title: 'Eleganz trifft Raumkomfort',
+            address: 'Other street',
+            price: '600000',
+            link: 'http://example.com/leaks-through',
+            description: 'Eleganz trifft Raumkomfort',
+          },
+        ]),
+      normalize: (l) => l,
+      filter: (l) => {
+        const text = `${l.title ?? ''} ${l.description ?? ''}`.toLowerCase();
+        return !blacklist.some((term) => text.includes(term));
+      },
+      fetchDetails: (listing) =>
+        Promise.resolve({
+          ...listing,
+          description: 'Mit allkauf Haus wird dein Traum vom Eigenheim wahr.',
+        }),
+      crawlFields: {
+        id: 'id',
+        title: 'title',
+        address: 'address',
+        price: 'price',
+        link: 'link',
+        description: 'description',
+      },
+      requiredFieldNames: ['id', 'title', 'address', 'price', 'link', 'description'],
+    };
+
+    const mockedJob = {
+      id: 'opt-out-job',
+      notificationAdapter: null,
+      specFilter: null,
+      spatialFilter: null,
+    };
+
+    const fredy = new Fredy(providerConfig, mockedJob, providerId, mockSimilarityCache, undefined);
+
+    const result = await fredy.execute();
+
+    // Listing leaks through because user has not opted in to the stricter check.
+    expect(result).toBeInstanceOf(Array);
+    expect(result.map((l) => l.id)).toContain('leaks-through');
+  });
+});

ui/src/locales/de.json

@@ -334,6 +334,11 @@
   "settings.providerDetailsPlaceholder": "Anbieter für Detail-Abruf auswählen...",
   "settings.providerDetailsUpdated": "Anbieter-Detail-Einstellung aktualisiert.",
   "settings.providerDetailsUpdateError": "Einstellung konnte nicht aktualisiert werden.",
+  "settings.blacklistFilterOnProviderDetails": "Blacklist-Filter auf Anbieter-Details anwenden",
+  "settings.blacklistFilterOnProviderDetailsHelp": "Wenn aktiv, wird die Blacklist zusätzlich gegen die vollständige Beschreibung geprüft, die durch den obigen Anbieter-Details-Schritt geladen wurde. Damit lassen sich Spam-Anbieter (z. B. 'allkauf', 'massa') herausfiltern, die nur tief in der Detail-Seite auftauchen und nicht im kurzen Vorschau-Text der Suchergebnisse stehen. Standardmäßig aus, weil die vollständige Beschreibung oft generischen Boilerplate-Text (Kontaktdaten, rechtliche Hinweise) enthält, der ein Blacklist-Wort versehentlich auslösen und passende Inserate entfernen kann. Hat keine Wirkung auf Anbieter, für die Anbieter-Details nicht aktiviert sind.",
+  "settings.blacklistFilterOnProviderDetailsEnable": "Blacklist auf die vollständige Detail-Beschreibung anwenden",
+  "settings.blacklistFilterOnProviderDetailsUpdated": "Einstellung Blacklist-auf-Details aktualisiert.",
+  "settings.blacklistFilterOnProviderDetailsUpdateError": "Einstellung konnte nicht aktualisiert werden.",
   "settings.listingDeletion": "Inserate löschen",
   "settings.listingDeletionHelp": "Wähle den Standard-Löschmodus. Soft Delete blendet Inserate aus ohne erneutes Scraping; Hard Delete entfernt sie aus der Datenbank.",
   "settings.listingDeletionSoftLabel": "Als gelöscht markieren (Soft Delete)",

ui/src/locales/en.json

@@ -334,6 +334,11 @@
   "settings.providerDetailsPlaceholder": "Select providers to fetch details from...",
   "settings.providerDetailsUpdated": "Provider details setting updated.",
   "settings.providerDetailsUpdateError": "Failed to update setting.",
+  "settings.blacklistFilterOnProviderDetails": "Blacklist-Filtering on Provider Details",
+  "settings.blacklistFilterOnProviderDetailsHelp": "When enabled, the blacklist is re-checked against the full description loaded by the Provider Details step above. This catches spam advertisers (e.g. 'allkauf', 'massa') that only appear deep in the detail page and not in the short search-result snippet. Off by default, because the full description often contains generic boilerplate (contact info, legal text) that may accidentally trigger a blacklist term and remove otherwise relevant listings. Has no effect on providers for which Provider Details is not enabled.",
+  "settings.blacklistFilterOnProviderDetailsEnable": "Apply blacklist to the full detail description",
+  "settings.blacklistFilterOnProviderDetailsUpdated": "Blacklist-on-details setting updated.",
+  "settings.blacklistFilterOnProviderDetailsUpdateError": "Failed to update setting.",
   "settings.listingDeletion": "Listing deletion",
   "settings.listingDeletionHelp": "Choose the default deletion mode. Soft delete hides them without re-scraping; hard delete removes them from the database.",
   "settings.listingDeletionSoftLabel": "Mark as deleted (Soft Delete)",

ui/src/services/state/store.js

@@ -337,6 +337,28 @@ export const useFredyState = create(
               throw Exception;
             }
           },
+          async setBlacklistFilterOnProviderDetails(enabled) {
+            try {
+              await xhrPost('/api/user/settings/blacklist-filter-on-details', {
+                blacklist_filter_on_provider_details: enabled,
+              });
+              set((state) => ({
+                userSettings: {
+                  ...state.userSettings,
+                  settings: {
+                    ...state.userSettings.settings,
+                    blacklist_filter_on_provider_details: enabled,
+                  },
+                },
+              }));
+            } catch (Exception) {
+              console.error(
+                'Error while trying to update blacklist-filter-on-provider-details setting. Error:',
+                Exception,
+              );
+              throw Exception;
+            }
+          },
           async setListingsViewMode(listings_view_mode) {
             try {
               await xhrPost('/api/user/settings/listings-view-mode', { listings_view_mode });

ui/src/views/generalSettings/GeneralSettings.jsx

@@ -130,6 +130,9 @@ const GeneralSettings = function GeneralSettings() {
   // User settings state
   const homeAddress = useSelector((state) => state.userSettings.settings.home_address);
   const providerDetails = useSelector((state) => state.userSettings.settings.provider_details);
+  const blacklistFilterOnProviderDetails = useSelector(
+    (state) => state.userSettings.settings.blacklist_filter_on_provider_details,
+  );
   const listingDeletionPreference = useSelector((state) => state.userSettings.settings.listing_deletion_preference);
   const allProviders = useSelector((state) => state.provider);
   const [address, setAddress] = useState(homeAddress?.address || '');
@@ -647,6 +650,25 @@ const GeneralSettings = function GeneralSettings() {
                   />
                 </SegmentPart>
 
+                <SegmentPart
+                  name={t('settings.blacklistFilterOnProviderDetails')}
+                  helpText={t('settings.blacklistFilterOnProviderDetailsHelp')}
+                >
+                  <Checkbox
+                    checked={blacklistFilterOnProviderDetails === true}
+                    onChange={async (e) => {
+                      try {
+                        await actions.userSettings.setBlacklistFilterOnProviderDetails(e.target.checked);
+                        Toast.success(t('settings.blacklistFilterOnProviderDetailsUpdated'));
+                      } catch {
+                        Toast.error(t('settings.blacklistFilterOnProviderDetailsUpdateError'));
+                      }
+                    }}
+                  >
+                    {t('settings.blacklistFilterOnProviderDetailsEnable')}
+                  </Checkbox>
+                </SegmentPart>
+
                 <SegmentPart name={t('settings.listingDeletion')} helpText={t('settings.listingDeletionHelp')}>
                   <RadioGroup
                     value={listingDeleteHard ? 'hard' : 'soft'}