From dd089a9cbc9720a929b8c190a4e73791276948fc Mon Sep 17 00:00:00 2001 From: rzuasti Date: Tue, 2 Jun 2026 16:11:27 -0400 Subject: [PATCH] Deduplicate device_events within a time window When the same scanner sees the same device (same MAC and IPv4) again within a configurable window (default 1 minute), only one device_events row is recorded, keeping the events table from filling with near-identical rows. Device last_seen updates and notifications are unaffected. Co-Authored-By: Claude Opus 4.8 --- README.md | 4 ++ TODO.md | 2 +- backend/src/db/device_events.rs | 70 +++++++++++++++++++++++++++- backend/src/events.rs | 82 ++++++++++++++++++++++++++------- backend/src/settings.rs | 39 ++++++++++++++++ examples/sample_oott.toml | 3 ++ nix/modules/oott-service.nix | 5 ++ 7 files changed, 187 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 783a0d6..58d26e5 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,7 @@ Finally, in your `configuration.nix` (or in an import file) enable and configure notifications.pushover.token = "YOUR API TOKEN GOES HERE"; notifications.pushover.user_key = "YOUR USER TOKEN GOES HERE"; retention.window = "365d"; + device_events.deduplication_window = "1m"; }; } ``` @@ -155,6 +156,7 @@ If you are using Docker I recommend writing the config using TOML, [here](https: |`notifications.pushover.token`||Your pushover token goes here, just copy&paste from their website after creating the app| |`notifications.pushover.user_key`||User key goes here, this is the account wide code for pushover| |`retention.window`|`365d`|How long to retain device events and notifications. Records older than this are purged daily. Accepts duration strings (e.g. `90d`, `1y`, `6m`). Defaults to one year.| +|`device_events.deduplication_window`|`1m`|Suppress duplicate device events: if the same scanner sees the same device (same MAC and IPv4) again within this window, only one event is recorded. Accepts duration strings (e.g. `30s`, `1m`, `5m`). Defaults to one minute.| ## Network ports OOTT binds to the following ports on the host where it runs: @@ -229,6 +231,8 @@ Storage is directly proportional to scan frequency and retention window. The two A 15-minute wait (the default) cuts storage to about one eighth of the worst-case figures above — the medium office drops from up to 18 GB to roughly 2 GB per year. +**Event deduplication** — `device_events.deduplication_window` caps how often the same scanner can record an event for the same device. With several scanners (ARP, mDNS, SSDP, DHCP) reporting overlapping sightings, this collapses near-identical rows into one per scanner per device per window, trimming storage without changing scan timings. Widen it to keep fewer events; narrow it (or set it very small) to keep a finer-grained history. + **Retention window** — `retention.window` sets how far back history is kept. Halving the window halves the storage. Useful reference points: | `retention.window` | Use case | diff --git a/TODO.md b/TODO.md index a98a9de..5731f05 100644 --- a/TODO.md +++ b/TODO.md @@ -12,7 +12,7 @@ - [x] Add "devices seen on last scan" to the ARP and SNMP scanners status - [x] Modify the passive scanners status to count devices seen in the last hour - [x] Simplify the code of scanners (repeated logic) -- [ ] Modify the event management to ignore duplicate events if they happen within a time threshold +- [x] Modify the event management to ignore duplicate events if they happen within a time threshold - [x] Review the recommended timings for the ARP scanner (change defaults) — SNMP defaults set to 10m/5s ## Frontend diff --git a/backend/src/db/device_events.rs b/backend/src/db/device_events.rs index 1644267..78fffad 100644 --- a/backend/src/db/device_events.rs +++ b/backend/src/db/device_events.rs @@ -4,7 +4,7 @@ use chrono::{DateTime, Utc}; use log::{debug, error}; use rusqlite::{params, params_from_iter}; -use crate::model::device_events::DeviceEvent; +use crate::model::device_events::{DeviceEvent, DeviceEventScanner}; use crate::utils::network::normalize_mac; pub fn insert(event: DeviceEvent) -> Result { @@ -33,6 +33,32 @@ pub fn insert(event: DeviceEvent) -> Result { } } +/// Returns true if an event with the same scanner, MAC and IPv4 address was already recorded +/// at or after `since`. Used to suppress duplicate sightings that the same scanner reports for +/// the same device within a short deduplication window. +pub fn recent_duplicate_exists( + mac_address: &str, + ipv4_address: &str, + scanner: &DeviceEventScanner, + since: DateTime, +) -> Result { + let conn = db::get_db_connection(); + let mac_address = normalize_mac(mac_address); + + let count: i64 = conn.query_row( + "SELECT COUNT(*) FROM device_events WHERE mac_address = ?1 AND ipv4_address = ?2 AND scanner = ?3 AND created_on >= ?4", + params![ + mac_address, + ipv4_address, + scanner, + since.to_rfc3339_opts(chrono::SecondsFormat::Nanos, false) + ], + |row| row.get(0), + )?; + + Ok(count > 0) +} + pub fn list( mac_address: Option, created_from: Option>, @@ -197,6 +223,48 @@ mod tests { assert_eq!(event.scanner, DeviceEventScanner::Mdns); } + #[tokio::test] + async fn test_recent_duplicate_exists() { + tests_common::setup().await; + + let mac = "ab:cd:ef:00:11:22".to_string(); + let ip = "192.168.5.5".to_string(); + let created_on = Utc::now(); + insert(DeviceEvent::new( + mac.clone(), + created_on, + DeviceEventType::DeviceSeen, + ip.clone(), + "Vendor".to_string(), + DeviceEventScanner::Arp, + )) + .unwrap(); + + let within_window = created_on - chrono::TimeDelta::seconds(60); + + // Same scanner, MAC and IP within the window is a duplicate. + assert!( + recent_duplicate_exists(&mac, &ip, &DeviceEventScanner::Arp, within_window).unwrap() + ); + + // A different IP is not a duplicate. + assert!( + !recent_duplicate_exists(&mac, "192.168.5.6", &DeviceEventScanner::Arp, within_window) + .unwrap() + ); + + // A different scanner is not a duplicate. + assert!( + !recent_duplicate_exists(&mac, &ip, &DeviceEventScanner::Mdns, within_window).unwrap() + ); + + // A cutoff after the stored event (outside the window) is not a duplicate. + let after_event = created_on + chrono::TimeDelta::seconds(1); + assert!( + !recent_duplicate_exists(&mac, &ip, &DeviceEventScanner::Arp, after_event).unwrap() + ); + } + #[tokio::test] async fn test_list() { tests_common::setup().await; diff --git a/backend/src/events.rs b/backend/src/events.rs index 4ba0814..99cbb5c 100644 --- a/backend/src/events.rs +++ b/backend/src/events.rs @@ -197,24 +197,45 @@ fn send_notification(notification: Notification) -> Result<(), Box> { Ok(()) } +/// Record a device event, skipping it when the same scanner already recorded an event for the +/// same device (same MAC and IPv4) within the configured deduplication window. This keeps the +/// events table from filling with near-identical rows when a scanner sees a device repeatedly. +fn record_event(event: DeviceEvent) { + let window: Duration = get_settings().device_events.deduplication_window.into(); + let since = Utc::now() - chrono::Duration::from_std(window).unwrap_or_default(); + + match db::device_events::recent_duplicate_exists( + &event.mac_address, + &event.ipv4_address, + &event.scanner, + since, + ) { + Ok(true) => { + debug!("Skipping duplicate device event within window: {event}"); + return; + } + Ok(false) => {} + // On a check error, fall through and record the event rather than silently drop it. + Err(err) => error!("Device event deduplication check failed ({err}); recording event"), + } + + if let Err(err) = db::device_events::insert(event) { + error!("Failed to record device event: {err}"); + } +} + pub fn trigger_new_device( device: Device, scanner: DeviceEventScanner, ) -> Result<(), Box> { - let event = DeviceEvent::new( + record_event(DeviceEvent::new( device.mac_address.clone(), Utc::now(), DeviceEventType::NewDevice, device.ipv4_address.clone(), device.vendor.clone(), scanner, - ); - if let Err(err) = db::device_events::insert(event) { - error!( - "Failed to record device event for {}: {}", - device.mac_address, err - ); - } + )); let (title, body) = render_new_device(&device); let notification = Notification::new( @@ -235,20 +256,14 @@ pub fn trigger_existing_device( new_device: Device, scanner: DeviceEventScanner, ) -> Result<(), Box> { - let event = DeviceEvent::new( + record_event(DeviceEvent::new( new_device.mac_address.clone(), Utc::now(), DeviceEventType::DeviceSeen, new_device.ipv4_address.clone(), new_device.vendor.clone(), scanner, - ); - if let Err(err) = db::device_events::insert(event) { - error!( - "Failed to record device event for {}: {}", - new_device.mac_address, err - ); - } + )); // Notify if the device comes back online after not being seen for the configured period let elapsed_since_last_seen: Duration = (Local::now().to_utc() - existing_device.last_seen) @@ -447,4 +462,39 @@ mod tests { assert!(body.contains("Vendor: Apple, Inc. -> Samsung Electronics")); assert!(body.contains("MAC spoofing")); } + + #[tokio::test] + async fn repeated_sighting_from_same_scanner_records_one_event() { + crate::tests_common::setup().await; + + let mut device = Device::new( + "fa:ce:fa:ce:00:01".to_string(), + "192.168.7.7".to_string(), + "Acme".to_string(), + Utc::now(), + ); + device.last_seen = Utc::now(); + let mac = device.mac_address.clone(); + + // Two sightings from the same scanner within the dedup window: only one event recorded. + trigger_existing_device(device.clone(), device.clone(), DeviceEventScanner::Arp).unwrap(); + trigger_existing_device(device.clone(), device.clone(), DeviceEventScanner::Arp).unwrap(); + + let after_arp = db::device_events::list(Some(mac.clone()), None, None, None).unwrap(); + assert_eq!( + after_arp.len(), + 1, + "A repeated same-scanner sighting should be deduplicated" + ); + + // A sighting from a different scanner is not a duplicate and is recorded. + trigger_existing_device(device.clone(), device.clone(), DeviceEventScanner::Mdns).unwrap(); + + let after_mdns = db::device_events::list(Some(mac), None, None, None).unwrap(); + assert_eq!( + after_mdns.len(), + 2, + "A sighting from a different scanner should be recorded" + ); + } } diff --git a/backend/src/settings.rs b/backend/src/settings.rs index 818911c..c356f10 100644 --- a/backend/src/settings.rs +++ b/backend/src/settings.rs @@ -148,6 +148,19 @@ impl Default for Retention { } } +#[derive(Debug, Deserialize, Clone)] +pub struct DeviceEvents { + pub deduplication_window: DurationString, +} + +impl Default for DeviceEvents { + fn default() -> Self { + DeviceEvents { + deduplication_window: DurationString::try_from("1m".to_string()).unwrap(), + } + } +} + #[derive(Debug, Deserialize, Clone)] pub struct Settings { pub database: Database, @@ -160,6 +173,8 @@ pub struct Settings { #[serde(default)] pub retention: Retention, #[serde(default)] + pub device_events: DeviceEvents, + #[serde(default)] pub mdns_scanner: MdnsScanner, #[serde(default)] pub ssdp_scanner: SsdpScanner, @@ -306,6 +321,30 @@ mod tests { ); } + #[test] + fn device_events_dedup_window_defaults_when_section_omitted() { + let settings = parse(BASE_CONFIG); + assert_eq!( + std::time::Duration::from(settings.device_events.deduplication_window), + std::time::Duration::from_secs(60) + ); + } + + #[test] + fn device_events_dedup_window_is_parsed() { + let toml = format!( + "{BASE_CONFIG} + [device_events] + deduplication_window = \"5m\" + " + ); + let settings = parse(&toml); + assert_eq!( + std::time::Duration::from(settings.device_events.deduplication_window), + std::time::Duration::from_secs(5 * 60) + ); + } + #[test] fn scanners_can_be_disabled() { let toml = format!( diff --git a/examples/sample_oott.toml b/examples/sample_oott.toml index 1481a9f..992007e 100644 --- a/examples/sample_oott.toml +++ b/examples/sample_oott.toml @@ -49,3 +49,6 @@ api_key="CHANGE_ME" # API Key to use the system's API, change this! [retention] window="365d" # How long to keep device events and notifications. Records older than this are deleted daily. Supports: d (days), w (weeks), h (hours), m (minutes) + +[device_events] +deduplication_window="1m" # If the same scanner sees the same device (same MAC and IP) again within this window, only one device event is recorded. Keeps the events table from filling with near-identical rows. Supports: d (days), w (weeks), h (hours), m (minutes), s (seconds) diff --git a/nix/modules/oott-service.nix b/nix/modules/oott-service.nix index 7f89d80..f60bbe8 100644 --- a/nix/modules/oott-service.nix +++ b/nix/modules/oott-service.nix @@ -126,6 +126,11 @@ in { description = "How long to retain device events and notifications. Records older than this are purged daily. Accepts duration strings (e.g. 90d, 1y, 6m)."; default = "365d"; }; + device_events.deduplication_window = mkOption { + type = types.str; + description = "If the same scanner sees the same device (same MAC and IP) again within this window, only one device event is recorded. Keeps the events table from filling with near-identical rows. Accepts duration strings (e.g. 30s, 1m, 5m)."; + default = "1m"; + }; web_server.ip_address = mkOption { type = types.str; description = "IP address to bind the web server (API and web UI) to. Use 0.0.0.0 to bind to all interfaces.";