From c7e16784072f1455028f820caa6dc0dc3d3d9ec5 Mon Sep 17 00:00:00 2001 From: rzuasti Date: Thu, 28 May 2026 09:00:36 -0400 Subject: [PATCH] Add retention policy for device_events and notifications Purges records older than a configurable window (default 365d) once per day. Also adds a composite index on device_events(mac_address, created_on DESC) for efficient scan history queries at scale, and fixes a non-deterministic pagination order by adding id as a tiebreaker to ORDER BY. Co-Authored-By: Claude Sonnet 4.6 --- .../04-add_device_events_index/up.sql | 1 + backend/run_tests.sh | 2 +- backend/src/db/device_events.rs | 95 +++++++++++++++++-- backend/src/db/notifications.rs | 53 +++++++++++ backend/src/main.rs | 5 +- backend/src/retention.rs | 33 +++++++ backend/src/settings.rs | 15 +++ examples/sample_oott.toml | 3 + 8 files changed, 197 insertions(+), 10 deletions(-) create mode 100644 backend/database_migrations/04-add_device_events_index/up.sql create mode 100644 backend/src/retention.rs diff --git a/backend/database_migrations/04-add_device_events_index/up.sql b/backend/database_migrations/04-add_device_events_index/up.sql new file mode 100644 index 0000000..e591c76 --- /dev/null +++ b/backend/database_migrations/04-add_device_events_index/up.sql @@ -0,0 +1 @@ +CREATE INDEX idx_device_events_mac_created ON device_events (mac_address, created_on DESC); diff --git a/backend/run_tests.sh b/backend/run_tests.sh index 1c04272..4861198 100755 --- a/backend/run_tests.sh +++ b/backend/run_tests.sh @@ -1,3 +1,3 @@ #!/bin/sh sudo rm -f oott.db -sudo CARGO_HOME=$HOME/.cargo cargo test -- --show-output +sudo CARGO_HOME=$HOME/.cargo cargo test -- --show-output --test-threads=1 diff --git a/backend/src/db/device_events.rs b/backend/src/db/device_events.rs index 7677022..f6fd403 100644 --- a/backend/src/db/device_events.rs +++ b/backend/src/db/device_events.rs @@ -1,5 +1,6 @@ use crate::db; use crate::db::error::DbError; +use chrono::{DateTime, Utc}; use log::{debug, error}; use rusqlite::{params, params_from_iter}; @@ -49,7 +50,7 @@ pub fn list( params.push(mac.into()); } - sql_statement.push_str(" ORDER BY created_on DESC"); + sql_statement.push_str(" ORDER BY created_on DESC, id DESC"); if let (Some(page_offset), Some(page_limit)) = (page_offset, page_limit) { debug!( @@ -79,6 +80,24 @@ pub fn list( Ok(events) } +pub fn purge_older_than(cutoff: DateTime) -> Result { + let conn = db::get_db_connection(); + + match conn.execute( + "DELETE FROM device_events WHERE created_on < ?1", + params![cutoff], + ) { + Ok(count) => { + debug!("Purged {} device event(s) older than {}", count, cutoff); + Ok(count) + } + Err(error) => { + error!("Error purging old device events: {error}"); + Err(DbError::from(error)) + } + } +} + #[cfg(test)] fn read(id: i64) -> Option { let conn = db::get_db_connection(); @@ -198,18 +217,80 @@ mod tests { assert!(events.is_empty(), "Unknown MAC should return empty list"); } + #[tokio::test] + async fn test_purge_older_than() { + tests_common::setup().await; + + let mac = "ee:ee:ee:ee:ee:ee".to_string(); + let old_id = insert(DeviceEvent::new( + mac.clone(), + chrono::DateTime::parse_from_rfc3339("2020-01-01T00:00:00+00:00") + .unwrap() + .into(), + DeviceEventType::NewDevice, + "10.0.0.200".to_string(), + "Old Vendor".to_string(), + )) + .unwrap(); + + let recent_id = insert(DeviceEvent::new( + mac.clone(), + Utc::now(), + DeviceEventType::DeviceSeen, + "10.0.0.200".to_string(), + "Old Vendor".to_string(), + )) + .unwrap(); + + let cutoff = Utc::now() - chrono::TimeDelta::days(365); + let purged = purge_older_than(cutoff).unwrap(); + + assert!(purged >= 1, "At least 1 device event should have been purged"); + assert!(read(old_id).is_none(), "Old device event should have been purged"); + assert!(read(recent_id).is_some(), "Recent device event should not have been purged"); + } + #[tokio::test] async fn test_list_pagination() { tests_common::setup().await; - let first_page = list(None, Some(0), Some(2)).unwrap(); + let mac = "cc:cc:cc:cc:cc:cc".to_string(); + insert(DeviceEvent::new( + mac.clone(), + chrono::DateTime::parse_from_rfc3339("2026-01-01T00:00:00Z") + .unwrap() + .into(), + DeviceEventType::NewDevice, + "10.0.0.1".to_string(), + "Vendor C".to_string(), + )) + .unwrap(); + insert(DeviceEvent::new( + mac.clone(), + chrono::DateTime::parse_from_rfc3339("2026-02-01T00:00:00Z") + .unwrap() + .into(), + DeviceEventType::DeviceSeen, + "10.0.0.1".to_string(), + "Vendor C".to_string(), + )) + .unwrap(); + insert(DeviceEvent::new( + mac.clone(), + chrono::DateTime::parse_from_rfc3339("2026-03-01T00:00:00Z") + .unwrap() + .into(), + DeviceEventType::DeviceSeen, + "10.0.0.1".to_string(), + "Vendor C".to_string(), + )) + .unwrap(); + + let first_page = list(Some(mac.clone()), Some(0), Some(2)).unwrap(); assert_eq!(first_page.len(), 2, "First page should have 2 events"); - let second_page = list(None, Some(2), Some(2)).unwrap(); - assert!( - second_page.len() >= 1, - "Second page should have at least 1 event" - ); + let second_page = list(Some(mac.clone()), Some(2), Some(2)).unwrap(); + assert_eq!(second_page.len(), 1, "Second page should have 1 event"); let first_ids: Vec = first_page.iter().map(|e| e.id).collect(); for event in &second_page { diff --git a/backend/src/db/notifications.rs b/backend/src/db/notifications.rs index 3bf2f10..d45e547 100644 --- a/backend/src/db/notifications.rs +++ b/backend/src/db/notifications.rs @@ -1,5 +1,6 @@ use crate::db; use crate::db::error::DbError; +use chrono::{DateTime, Utc}; use log::{debug, error}; use rusqlite::{params, params_from_iter}; @@ -123,6 +124,24 @@ pub fn mark_all_as_old() -> Result<(), DbError> { } } +pub fn purge_older_than(cutoff: DateTime) -> Result { + let conn = db::get_db_connection(); + + match conn.execute( + "DELETE FROM notifications WHERE created_on < ?1", + params![cutoff], + ) { + Ok(count) => { + debug!("Purged {} notification(s) older than {}", count, cutoff); + Ok(count) + } + Err(error) => { + error!("Error purging old notifications: {error}"); + Err(DbError::from(error)) + } + } +} + pub fn read(id: i64) -> Option { let conn = db::get_db_connection(); @@ -163,6 +182,40 @@ mod tests { use super::*; use crate::{model::notifications::NotificationType, tests_common}; + #[tokio::test] + async fn test_purge_older_than() { + tests_common::setup().await; + + let old_id = insert(Notification::new( + chrono::DateTime::parse_from_rfc3339("2020-01-01T00:00:00+00:00") + .unwrap() + .into(), + NotificationType::Other, + "Old notification".to_string(), + "Old body".to_string(), + false, + None, + )) + .unwrap(); + + let recent_id = insert(Notification::new( + Utc::now(), + NotificationType::Other, + "Recent notification".to_string(), + "Recent body".to_string(), + false, + None, + )) + .unwrap(); + + let cutoff = Utc::now() - chrono::TimeDelta::days(365); + let purged = purge_older_than(cutoff).unwrap(); + + assert!(purged >= 1, "At least 1 notification should have been purged"); + assert!(read(old_id).is_none(), "Old notification should have been purged"); + assert!(read(recent_id).is_some(), "Recent notification should not have been purged"); + } + #[tokio::test] async fn test_mark_as_old() { tests_common::setup().await; diff --git a/backend/src/main.rs b/backend/src/main.rs index d74250e..4558b60 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -7,6 +7,7 @@ mod device_finders; mod events; mod mac_vendor_finder; mod model; +mod retention; mod scanner; mod settings; mod utils; @@ -56,6 +57,6 @@ async fn main() -> Result<(), Box> { // Initialize database db::init_db().await?; - // Start the device scanner and web server (for API and UI) in parallel - tokio::join!(scanner::scan(), web_server::serve()).0 + // Start the device scanner, web server, and retention cleaner in parallel + tokio::join!(scanner::scan(), web_server::serve(), retention::run()).0 } diff --git a/backend/src/retention.rs b/backend/src/retention.rs new file mode 100644 index 0000000..d6127b6 --- /dev/null +++ b/backend/src/retention.rs @@ -0,0 +1,33 @@ +use crate::db; +use crate::settings::get_settings; +use chrono::Utc; +use log::{error, info}; +use tokio::time::{Duration, sleep}; + +pub async fn run() { + loop { + let window: std::time::Duration = get_settings().retention.window.into(); + let cutoff = match chrono::Duration::from_std(window) { + Ok(d) => Utc::now() - d, + Err(e) => { + error!("Retention: invalid window duration: {}", e); + sleep(Duration::from_secs(24 * 60 * 60)).await; + continue; + } + }; + + info!("Retention: purging records older than {}", cutoff); + + match db::device_events::purge_older_than(cutoff) { + Ok(count) => info!("Retention: purged {} device event(s)", count), + Err(e) => error!("Retention: error purging device events: {}", e), + } + + match db::notifications::purge_older_than(cutoff) { + Ok(count) => info!("Retention: purged {} notification(s)", count), + Err(e) => error!("Retention: error purging notifications: {}", e), + } + + sleep(Duration::from_secs(24 * 60 * 60)).await; + } +} diff --git a/backend/src/settings.rs b/backend/src/settings.rs index 2f000ec..ed1432d 100644 --- a/backend/src/settings.rs +++ b/backend/src/settings.rs @@ -48,6 +48,19 @@ pub struct WebServer { pub api_key: String, } +#[derive(Debug, Deserialize, Clone)] +pub struct Retention { + pub window: DurationString, +} + +impl Default for Retention { + fn default() -> Self { + Retention { + window: DurationString::try_from("365d".to_string()).unwrap(), + } + } +} + #[derive(Debug, Deserialize, Clone)] pub struct Settings { pub database: Database, @@ -56,6 +69,8 @@ pub struct Settings { pub timings: Timings, pub notifications: Notifications, pub web_server: WebServer, + #[serde(default)] + pub retention: Retention, } // End configuration structure // ----------------------------------------------------------- diff --git a/examples/sample_oott.toml b/examples/sample_oott.toml index 266d801..5836ecb 100644 --- a/examples/sample_oott.toml +++ b/examples/sample_oott.toml @@ -24,3 +24,6 @@ user_key="" # User key goes here, this is the account wide code for pushover ip_address="0.0.0.0" # IP to bind the web server for the API and web UI to, use 0.0.0.0 to bind it to all interfaces port=3000 # Port to listen on api_key="CHANGE_ME" # API Key to use the system's API, change this! + +[retention] +window="365d" # How long to keep device events and notifications. Records older than this are deleted daily. Supports: d (days), w (weeks), h (hours), m (minutes)