diff --git a/server.js b/server.js index 2cc5073..9168e32 100644 --- a/server.js +++ b/server.js @@ -44,6 +44,17 @@ DB.exec(` CREATE INDEX IF NOT EXISTS idx_site ON attacks(site_id); CREATE INDEX IF NOT EXISTS idx_attack_type ON attacks(attack_type); CREATE INDEX IF NOT EXISTS idx_source ON attacks(source); + + CREATE TABLE IF NOT EXISTS daily_summary ( + day TEXT NOT NULL, + site_id TEXT NOT NULL DEFAULT '', + attack_type TEXT NOT NULL DEFAULT '', + source TEXT NOT NULL DEFAULT '', + country TEXT NOT NULL DEFAULT '', + count INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (day, site_id, attack_type, source, country) + ); + CREATE INDEX IF NOT EXISTS idx_daily_day ON daily_summary(day); `); // Migrations – silently ignored if columns already exist @@ -51,6 +62,42 @@ DB.exec(` try { DB.exec(`ALTER TABLE attacks ADD COLUMN ${col} TEXT NOT NULL DEFAULT ''`); } catch {} }); +// ── Retention: archive rows older than 30 days into daily_summary, then +// delete the raw rows. Matches bot-api's fix (2026-08-22) - same lesson: +// ORDER BY must be on the indexed received_at column, not id, or every +// batch forces a full table scan regardless of batch size. + +const PRUNE_AGE = 30 * 86400; +const PRUNE_BATCH = 200; + +const stmtCountOld = DB.prepare('SELECT COUNT(*) n FROM attacks WHERE received_at < ?'); +const stmtSummarizeBatch = DB.prepare(` + INSERT INTO daily_summary (day, site_id, attack_type, source, country, count) + SELECT strftime('%Y-%m-%d', received_at, 'unixepoch'), site_id, attack_type, source, country, COUNT(*) + FROM attacks + WHERE id IN (SELECT id FROM attacks WHERE received_at < ? ORDER BY received_at LIMIT ?) + GROUP BY 1, 2, 3, 4, 5 + ON CONFLICT (day, site_id, attack_type, source, country) + DO UPDATE SET count = count + excluded.count +`); +const stmtDeleteBatch = DB.prepare( + 'DELETE FROM attacks WHERE id IN (SELECT id FROM attacks WHERE received_at < ? ORDER BY received_at LIMIT ?)' +); + +function pruneOldRowsBatch(cutoff) { + if (stmtCountOld.get(cutoff).n === 0) return; + stmtSummarizeBatch.run(cutoff, PRUNE_BATCH); + stmtDeleteBatch.run(cutoff, PRUNE_BATCH); + setImmediate(() => pruneOldRowsBatch(cutoff)); +} + +function pruneOldRows() { + const cutoff = Math.floor(Date.now() / 1000) - PRUNE_AGE; + pruneOldRowsBatch(cutoff); +} +pruneOldRows(); +setInterval(pruneOldRows, 6 * 3600 * 1000); + // ── Auth ────────────────────────────────────────────────────────────────────── const API_TOKEN = (process.env.API_TOKEN || '').trim(); @@ -67,6 +114,11 @@ function requireToken(req, res, next) { } // ── IP geo-enrichment ───────────────────────────────────────────────────────── +// Off by default: this jail (attackapi, 10.20.0.19) has zero outbound +// internet access by design (see /etc/pf.conf's table), +// so every lookup would just burn its full timeout. Set GEO_ENRICH=1 in the +// environment for deployments that do have egress. +const GEO_ENRICH = process.env.GEO_ENRICH === '1'; const stmtEnrich = DB.prepare('UPDATE attacks SET country=?, asn=? WHERE id=?'); const enrichCache = new Map(); @@ -76,12 +128,12 @@ function isPrivateIP(ip) { } function enrichIP(rowId, ip) { - if (!ip || ip === '?' || isPrivateIP(ip)) return; + if (!GEO_ENRICH || !ip || ip === '?' || isPrivateIP(ip)) return; const now = Date.now(); if ((enrichCache.get(ip) || 0) > now) return; enrichCache.set(ip, now + 3_600_000); - http.get( + const req = http.get( `http://ip-api.com/json/${encodeURIComponent(ip)}?fields=status,countryCode,as`, { timeout: 5000 }, res => { @@ -100,16 +152,24 @@ function enrichIP(rowId, ip) { } catch {} }); } - ).on('error', () => enrichCache.delete(ip)); + ); + // Without an explicit abort, the `timeout` option only emits an event - + // it doesn't destroy the socket, so an unreachable ip-api.com leaks an + // open connection instead of cutting off at 5s (see bot-api's server.js + // for the sibling fix, 2026-08-22). + req.on('timeout', () => req.destroy()); + req.on('error', () => enrichCache.delete(ip)); } // Background enrichment of unenriched rows const stmtUnenriched = DB.prepare( "SELECT id, ip FROM attacks WHERE country='' AND ip != '' AND ip != '?' LIMIT 5" ); -setInterval(() => { - for (const row of stmtUnenriched.all()) enrichIP(row.id, row.ip); -}, 20_000); +if (GEO_ENRICH) { + setInterval(() => { + for (const row of stmtUnenriched.all()) enrichIP(row.id, row.ip); + }, 20_000); +} // ── Rate limiter ──────────────────────────────────────────────────────────────