feat: store raw UA strings, add separate Top User Agents panel

- Add user_agent column to bots table (migration-safe)
- Store raw UA string (up to 300 chars) alongside ua_family on insert
- selfObserve stores raw UA from incoming request headers
- getStats() adds top_user_agents query (top 15 by count, last 30d)
- Dashboard: revert actions+reasons to 2-col, remove embedded UA col
- Dashboard: new separate panel below actions+reasons showing raw UA
  strings with hit counts in monospace, truncated with title tooltip

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-04-10 09:00:59 +02:00
parent 85a524784c
commit 379e993384
2 changed files with 44 additions and 14 deletions

View File

@@ -44,7 +44,7 @@ DB.exec(`
`);
// Migrations silently ignored if columns already exist
['country', 'asn', 'request_uri'].forEach(col => {
['country', 'asn', 'request_uri', 'user_agent'].forEach(col => {
try { DB.exec(`ALTER TABLE bots ADD COLUMN ${col} TEXT NOT NULL DEFAULT ''`); } catch {}
});
@@ -190,6 +190,11 @@ function getStats() {
FROM bots WHERE received_at > ?
GROUP BY ua_family ORDER BY hits DESC LIMIT 8
`).all(now - 2592000),
top_user_agents: DB.prepare(`
SELECT user_agent ua, COUNT(*) hits
FROM bots WHERE received_at > ? AND user_agent != ''
GROUP BY user_agent ORDER BY hits DESC LIMIT 15
`).all(now - 2592000),
recent: DB.prepare(`
SELECT received_at, ip_masked ip, country, bot_type, action, reason, ua_family, site_id
FROM bots ORDER BY id DESC LIMIT 40
@@ -221,8 +226,8 @@ setInterval(() => {
// ── Prepared statements ───────────────────────────────────────────────────────
const stmtIns = DB.prepare(`
INSERT INTO bots (received_at, site_id, ip_masked, bot_type, action, reason, ua_family, request_uri, country, asn)
VALUES (?,?,?,?,?,?,?,?,?,?)
INSERT INTO bots (received_at, site_id, ip_masked, bot_type, action, reason, ua_family, request_uri, country, asn, user_agent)
VALUES (?,?,?,?,?,?,?,?,?,?,?)
`);
const stmtSite = DB.prepare(`
INSERT INTO sites (site_id, first_seen, last_seen, block_count) VALUES (?,?,?,?)
@@ -244,7 +249,8 @@ const insertBatch = DB.transaction((siteId, bots) => {
String(b.reason || '').slice(0, 255),
parseUA(b.user_agent || ''),
String(b.request_uri || '').slice(0, 500),
'', '' // country/asn filled async
'', '', // country/asn filled async
String(b.user_agent || '').slice(0, 300)
);
ids.push({ id: Number(r.lastInsertRowid), ip });
}
@@ -274,7 +280,7 @@ function selfObserve(req, res, next) {
try {
const r = stmtIns.run(
now, 'self', ip, fam, 'observed', 'Direct API visitor', fam, req.path, '', ''
now, 'self', ip, fam, 'observed', 'Direct API visitor', fam, req.path, '', '', ua.slice(0, 300)
);
_cache = null;
setImmediate(() => enrichIP(Number(r.lastInsertRowid), ip));