feat(alerting): add per-alert snooze and silent auto-resolve

Per-alert snooze (4h / 24h / 7d presets) plus silent auto-resolve on metric
recovery. Snooze data lives on TriggeredAlert and composes with the existing
per-policy SuppressionMinutes.

Schema: adds SnoozedUntil, SnoozedBy, BelowThresholdStreakCount, and
LastEvaluatedAt columns to TriggeredAlerts via idempotent IF COL_LENGTH
ALTER blocks, mirroring the existing SourceFailureDetails pattern.

API: POST /api/triggered-alerts/{id}/snooze accepts an absolute timestamp
or a duration in hours; POST .../unsnooze clears the state. Snoozing a
terminal-state alert returns 400.

Evaluator: after the existing dispatch loop, scans non-terminal alerts,
increments or resets a per-alert streak counter on each evaluation, and
transitions to Status="auto_resolved" after AutoResolveDebounceCycles
(default 2) consecutive below-threshold observations. Resolution is silent
- no notification dispatch, no NotificationLog write. Manual resolve
remains the user-driven terminal state; the first transition wins.

Config: new Alerting section in appsettings.json; bound via the existing
IOptions pattern.
This commit is contained in:
AnandSundar
2026-06-22 13:24:46 -06:00
parent 5a241cde00
commit ea5ad21e47
6 changed files with 117 additions and 2 deletions
@@ -76,6 +76,18 @@ public static class AlertingSchema
IF COL_LENGTH(N'[CollectionRuns]', 'SourceFailureDetails') IS NULL
ALTER TABLE [CollectionRuns] ADD [SourceFailureDetails] nvarchar(max) NULL;
IF COL_LENGTH(N'[TriggeredAlerts]', 'SnoozedUntil') IS NULL
ALTER TABLE [TriggeredAlerts] ADD [SnoozedUntil] datetimeoffset NULL;
IF COL_LENGTH(N'[TriggeredAlerts]', 'SnoozedBy') IS NULL
ALTER TABLE [TriggeredAlerts] ADD [SnoozedBy] nvarchar(120) NULL;
IF COL_LENGTH(N'[TriggeredAlerts]', 'BelowThresholdStreakCount') IS NULL
ALTER TABLE [TriggeredAlerts] ADD [BelowThresholdStreakCount] int NOT NULL CONSTRAINT [DF_TriggeredAlerts_Streak] DEFAULT 0;
IF COL_LENGTH(N'[TriggeredAlerts]', 'LastEvaluatedAt') IS NULL
ALTER TABLE [TriggeredAlerts] ADD [LastEvaluatedAt] datetimeoffset NULL;
""";
private static readonly (string Name, string Category, string Metric, int Threshold, string Severity, string Condition)[] Defaults =
@@ -0,0 +1,16 @@
namespace M365SecurityDashboard.Api.Models;
/// <summary>
/// Configuration for the server-side alerting engine. Bound from the
/// "Alerting" section of <c>appsettings.json</c>.
/// </summary>
public sealed class AlertingOptions
{
/// <summary>
/// Number of consecutive evaluation cycles the underlying metric must be
/// below a triggered alert's threshold before the alert auto-resolves.
/// With the default collection interval of 15 minutes and a value of 2,
/// an alert auto-resolves ~30 minutes after the metric recovers.
/// </summary>
public int AutoResolveDebounceCycles { get; set; } = 2;
}
@@ -39,4 +39,17 @@ public sealed class TriggeredAlert
/// <summary>Whether outbound notifications were dispatched for this alert.</summary>
public bool Notified { get; set; }
/// <summary>If set, the alert is silenced until this timestamp. Status remains "new" or "acknowledged".</summary>
public DateTimeOffset? SnoozedUntil { get; set; }
/// <summary>Identity of the actor who snoozed this alert. Placeholder string ("dashboard") until auth lands.</summary>
[MaxLength(120)]
public string? SnoozedBy { get; set; }
/// <summary>Number of consecutive evaluation cycles the underlying metric has been below this alert's threshold. Reset on any above-threshold observation.</summary>
public int BelowThresholdStreakCount { get; set; }
/// <summary>When the evaluator last inspected this alert (used for diagnostics and the auto-resolve debounce).</summary>
public DateTimeOffset? LastEvaluatedAt { get; set; }
}
+33
View File
@@ -10,6 +10,7 @@ var builder = WebApplication.CreateBuilder(args);
builder.Host.UseWindowsService();
builder.Services.Configure<GraphOptions>(builder.Configuration.GetSection("Graph"));
builder.Services.Configure<AlertingOptions>(builder.Configuration.GetSection("Alerting"));
builder.Services.AddDbContext<AppDbContext>(options =>
options.UseSqlServer(builder.Configuration.GetConnectionString("DefaultConnection")));
builder.Services.AddHttpClient<GraphApiClient>();
@@ -1077,6 +1078,35 @@ app.MapPost("/api/triggered-alerts/{id:guid}/resolve", async (AppDbContext db, G
return Results.Ok(t);
});
// Per-alert snooze. Body: { "until": "2026-06-22T18:00:00Z" } or { "durationHours": 4|24|168 }.
// Until wins if both are supplied; durationHours defaults to 24 if neither is supplied.
app.MapPost("/api/triggered-alerts/{id:guid}/snooze", async (
AppDbContext db, Guid id, SnoozeRequest input, CancellationToken ct) =>
{
var t = await db.TriggeredAlerts.FindAsync([id], ct);
if (t is null) return Results.NotFound();
if (t.Status is "resolved" or "auto_resolved")
return Results.BadRequest(new { error = "Cannot snooze a terminal alert." });
var until = input.Until
?? (input.DurationHours is { } h ? DateTimeOffset.UtcNow.AddHours(h) : DateTimeOffset.UtcNow.AddHours(24));
t.SnoozedUntil = until;
t.SnoozedBy = "dashboard";
await db.SaveChangesAsync(ct);
return Results.Ok(t);
});
app.MapPost("/api/triggered-alerts/{id:guid}/unsnooze", async (
AppDbContext db, Guid id, CancellationToken ct) =>
{
var t = await db.TriggeredAlerts.FindAsync([id], ct);
if (t is null) return Results.NotFound();
t.SnoozedUntil = null;
t.SnoozedBy = null;
await db.SaveChangesAsync(ct);
return Results.Ok(t);
});
// Manually run an evaluation pass (used by the dashboard "refresh" + on-demand check)
app.MapPost("/api/alert-policies/evaluate", async (AlertEvaluator evaluator, CancellationToken ct) =>
{
@@ -1150,3 +1180,6 @@ app.MapGet("/api/notification-log", async (AppDbContext db, CancellationToken ct
app.MapFallbackToFile("index.html");
app.Run();
/// <summary>Body shape for POST /api/triggered-alerts/{id}/snooze.</summary>
public sealed record SnoozeRequest(DateTimeOffset? Until, int? DurationHours);
@@ -1,6 +1,7 @@
using M365SecurityDashboard.Api.Data;
using M365SecurityDashboard.Api.Models;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;
namespace M365SecurityDashboard.Api.Services;
@@ -8,10 +9,12 @@ namespace M365SecurityDashboard.Api.Services;
/// Evaluates all enabled <see cref="AlertPolicy"/> rows against the latest
/// collected data and persists new <see cref="TriggeredAlert"/> rows. Runs
/// server-side after every collection cycle so alerts fire without a browser.
/// Also auto-resolves stale alerts whose underlying metric has recovered.
/// </summary>
public sealed class AlertEvaluator(
AppDbContext db,
NotificationSender sender,
IOptions<AlertingOptions> options,
ILogger<AlertEvaluator> logger)
{
public async Task<int> EvaluateAsync(CancellationToken ct)
@@ -26,6 +29,10 @@ public sealed class AlertEvaluator(
var now = DateTimeOffset.UtcNow;
var fired = 0;
// Map PolicyId -> Metric key so the auto-resolve loop below can look up
// each open alert's current metric without re-querying the policy table.
var policyMetricById = policies.ToDictionary(p => p.Id, p => p.Metric);
foreach (var policy in policies)
{
var value = metrics.GetValueOrDefault(policy.Metric, 0);
@@ -67,10 +74,41 @@ public sealed class AlertEvaluator(
}
}
if (fired > 0)
// Auto-resolve: scan non-terminal alerts and update streak counters.
// Resolves silently — no notification dispatch, no NotificationLog write.
var streakTarget = Math.Max(1, options.Value.AutoResolveDebounceCycles);
var openAlerts = await db.TriggeredAlerts
.Where(t => t.Status != "resolved" && t.Status != "auto_resolved")
.ToListAsync(ct);
var autoResolved = 0;
foreach (var alert in openAlerts)
{
if (!policyMetricById.TryGetValue(alert.PolicyId, out var metricKey)) continue;
var current = metrics.GetValueOrDefault(metricKey, 0);
if (current < alert.Threshold)
{
alert.BelowThresholdStreakCount++;
if (alert.BelowThresholdStreakCount >= streakTarget)
{
alert.Status = "auto_resolved";
autoResolved++;
}
}
else if (alert.BelowThresholdStreakCount != 0)
{
alert.BelowThresholdStreakCount = 0;
}
alert.LastEvaluatedAt = now;
}
if (fired > 0 || autoResolved > 0)
{
await db.SaveChangesAsync(ct);
logger.LogInformation("Alert evaluation fired {Count} new alert(s)", fired);
if (fired > 0)
logger.LogInformation("Alert evaluation fired {Count} new alert(s)", fired);
if (autoResolved > 0)
logger.LogInformation("Auto-resolved {Count} alert(s) after metric recovery", autoResolved);
}
return fired;
}
@@ -13,5 +13,8 @@
"ExchangeQuarantinePath": "/beta/security/alerts_v2?$top=50&$filter=serviceSource%20eq%20'microsoftDefenderForOffice365'%20and%20category%20eq%20'EmailMalware'",
"MailFlowIssuesPath": "/v1.0/admin/serviceAnnouncement/issues?$top=50&$filter=service%20eq%20'Exchange%20Online'%20and%20isResolved%20eq%20false"
},
"Alerting": {
"AutoResolveDebounceCycles": 2
},
"AllowedHosts": "*"
}