feat(dashboard): cache overview in background

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

backend/apps/alerts/migrations/0003_alert_alert_event_time_idx.py

@@ -0,0 +1,22 @@
+# Generated by Django 6.0.6 on 2026-07-24 01:20
+
+import django.db.models.functions.comparison
+from django.contrib.postgres.operations import AddIndexConcurrently
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    atomic = False
+
+    dependencies = [
+        ('alerts', '0002_alert_alert_created_id_idx_and_more'),
+        ('artifacts', '0002_artifact_artifact_created_id_idx'),
+        ('cases', '0003_case_case_status_severity_idx_and_more'),
+    ]
+
+    operations = [
+        AddIndexConcurrently(
+            model_name='alert',
+            index=models.Index(django.db.models.functions.comparison.Coalesce('last_seen_time', 'first_seen_time', 'created_at'), name='alert_event_time_idx'),
+        ),
+    ]

backend/apps/alerts/models.py

@@ -1,4 +1,5 @@
 from django.db import models
+from django.db.models.functions import Coalesce
 
 from apps.common.models import BaseModel
 from apps.common.readable_ids import save_with_readable_id
@@ -225,6 +226,10 @@ class Alert(BaseModel):
         indexes = [
             models.Index(fields=["-created_at", "-id"], name="alert_created_id_idx"),
             models.Index(fields=["-first_seen_time", "-id"], name="alert_first_seen_id_idx"),
+            models.Index(
+                Coalesce("last_seen_time", "first_seen_time", "created_at"),
+                name="alert_event_time_idx",
+            ),
         ]
 
     def __str__(self):

backend/apps/cases/migrations/0003_case_case_status_severity_idx_and_more.py

@@ -0,0 +1,33 @@
+# Generated by Django 6.0.6 on 2026-07-24 01:20
+
+from django.conf import settings
+from django.contrib.postgres.operations import AddIndexConcurrently
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    atomic = False
+
+    dependencies = [
+        ('cases', '0002_case_case_created_id_idx'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        AddIndexConcurrently(
+            model_name='case',
+            index=models.Index(fields=['status', 'severity'], name='case_status_severity_idx'),
+        ),
+        AddIndexConcurrently(
+            model_name='case',
+            index=models.Index(fields=['updated_at'], name='case_updated_at_idx'),
+        ),
+        AddIndexConcurrently(
+            model_name='case',
+            index=models.Index(fields=['acknowledged_time'], name='case_ack_time_idx'),
+        ),
+        AddIndexConcurrently(
+            model_name='case',
+            index=models.Index(fields=['closed_time'], name='case_closed_time_idx'),
+        ),
+    ]

backend/apps/cases/models.py

@@ -117,6 +117,10 @@ class Case(BaseModel):
         ordering = ["-created_at"]
         indexes = [
             models.Index(fields=["-created_at", "-id"], name="case_created_id_idx"),
+            models.Index(fields=["status", "severity"], name="case_status_severity_idx"),
+            models.Index(fields=["updated_at"], name="case_updated_at_idx"),
+            models.Index(fields=["acknowledged_time"], name="case_ack_time_idx"),
+            models.Index(fields=["closed_time"], name="case_closed_time_idx"),
         ]
 
     def __str__(self):

backend/apps/common/logging.py

@@ -16,6 +16,7 @@ LOG_ROLE_FILES = {
     "agentic-case-analysis-worker": "agentic-case-analysis-worker.log",
     "agentic-module-worker": "agentic-module-worker.log",
     "elk-action-worker": "elk-action-worker.log",
+    "dashboard-cache-worker": "dashboard-cache-worker.log",
 }
 ROOT_PROCESS_FILE_LOGGERS = [""]
 SERVER_PROCESS_FILE_LOGGERS = {

backend/apps/dashboard/cache.py

@@ -0,0 +1,76 @@
+import logging
+
+from django.core.cache import cache
+from django.utils import timezone
+from django.utils.dateparse import parse_datetime
+from django_redis import get_redis_connection
+from redis.exceptions import LockNotOwnedError
+
+
+logger = logging.getLogger(__name__)
+
+DASHBOARD_CACHE_KEY = "dashboard:overview:v1:{window}"
+DASHBOARD_REFRESH_LOCK_KEY = "dashboard:overview:refresh:v1:{window}"
+DASHBOARD_REFRESH_LOCK_TIMEOUT_SECONDS = 600
+DASHBOARD_STALE_WARNING_INTERVALS = 3
+
+
+def _cache_key(window):
+    return DASHBOARD_CACHE_KEY.format(window=window)
+
+
+def set_cached_dashboard_overview(window, overview, refresh_interval_seconds):
+    cache.set(
+        _cache_key(window),
+        {
+            "overview": overview,
+            "refreshed_at": timezone.now().isoformat(),
+            "refresh_interval_seconds": int(refresh_interval_seconds),
+        },
+        timeout=None,
+    )
+
+
+def get_cached_dashboard_overview(window):
+    snapshot = cache.get(_cache_key(window))
+    if snapshot is None:
+        return None
+
+    refreshed_at = parse_datetime(snapshot["refreshed_at"])
+    if refreshed_at is None:
+        raise ValueError(f"Invalid dashboard cache timestamp for window {window}.")
+
+    refresh_interval_seconds = int(snapshot["refresh_interval_seconds"])
+    age_seconds = max(0, int((timezone.now() - refreshed_at).total_seconds()))
+    overview = dict(snapshot["overview"])
+    overview["cache"] = {
+        "generated_at": overview["generated_at"],
+        "refreshed_at": refreshed_at.isoformat(),
+        "refresh_interval_seconds": refresh_interval_seconds,
+        "age_seconds": age_seconds,
+        "stale_warning": age_seconds > refresh_interval_seconds * DASHBOARD_STALE_WARNING_INTERVALS,
+    }
+    return overview
+
+
+def refresh_cached_dashboard_overview(window, refresh_interval_seconds):
+    connection = get_redis_connection("default")
+    lock = connection.lock(
+        DASHBOARD_REFRESH_LOCK_KEY.format(window=window),
+        timeout=DASHBOARD_REFRESH_LOCK_TIMEOUT_SECONDS,
+        blocking_timeout=0,
+    )
+    if not lock.acquire(blocking=False):
+        return False
+
+    try:
+        from .views import build_dashboard_overview
+
+        overview = build_dashboard_overview(window)
+        set_cached_dashboard_overview(window, overview, refresh_interval_seconds)
+        return True
+    finally:
+        try:
+            lock.release()
+        except LockNotOwnedError:
+            logger.warning("Dashboard refresh lock expired before release: window=%s", window)

backend/apps/dashboard/management/commands/run_dashboard_cache_worker.py

@@ -0,0 +1,85 @@
+import logging
+import time
+
+from django.core.management.base import BaseCommand
+
+from apps.common.worker_runner import SLEEP_ALWAYS, WorkerIterationResult, add_worker_arguments, run_worker
+from apps.dashboard.cache import refresh_cached_dashboard_overview
+from apps.dashboard.views import WINDOW_DELTAS
+from apps.settings.runtime_config import get_dashboard_refresh_interval_seconds
+
+
+logger = logging.getLogger(__name__)
+
+DASHBOARD_REFRESH_WARNING_SECONDS = 60
+
+
+class Command(BaseCommand):
+    help = "Refresh the Redis-backed dashboard overview cache."
+
+    def add_arguments(self, parser):
+        add_worker_arguments(
+            parser,
+            interval_help="Seconds between refreshes. Defaults to the Runtime Settings value.",
+        )
+
+    def handle(self, *args, **options):
+        next_sleep_seconds = None
+
+        def effective_interval():
+            return options["interval"] or get_dashboard_refresh_interval_seconds()
+
+        def refresh_once():
+            nonlocal next_sleep_seconds
+            interval = effective_interval()
+            started = time.perf_counter()
+            refreshed = []
+            skipped = []
+            failed = []
+
+            for window in WINDOW_DELTAS:
+                window_started = time.perf_counter()
+                try:
+                    if refresh_cached_dashboard_overview(window, interval):
+                        refreshed.append(window)
+                        logger.info(
+                            "Dashboard cache refreshed: window=%s duration_ms=%.2f",
+                            window,
+                            (time.perf_counter() - window_started) * 1000,
+                        )
+                    else:
+                        skipped.append(window)
+                        logger.info("Dashboard cache refresh skipped because lock is held: window=%s", window)
+                except Exception as exc:  # noqa: BLE001 - each window refresh must fail independently.
+                    failed.append(window)
+                    logger.exception("Dashboard cache refresh failed: window=%s error=%s", window, exc)
+
+            duration_seconds = time.perf_counter() - started
+            if duration_seconds > DASHBOARD_REFRESH_WARNING_SECONDS:
+                logger.warning(
+                    "Dashboard cache refresh exceeded target: duration_seconds=%.2f target_seconds=%s",
+                    duration_seconds,
+                    DASHBOARD_REFRESH_WARNING_SECONDS,
+                )
+            next_sleep_seconds = min(60, interval) if failed else interval
+
+            return WorkerIterationResult(
+                processed=bool(refreshed),
+                message=(
+                    f"Dashboard cache refresh completed in {duration_seconds:.2f}s; "
+                    f"refreshed={','.join(refreshed) or 'none'}; "
+                    f"skipped={','.join(skipped) or 'none'}; "
+                    f"failed={','.join(failed) or 'none'}."
+                ),
+            )
+
+        run_worker(
+            self,
+            options=options,
+            worker_name="dashboard cache",
+            run_once=refresh_once,
+            default_interval=get_dashboard_refresh_interval_seconds,
+            sleep_policy=SLEEP_ALWAYS,
+            sleep_seconds=lambda: next_sleep_seconds or effective_interval(),
+            log_role="dashboard-cache-worker",
+        )

backend/apps/dashboard/views.py

@@ -1,3 +1,4 @@
+import logging
 import re
 from collections import Counter
 from datetime import timedelta
@@ -6,6 +7,8 @@ from django.db import connection
 from django.db.models import Case as DbCase, Count, DateTimeField, FloatField, Min, Q, Sum, Value, When
 from django.db.models.functions import Coalesce, TruncDay, TruncHour
 from django.utils import timezone
+from django_redis.exceptions import ConnectionInterrupted
+from redis.exceptions import RedisError
 from rest_framework import permissions, status
 from rest_framework.response import Response
 from rest_framework.views import APIView
@@ -16,8 +19,11 @@ from apps.cases.models import Case, CaseStatus
 from apps.enrichments.models import Enrichment
 from apps.knowledge.models import Knowledge, KnowledgeSource
 from apps.playbooks.models import Playbook, PlaybookJobStatus
+from .cache import get_cached_dashboard_overview
 
 
+logger = logging.getLogger(__name__)
+
 WINDOW_DELTAS = {
     "24h": timedelta(hours=24),
     "7d": timedelta(days=7),
@@ -523,4 +529,17 @@ class DashboardOverviewView(APIView):
                 status=status.HTTP_400_BAD_REQUEST,
             )
 
-        return Response(build_dashboard_overview(window))
+        try:
+            overview = get_cached_dashboard_overview(window)
+        except (ConnectionInterrupted, RedisError, KeyError, TypeError, ValueError):
+            logger.exception("Dashboard cache read failed: window=%s", window)
+            return Response(
+                {"detail": "Dashboard cache is temporarily unavailable."},
+                status=status.HTTP_503_SERVICE_UNAVAILABLE,
+            )
+        if overview is None:
+            return Response(
+                {"detail": "Dashboard cache is not ready. Wait for the background refresh worker."},
+                status=status.HTTP_503_SERVICE_UNAVAILABLE,
+            )
+        return Response(overview)

backend/apps/enrichments/migrations/0003_enrichment_enrichment_created_idx.py

@@ -0,0 +1,22 @@
+# Generated by Django 6.0.6 on 2026-07-24 01:20
+
+from django.contrib.postgres.operations import AddIndexConcurrently
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    atomic = False
+
+    dependencies = [
+        ('alerts', '0003_alert_alert_event_time_idx'),
+        ('artifacts', '0002_artifact_artifact_created_id_idx'),
+        ('cases', '0003_case_case_status_severity_idx_and_more'),
+        ('enrichments', '0002_remove_mcp_provider_choice'),
+    ]
+
+    operations = [
+        AddIndexConcurrently(
+            model_name='enrichment',
+            index=models.Index(fields=['created_at'], name='enrichment_created_idx'),
+        ),
+    ]

backend/apps/enrichments/models.py

@@ -185,6 +185,9 @@ class Enrichment(BaseModel):
     class Meta:
         db_table = "enrichments"
         ordering = ["-created_at"]
+        indexes = [
+            models.Index(fields=["created_at"], name="enrichment_created_idx"),
+        ]
 
     def save(self, *args, **kwargs):
         return save_with_readable_id(self, "enrichment_id", "enrichment", *args, **kwargs)

backend/apps/knowledge/migrations/0002_knowledge_knowledge_created_src_idx.py

@@ -0,0 +1,20 @@
+# Generated by Django 6.0.6 on 2026-07-24 01:20
+
+from django.contrib.postgres.operations import AddIndexConcurrently
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    atomic = False
+
+    dependencies = [
+        ('cases', '0003_case_case_status_severity_idx_and_more'),
+        ('knowledge', '0001_initial'),
+    ]
+
+    operations = [
+        AddIndexConcurrently(
+            model_name='knowledge',
+            index=models.Index(fields=['created_at', 'source'], name='knowledge_created_src_idx'),
+        ),
+    ]

backend/apps/knowledge/models.py

@@ -29,6 +29,9 @@ class Knowledge(BaseModel):
     class Meta:
         db_table = "knowledge"
         ordering = ["-created_at"]
+        indexes = [
+            models.Index(fields=["created_at", "source"], name="knowledge_created_src_idx"),
+        ]
 
     def save(self, *args, **kwargs):
         return save_with_readable_id(self, "knowledge_id", "knowledge", *args, **kwargs)

backend/apps/playbooks/migrations/0002_playbook_playbook_created_job_idx.py

@@ -0,0 +1,22 @@
+# Generated by Django 6.0.6 on 2026-07-24 01:20
+
+from django.conf import settings
+from django.contrib.postgres.operations import AddIndexConcurrently
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    atomic = False
+
+    dependencies = [
+        ('cases', '0003_case_case_status_severity_idx_and_more'),
+        ('playbooks', '0001_initial'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        AddIndexConcurrently(
+            model_name='playbook',
+            index=models.Index(fields=['created_at', 'job_status'], name='playbook_created_job_idx'),
+        ),
+    ]

backend/apps/playbooks/models.py

@@ -32,6 +32,9 @@ class Playbook(BaseModel):
     class Meta:
         db_table = "playbooks"
         ordering = ["-created_at"]
+        indexes = [
+            models.Index(fields=["created_at", "job_status"], name="playbook_created_job_idx"),
+        ]
 
     def save(self, *args, **kwargs):
         return save_with_readable_id(self, "playbook_id", "playbook", *args, **kwargs)

backend/apps/settings/migrations/0004_runtimeconfig_dashboard_refresh_interval_seconds.py

@@ -0,0 +1,18 @@
+# Generated by Django 6.0.6 on 2026-07-24 01:20
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('settings', '0003_remove_siemelkconfig_request_timeout_seconds_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='runtimeconfig',
+            name='dashboard_refresh_interval_seconds',
+            field=models.PositiveIntegerField(default=300),
+        ),
+    ]

backend/apps/settings/models.py

@@ -142,6 +142,7 @@ class RuntimeConfig(models.Model):
     singleton_id = models.PositiveSmallIntegerField(default=1, unique=True, editable=False)
     prompt_language = models.CharField(max_length=10, default="en")
     stream_maxlen = models.PositiveIntegerField(default=10000)
+    dashboard_refresh_interval_seconds = models.PositiveIntegerField(default=300)
     created_at = models.DateTimeField(auto_now_add=True)
     updated_at = models.DateTimeField(auto_now=True)
 

backend/apps/settings/runtime_config.py

@@ -100,6 +100,7 @@ def get_runtime_config():
     return {
         "prompt_language": config.prompt_language,
         "stream_maxlen": config.stream_maxlen,
+        "dashboard_refresh_interval_seconds": config.dashboard_refresh_interval_seconds,
     }
 
 
@@ -116,6 +117,10 @@ def get_stream_maxlen():
         raise
 
 
+def get_dashboard_refresh_interval_seconds():
+    return get_runtime_config()["dashboard_refresh_interval_seconds"]
+
+
 def invalidate(group=None):
     if group in {None, "llm"}:
         get_llm_configs.cache_clear()

backend/apps/settings/serializers.py

@@ -381,11 +381,14 @@ class LdapConfigSerializer(serializers.ModelSerializer):
 
 
 class RuntimeConfigSerializer(serializers.ModelSerializer):
+    DASHBOARD_REFRESH_INTERVALS = {300, 900, 1800, 3600}
+
     class Meta:
         model = RuntimeConfig
         fields = (
             "prompt_language",
             "stream_maxlen",
+            "dashboard_refresh_interval_seconds",
             "updated_at",
         )
         read_only_fields = ("updated_at",)
@@ -400,3 +403,8 @@ class RuntimeConfigSerializer(serializers.ModelSerializer):
         if value <= 0:
             raise serializers.ValidationError("Stream maxlen must be greater than 0.")
         return value
+
+    def validate_dashboard_refresh_interval_seconds(self, value):
+        if value not in self.DASHBOARD_REFRESH_INTERVALS:
+            raise serializers.ValidationError("Dashboard refresh interval must be 300, 900, 1800, or 3600 seconds.")
+        return value

backend/apps/settings/views.py

@@ -62,6 +62,7 @@ LDAP_AUDIT_FIELDS = (
 RUNTIME_AUDIT_FIELDS = (
     "prompt_language",
     "stream_maxlen",
+    "dashboard_refresh_interval_seconds",
 )
 
 

backend/readme.md

@@ -18,6 +18,12 @@ For websocket development, also run the ASGI server:
 .\.venv\Scripts\python.exe -m uvicorn asp.asgi:application --host 127.0.0.1 --port 8001
 ```
 
+Run the dashboard cache worker in another terminal so dashboard snapshots stay available:
+
+```powershell
+.\.venv\Scripts\python.exe manage.py run_dashboard_cache_worker
+```
+
 ## Admin user
 
 ASP admin users are Django superusers and are maintained from the backend command line:

deploy/asp-compose/compose.yaml

@@ -101,6 +101,11 @@ services:
     restart: unless-stopped
     command: ["python", "manage.py", "run_elk_action_worker"]
 
+  asp-worker-dashboard-cache:
+    <<: *asp-backend
+    restart: unless-stopped
+    command: ["python", "manage.py", "run_dashboard_cache_worker"]
+
   asp-migrate:
     <<: *asp-backend
     restart: "no"

frontend/src/api/dashboard.ts

@@ -71,10 +71,19 @@ export interface DashboardHighlight {
   subtitle: string
 }
 
+export interface DashboardCacheMetadata {
+  generated_at: string
+  refreshed_at: string
+  refresh_interval_seconds: number
+  age_seconds: number
+  stale_warning: boolean
+}
+
 export interface DashboardOverview {
   window: DashboardWindow
   window_start: string
   generated_at: string
+  cache: DashboardCacheMetadata
   summary: DashboardSummary
   mean_times: {
     mttd: DashboardMeanTime

frontend/src/pages/Dashboard.tsx

@@ -142,7 +142,14 @@ function eventAccentColor(item: DashboardHighlight) {
 
 function generatedAtLabel(value: string | undefined) {
   if (!value) return 'Waiting for telemetry'
-  return `Last refresh ${new Date(value).toLocaleString()}`
+  return `Data generated ${new Date(value).toLocaleString()}`
+}
+
+function cacheAgeLabel(seconds: number) {
+  if (seconds < 60) return `${seconds}s`
+  if (seconds < 3600) return `${Math.floor(seconds / 60)}m`
+  if (seconds < 86400) return `${Math.floor(seconds / 3600)}h`
+  return `${Math.floor(seconds / 86400)}d`
 }
 
 function CyberEmpty({ title = 'Signal Quiet', description = 'No telemetry in this window.' }: { title?: string; description?: string }) {
@@ -691,8 +698,11 @@ export default function Dashboard() {
     try {
       const overview = await fetchDashboardOverview(targetWindow)
       setData(overview)
-    } catch {
-      setError('Telemetry link degraded. Showing last known dashboard frame.')
+    } catch (requestError: unknown) {
+      const responseStatus = (requestError as { response?: { status?: number } }).response?.status
+      setError(responseStatus === 503
+        ? 'Dashboard cache is preparing. Data will appear after the background worker completes its first refresh.'
+        : 'Telemetry link degraded. Showing the last known dashboard frame when available.')
     } finally {
       setLoading(false)
     }
@@ -703,6 +713,14 @@ export default function Dashboard() {
     void loadOverview(selectedWindow)
   }, [loadOverview, selectedWindow])
 
+  useEffect(() => {
+    const intervalSeconds = data?.cache.refresh_interval_seconds ?? 300
+    const timer = window.setInterval(() => {
+      void loadOverview(selectedWindow)
+    }, intervalSeconds * 1000)
+    return () => window.clearInterval(timer)
+  }, [data?.cache.refresh_interval_seconds, loadOverview, selectedWindow])
+
   const summary = data?.summary
   const riskLevel = getRiskPostureLevel(summary?.active_risk_index)
 
@@ -721,6 +739,14 @@ export default function Dashboard() {
             </div>
             <div className="dashboard-hero-actions">
               {error && <Alert className="dashboard-soft-alert" type="warning" title={error} showIcon />}
+              {data?.cache.stale_warning && (
+                <Alert
+                  className="dashboard-soft-alert"
+                  type="warning"
+                  title={`Dashboard data is stale. Last successful refresh was ${cacheAgeLabel(data.cache.age_seconds)} ago.`}
+                  showIcon
+                />
+              )}
               <div className="dashboard-control-panel">
                 <Text className="dashboard-refresh-time">{generatedAtLabel(data?.generated_at)}</Text>
                 <div className="dashboard-control-row">

frontend/src/pages/RuntimeSettings.tsx

@@ -7,6 +7,7 @@ import client from '../api/client'
 interface RuntimeConfig {
   prompt_language: 'en' | 'zh'
   stream_maxlen: number
+  dashboard_refresh_interval_seconds: 300 | 900 | 1800 | 3600
   updated_at?: string
 }
 
@@ -14,6 +15,7 @@ function initialValues(): RuntimeConfig {
   return {
     prompt_language: 'en',
     stream_maxlen: 10000,
+    dashboard_refresh_interval_seconds: 300,
   }
 }
 
@@ -111,6 +113,25 @@ export default function RuntimeSettings() {
               </Form.Item>
             </Col>
           </Row>
+
+          <Typography.Text strong>Dashboard Runtime</Typography.Text>
+          <Divider style={{ margin: '8px 0 16px' }} />
+          <Row gutter={16}>
+            <Col span={12}>
+              <Form.Item
+                name="dashboard_refresh_interval_seconds"
+                label={helpLabel('Dashboard Refresh Interval', 'Controls how often the background worker recalculates the 24h, 7d, and 30d dashboard caches.')}
+                rules={[{ required: true }]}
+              >
+                <Select options={[
+                  { label: '5 minutes', value: 300 },
+                  { label: '15 minutes', value: 900 },
+                  { label: '30 minutes', value: 1800 },
+                  { label: '60 minutes', value: 3600 },
+                ]} />
+              </Form.Item>
+            </Col>
+          </Row>
           <Space>
             <Button type="primary" onClick={saveConfig} loading={saving}>Save</Button>
           </Space>