diff --git a/backend/apps/alerts/migrations/0003_alert_alert_event_time_idx.py b/backend/apps/alerts/migrations/0003_alert_alert_event_time_idx.py new file mode 100644 index 0000000..9952767 --- /dev/null +++ b/backend/apps/alerts/migrations/0003_alert_alert_event_time_idx.py @@ -0,0 +1,22 @@ +# Generated by Django 6.0.6 on 2026-07-24 01:20 + +import django.db.models.functions.comparison +from django.contrib.postgres.operations import AddIndexConcurrently +from django.db import migrations, models + + +class Migration(migrations.Migration): + atomic = False + + dependencies = [ + ('alerts', '0002_alert_alert_created_id_idx_and_more'), + ('artifacts', '0002_artifact_artifact_created_id_idx'), + ('cases', '0003_case_case_status_severity_idx_and_more'), + ] + + operations = [ + AddIndexConcurrently( + model_name='alert', + index=models.Index(django.db.models.functions.comparison.Coalesce('last_seen_time', 'first_seen_time', 'created_at'), name='alert_event_time_idx'), + ), + ] diff --git a/backend/apps/alerts/models.py b/backend/apps/alerts/models.py index c20264b..110b405 100644 --- a/backend/apps/alerts/models.py +++ b/backend/apps/alerts/models.py @@ -1,4 +1,5 @@ from django.db import models +from django.db.models.functions import Coalesce from apps.common.models import BaseModel from apps.common.readable_ids import save_with_readable_id @@ -225,6 +226,10 @@ class Alert(BaseModel): indexes = [ models.Index(fields=["-created_at", "-id"], name="alert_created_id_idx"), models.Index(fields=["-first_seen_time", "-id"], name="alert_first_seen_id_idx"), + models.Index( + Coalesce("last_seen_time", "first_seen_time", "created_at"), + name="alert_event_time_idx", + ), ] def __str__(self): diff --git a/backend/apps/cases/migrations/0003_case_case_status_severity_idx_and_more.py b/backend/apps/cases/migrations/0003_case_case_status_severity_idx_and_more.py new file mode 100644 index 0000000..ad27c34 --- /dev/null +++ b/backend/apps/cases/migrations/0003_case_case_status_severity_idx_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 6.0.6 on 2026-07-24 01:20 + +from django.conf import settings +from django.contrib.postgres.operations import AddIndexConcurrently +from django.db import migrations, models + + +class Migration(migrations.Migration): + atomic = False + + dependencies = [ + ('cases', '0002_case_case_created_id_idx'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + AddIndexConcurrently( + model_name='case', + index=models.Index(fields=['status', 'severity'], name='case_status_severity_idx'), + ), + AddIndexConcurrently( + model_name='case', + index=models.Index(fields=['updated_at'], name='case_updated_at_idx'), + ), + AddIndexConcurrently( + model_name='case', + index=models.Index(fields=['acknowledged_time'], name='case_ack_time_idx'), + ), + AddIndexConcurrently( + model_name='case', + index=models.Index(fields=['closed_time'], name='case_closed_time_idx'), + ), + ] diff --git a/backend/apps/cases/models.py b/backend/apps/cases/models.py index ebffe5f..8b75649 100644 --- a/backend/apps/cases/models.py +++ b/backend/apps/cases/models.py @@ -117,6 +117,10 @@ class Case(BaseModel): ordering = ["-created_at"] indexes = [ models.Index(fields=["-created_at", "-id"], name="case_created_id_idx"), + models.Index(fields=["status", "severity"], name="case_status_severity_idx"), + models.Index(fields=["updated_at"], name="case_updated_at_idx"), + models.Index(fields=["acknowledged_time"], name="case_ack_time_idx"), + models.Index(fields=["closed_time"], name="case_closed_time_idx"), ] def __str__(self): diff --git a/backend/apps/common/logging.py b/backend/apps/common/logging.py index 358e037..1743310 100644 --- a/backend/apps/common/logging.py +++ b/backend/apps/common/logging.py @@ -16,6 +16,7 @@ LOG_ROLE_FILES = { "agentic-case-analysis-worker": "agentic-case-analysis-worker.log", "agentic-module-worker": "agentic-module-worker.log", "elk-action-worker": "elk-action-worker.log", + "dashboard-cache-worker": "dashboard-cache-worker.log", } ROOT_PROCESS_FILE_LOGGERS = [""] SERVER_PROCESS_FILE_LOGGERS = { diff --git a/backend/apps/dashboard/cache.py b/backend/apps/dashboard/cache.py new file mode 100644 index 0000000..881b90a --- /dev/null +++ b/backend/apps/dashboard/cache.py @@ -0,0 +1,76 @@ +import logging + +from django.core.cache import cache +from django.utils import timezone +from django.utils.dateparse import parse_datetime +from django_redis import get_redis_connection +from redis.exceptions import LockNotOwnedError + + +logger = logging.getLogger(__name__) + +DASHBOARD_CACHE_KEY = "dashboard:overview:v1:{window}" +DASHBOARD_REFRESH_LOCK_KEY = "dashboard:overview:refresh:v1:{window}" +DASHBOARD_REFRESH_LOCK_TIMEOUT_SECONDS = 600 +DASHBOARD_STALE_WARNING_INTERVALS = 3 + + +def _cache_key(window): + return DASHBOARD_CACHE_KEY.format(window=window) + + +def set_cached_dashboard_overview(window, overview, refresh_interval_seconds): + cache.set( + _cache_key(window), + { + "overview": overview, + "refreshed_at": timezone.now().isoformat(), + "refresh_interval_seconds": int(refresh_interval_seconds), + }, + timeout=None, + ) + + +def get_cached_dashboard_overview(window): + snapshot = cache.get(_cache_key(window)) + if snapshot is None: + return None + + refreshed_at = parse_datetime(snapshot["refreshed_at"]) + if refreshed_at is None: + raise ValueError(f"Invalid dashboard cache timestamp for window {window}.") + + refresh_interval_seconds = int(snapshot["refresh_interval_seconds"]) + age_seconds = max(0, int((timezone.now() - refreshed_at).total_seconds())) + overview = dict(snapshot["overview"]) + overview["cache"] = { + "generated_at": overview["generated_at"], + "refreshed_at": refreshed_at.isoformat(), + "refresh_interval_seconds": refresh_interval_seconds, + "age_seconds": age_seconds, + "stale_warning": age_seconds > refresh_interval_seconds * DASHBOARD_STALE_WARNING_INTERVALS, + } + return overview + + +def refresh_cached_dashboard_overview(window, refresh_interval_seconds): + connection = get_redis_connection("default") + lock = connection.lock( + DASHBOARD_REFRESH_LOCK_KEY.format(window=window), + timeout=DASHBOARD_REFRESH_LOCK_TIMEOUT_SECONDS, + blocking_timeout=0, + ) + if not lock.acquire(blocking=False): + return False + + try: + from .views import build_dashboard_overview + + overview = build_dashboard_overview(window) + set_cached_dashboard_overview(window, overview, refresh_interval_seconds) + return True + finally: + try: + lock.release() + except LockNotOwnedError: + logger.warning("Dashboard refresh lock expired before release: window=%s", window) diff --git a/backend/apps/dashboard/management/__init__.py b/backend/apps/dashboard/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/apps/dashboard/management/commands/__init__.py b/backend/apps/dashboard/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/apps/dashboard/management/commands/run_dashboard_cache_worker.py b/backend/apps/dashboard/management/commands/run_dashboard_cache_worker.py new file mode 100644 index 0000000..deeedd1 --- /dev/null +++ b/backend/apps/dashboard/management/commands/run_dashboard_cache_worker.py @@ -0,0 +1,85 @@ +import logging +import time + +from django.core.management.base import BaseCommand + +from apps.common.worker_runner import SLEEP_ALWAYS, WorkerIterationResult, add_worker_arguments, run_worker +from apps.dashboard.cache import refresh_cached_dashboard_overview +from apps.dashboard.views import WINDOW_DELTAS +from apps.settings.runtime_config import get_dashboard_refresh_interval_seconds + + +logger = logging.getLogger(__name__) + +DASHBOARD_REFRESH_WARNING_SECONDS = 60 + + +class Command(BaseCommand): + help = "Refresh the Redis-backed dashboard overview cache." + + def add_arguments(self, parser): + add_worker_arguments( + parser, + interval_help="Seconds between refreshes. Defaults to the Runtime Settings value.", + ) + + def handle(self, *args, **options): + next_sleep_seconds = None + + def effective_interval(): + return options["interval"] or get_dashboard_refresh_interval_seconds() + + def refresh_once(): + nonlocal next_sleep_seconds + interval = effective_interval() + started = time.perf_counter() + refreshed = [] + skipped = [] + failed = [] + + for window in WINDOW_DELTAS: + window_started = time.perf_counter() + try: + if refresh_cached_dashboard_overview(window, interval): + refreshed.append(window) + logger.info( + "Dashboard cache refreshed: window=%s duration_ms=%.2f", + window, + (time.perf_counter() - window_started) * 1000, + ) + else: + skipped.append(window) + logger.info("Dashboard cache refresh skipped because lock is held: window=%s", window) + except Exception as exc: # noqa: BLE001 - each window refresh must fail independently. + failed.append(window) + logger.exception("Dashboard cache refresh failed: window=%s error=%s", window, exc) + + duration_seconds = time.perf_counter() - started + if duration_seconds > DASHBOARD_REFRESH_WARNING_SECONDS: + logger.warning( + "Dashboard cache refresh exceeded target: duration_seconds=%.2f target_seconds=%s", + duration_seconds, + DASHBOARD_REFRESH_WARNING_SECONDS, + ) + next_sleep_seconds = min(60, interval) if failed else interval + + return WorkerIterationResult( + processed=bool(refreshed), + message=( + f"Dashboard cache refresh completed in {duration_seconds:.2f}s; " + f"refreshed={','.join(refreshed) or 'none'}; " + f"skipped={','.join(skipped) or 'none'}; " + f"failed={','.join(failed) or 'none'}." + ), + ) + + run_worker( + self, + options=options, + worker_name="dashboard cache", + run_once=refresh_once, + default_interval=get_dashboard_refresh_interval_seconds, + sleep_policy=SLEEP_ALWAYS, + sleep_seconds=lambda: next_sleep_seconds or effective_interval(), + log_role="dashboard-cache-worker", + ) diff --git a/backend/apps/dashboard/views.py b/backend/apps/dashboard/views.py index 3ba19f1..f344299 100644 --- a/backend/apps/dashboard/views.py +++ b/backend/apps/dashboard/views.py @@ -1,3 +1,4 @@ +import logging import re from collections import Counter from datetime import timedelta @@ -6,6 +7,8 @@ from django.db import connection from django.db.models import Case as DbCase, Count, DateTimeField, FloatField, Min, Q, Sum, Value, When from django.db.models.functions import Coalesce, TruncDay, TruncHour from django.utils import timezone +from django_redis.exceptions import ConnectionInterrupted +from redis.exceptions import RedisError from rest_framework import permissions, status from rest_framework.response import Response from rest_framework.views import APIView @@ -16,8 +19,11 @@ from apps.cases.models import Case, CaseStatus from apps.enrichments.models import Enrichment from apps.knowledge.models import Knowledge, KnowledgeSource from apps.playbooks.models import Playbook, PlaybookJobStatus +from .cache import get_cached_dashboard_overview +logger = logging.getLogger(__name__) + WINDOW_DELTAS = { "24h": timedelta(hours=24), "7d": timedelta(days=7), @@ -523,4 +529,17 @@ class DashboardOverviewView(APIView): status=status.HTTP_400_BAD_REQUEST, ) - return Response(build_dashboard_overview(window)) + try: + overview = get_cached_dashboard_overview(window) + except (ConnectionInterrupted, RedisError, KeyError, TypeError, ValueError): + logger.exception("Dashboard cache read failed: window=%s", window) + return Response( + {"detail": "Dashboard cache is temporarily unavailable."}, + status=status.HTTP_503_SERVICE_UNAVAILABLE, + ) + if overview is None: + return Response( + {"detail": "Dashboard cache is not ready. Wait for the background refresh worker."}, + status=status.HTTP_503_SERVICE_UNAVAILABLE, + ) + return Response(overview) diff --git a/backend/apps/enrichments/migrations/0003_enrichment_enrichment_created_idx.py b/backend/apps/enrichments/migrations/0003_enrichment_enrichment_created_idx.py new file mode 100644 index 0000000..b481a6e --- /dev/null +++ b/backend/apps/enrichments/migrations/0003_enrichment_enrichment_created_idx.py @@ -0,0 +1,22 @@ +# Generated by Django 6.0.6 on 2026-07-24 01:20 + +from django.contrib.postgres.operations import AddIndexConcurrently +from django.db import migrations, models + + +class Migration(migrations.Migration): + atomic = False + + dependencies = [ + ('alerts', '0003_alert_alert_event_time_idx'), + ('artifacts', '0002_artifact_artifact_created_id_idx'), + ('cases', '0003_case_case_status_severity_idx_and_more'), + ('enrichments', '0002_remove_mcp_provider_choice'), + ] + + operations = [ + AddIndexConcurrently( + model_name='enrichment', + index=models.Index(fields=['created_at'], name='enrichment_created_idx'), + ), + ] diff --git a/backend/apps/enrichments/models.py b/backend/apps/enrichments/models.py index 163810e..98a0c54 100644 --- a/backend/apps/enrichments/models.py +++ b/backend/apps/enrichments/models.py @@ -185,6 +185,9 @@ class Enrichment(BaseModel): class Meta: db_table = "enrichments" ordering = ["-created_at"] + indexes = [ + models.Index(fields=["created_at"], name="enrichment_created_idx"), + ] def save(self, *args, **kwargs): return save_with_readable_id(self, "enrichment_id", "enrichment", *args, **kwargs) diff --git a/backend/apps/knowledge/migrations/0002_knowledge_knowledge_created_src_idx.py b/backend/apps/knowledge/migrations/0002_knowledge_knowledge_created_src_idx.py new file mode 100644 index 0000000..9adc62e --- /dev/null +++ b/backend/apps/knowledge/migrations/0002_knowledge_knowledge_created_src_idx.py @@ -0,0 +1,20 @@ +# Generated by Django 6.0.6 on 2026-07-24 01:20 + +from django.contrib.postgres.operations import AddIndexConcurrently +from django.db import migrations, models + + +class Migration(migrations.Migration): + atomic = False + + dependencies = [ + ('cases', '0003_case_case_status_severity_idx_and_more'), + ('knowledge', '0001_initial'), + ] + + operations = [ + AddIndexConcurrently( + model_name='knowledge', + index=models.Index(fields=['created_at', 'source'], name='knowledge_created_src_idx'), + ), + ] diff --git a/backend/apps/knowledge/models.py b/backend/apps/knowledge/models.py index f0342dc..7d58dbb 100644 --- a/backend/apps/knowledge/models.py +++ b/backend/apps/knowledge/models.py @@ -29,6 +29,9 @@ class Knowledge(BaseModel): class Meta: db_table = "knowledge" ordering = ["-created_at"] + indexes = [ + models.Index(fields=["created_at", "source"], name="knowledge_created_src_idx"), + ] def save(self, *args, **kwargs): return save_with_readable_id(self, "knowledge_id", "knowledge", *args, **kwargs) diff --git a/backend/apps/playbooks/migrations/0002_playbook_playbook_created_job_idx.py b/backend/apps/playbooks/migrations/0002_playbook_playbook_created_job_idx.py new file mode 100644 index 0000000..b6f040b --- /dev/null +++ b/backend/apps/playbooks/migrations/0002_playbook_playbook_created_job_idx.py @@ -0,0 +1,22 @@ +# Generated by Django 6.0.6 on 2026-07-24 01:20 + +from django.conf import settings +from django.contrib.postgres.operations import AddIndexConcurrently +from django.db import migrations, models + + +class Migration(migrations.Migration): + atomic = False + + dependencies = [ + ('cases', '0003_case_case_status_severity_idx_and_more'), + ('playbooks', '0001_initial'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + AddIndexConcurrently( + model_name='playbook', + index=models.Index(fields=['created_at', 'job_status'], name='playbook_created_job_idx'), + ), + ] diff --git a/backend/apps/playbooks/models.py b/backend/apps/playbooks/models.py index 477dc5c..821cd31 100644 --- a/backend/apps/playbooks/models.py +++ b/backend/apps/playbooks/models.py @@ -32,6 +32,9 @@ class Playbook(BaseModel): class Meta: db_table = "playbooks" ordering = ["-created_at"] + indexes = [ + models.Index(fields=["created_at", "job_status"], name="playbook_created_job_idx"), + ] def save(self, *args, **kwargs): return save_with_readable_id(self, "playbook_id", "playbook", *args, **kwargs) diff --git a/backend/apps/settings/migrations/0004_runtimeconfig_dashboard_refresh_interval_seconds.py b/backend/apps/settings/migrations/0004_runtimeconfig_dashboard_refresh_interval_seconds.py new file mode 100644 index 0000000..0c67b77 --- /dev/null +++ b/backend/apps/settings/migrations/0004_runtimeconfig_dashboard_refresh_interval_seconds.py @@ -0,0 +1,18 @@ +# Generated by Django 6.0.6 on 2026-07-24 01:20 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('settings', '0003_remove_siemelkconfig_request_timeout_seconds_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='runtimeconfig', + name='dashboard_refresh_interval_seconds', + field=models.PositiveIntegerField(default=300), + ), + ] diff --git a/backend/apps/settings/models.py b/backend/apps/settings/models.py index 292b86a..3766fde 100644 --- a/backend/apps/settings/models.py +++ b/backend/apps/settings/models.py @@ -142,6 +142,7 @@ class RuntimeConfig(models.Model): singleton_id = models.PositiveSmallIntegerField(default=1, unique=True, editable=False) prompt_language = models.CharField(max_length=10, default="en") stream_maxlen = models.PositiveIntegerField(default=10000) + dashboard_refresh_interval_seconds = models.PositiveIntegerField(default=300) created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) diff --git a/backend/apps/settings/runtime_config.py b/backend/apps/settings/runtime_config.py index dbdfa37..44b9df6 100644 --- a/backend/apps/settings/runtime_config.py +++ b/backend/apps/settings/runtime_config.py @@ -100,6 +100,7 @@ def get_runtime_config(): return { "prompt_language": config.prompt_language, "stream_maxlen": config.stream_maxlen, + "dashboard_refresh_interval_seconds": config.dashboard_refresh_interval_seconds, } @@ -116,6 +117,10 @@ def get_stream_maxlen(): raise +def get_dashboard_refresh_interval_seconds(): + return get_runtime_config()["dashboard_refresh_interval_seconds"] + + def invalidate(group=None): if group in {None, "llm"}: get_llm_configs.cache_clear() diff --git a/backend/apps/settings/serializers.py b/backend/apps/settings/serializers.py index 7602f1c..7fe5ddf 100644 --- a/backend/apps/settings/serializers.py +++ b/backend/apps/settings/serializers.py @@ -381,11 +381,14 @@ class LdapConfigSerializer(serializers.ModelSerializer): class RuntimeConfigSerializer(serializers.ModelSerializer): + DASHBOARD_REFRESH_INTERVALS = {300, 900, 1800, 3600} + class Meta: model = RuntimeConfig fields = ( "prompt_language", "stream_maxlen", + "dashboard_refresh_interval_seconds", "updated_at", ) read_only_fields = ("updated_at",) @@ -400,3 +403,8 @@ class RuntimeConfigSerializer(serializers.ModelSerializer): if value <= 0: raise serializers.ValidationError("Stream maxlen must be greater than 0.") return value + + def validate_dashboard_refresh_interval_seconds(self, value): + if value not in self.DASHBOARD_REFRESH_INTERVALS: + raise serializers.ValidationError("Dashboard refresh interval must be 300, 900, 1800, or 3600 seconds.") + return value diff --git a/backend/apps/settings/views.py b/backend/apps/settings/views.py index 649c215..251863b 100644 --- a/backend/apps/settings/views.py +++ b/backend/apps/settings/views.py @@ -62,6 +62,7 @@ LDAP_AUDIT_FIELDS = ( RUNTIME_AUDIT_FIELDS = ( "prompt_language", "stream_maxlen", + "dashboard_refresh_interval_seconds", ) diff --git a/backend/readme.md b/backend/readme.md index 38ce35d..dba6c61 100644 --- a/backend/readme.md +++ b/backend/readme.md @@ -18,6 +18,12 @@ For websocket development, also run the ASGI server: .\.venv\Scripts\python.exe -m uvicorn asp.asgi:application --host 127.0.0.1 --port 8001 ``` +Run the dashboard cache worker in another terminal so dashboard snapshots stay available: + +```powershell +.\.venv\Scripts\python.exe manage.py run_dashboard_cache_worker +``` + ## Admin user ASP admin users are Django superusers and are maintained from the backend command line: diff --git a/deploy/asp-compose/compose.yaml b/deploy/asp-compose/compose.yaml index 5b2159d..b122d21 100644 --- a/deploy/asp-compose/compose.yaml +++ b/deploy/asp-compose/compose.yaml @@ -101,6 +101,11 @@ services: restart: unless-stopped command: ["python", "manage.py", "run_elk_action_worker"] + asp-worker-dashboard-cache: + <<: *asp-backend + restart: unless-stopped + command: ["python", "manage.py", "run_dashboard_cache_worker"] + asp-migrate: <<: *asp-backend restart: "no" diff --git a/frontend/src/api/dashboard.ts b/frontend/src/api/dashboard.ts index f4662f9..4b755ee 100644 --- a/frontend/src/api/dashboard.ts +++ b/frontend/src/api/dashboard.ts @@ -71,10 +71,19 @@ export interface DashboardHighlight { subtitle: string } +export interface DashboardCacheMetadata { + generated_at: string + refreshed_at: string + refresh_interval_seconds: number + age_seconds: number + stale_warning: boolean +} + export interface DashboardOverview { window: DashboardWindow window_start: string generated_at: string + cache: DashboardCacheMetadata summary: DashboardSummary mean_times: { mttd: DashboardMeanTime diff --git a/frontend/src/pages/Dashboard.tsx b/frontend/src/pages/Dashboard.tsx index 53da91b..af228b1 100644 --- a/frontend/src/pages/Dashboard.tsx +++ b/frontend/src/pages/Dashboard.tsx @@ -142,7 +142,14 @@ function eventAccentColor(item: DashboardHighlight) { function generatedAtLabel(value: string | undefined) { if (!value) return 'Waiting for telemetry' - return `Last refresh ${new Date(value).toLocaleString()}` + return `Data generated ${new Date(value).toLocaleString()}` +} + +function cacheAgeLabel(seconds: number) { + if (seconds < 60) return `${seconds}s` + if (seconds < 3600) return `${Math.floor(seconds / 60)}m` + if (seconds < 86400) return `${Math.floor(seconds / 3600)}h` + return `${Math.floor(seconds / 86400)}d` } function CyberEmpty({ title = 'Signal Quiet', description = 'No telemetry in this window.' }: { title?: string; description?: string }) { @@ -691,8 +698,11 @@ export default function Dashboard() { try { const overview = await fetchDashboardOverview(targetWindow) setData(overview) - } catch { - setError('Telemetry link degraded. Showing last known dashboard frame.') + } catch (requestError: unknown) { + const responseStatus = (requestError as { response?: { status?: number } }).response?.status + setError(responseStatus === 503 + ? 'Dashboard cache is preparing. Data will appear after the background worker completes its first refresh.' + : 'Telemetry link degraded. Showing the last known dashboard frame when available.') } finally { setLoading(false) } @@ -703,6 +713,14 @@ export default function Dashboard() { void loadOverview(selectedWindow) }, [loadOverview, selectedWindow]) + useEffect(() => { + const intervalSeconds = data?.cache.refresh_interval_seconds ?? 300 + const timer = window.setInterval(() => { + void loadOverview(selectedWindow) + }, intervalSeconds * 1000) + return () => window.clearInterval(timer) + }, [data?.cache.refresh_interval_seconds, loadOverview, selectedWindow]) + const summary = data?.summary const riskLevel = getRiskPostureLevel(summary?.active_risk_index) @@ -721,6 +739,14 @@ export default function Dashboard() {