From da94d2f194e027ef2ec1c9de9a146ee0e6ea0269 Mon Sep 17 00:00:00 2001 From: funnywolf Date: Sun, 28 Jun 2026 15:09:10 +0800 Subject: [PATCH] Add custom definitions console Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- asf-doc | 2 +- backend/apps/agentic/runtime/base.py | 1 - backend/apps/agentic/services/custom.py | 181 ++++-- backend/apps/common/redis_stream.py | 10 + backend/apps/settings/custom_views.py | 119 ++++ backend/apps/settings/urls.py | 14 +- backend/apps/settings/views.py | 18 - backend/custom/playbooks/case_summary.py | 1 - backend/integrations/siem/registry.py | 11 +- .../2026-06-28-custom-definitions-design.md | 13 +- frontend/src/App.tsx | 2 + frontend/src/components/MainLayout.tsx | 6 +- frontend/src/pages/CustomDefinitions.tsx | 594 ++++++++++++++++++ frontend/src/pages/RuntimeSettings.tsx | 134 +--- 14 files changed, 876 insertions(+), 230 deletions(-) create mode 100644 backend/apps/settings/custom_views.py create mode 100644 frontend/src/pages/CustomDefinitions.tsx diff --git a/asf-doc b/asf-doc index 174fc37..33d5d56 160000 --- a/asf-doc +++ b/asf-doc @@ -1 +1 @@ -Subproject commit 174fc379498e07dc39c301bc27b1533f9b83681c +Subproject commit 33d5d5623a07e7ef8a6036d0ea0823151b36d071 diff --git a/backend/apps/agentic/runtime/base.py b/backend/apps/agentic/runtime/base.py index 9a6ca2c..b4370e9 100644 --- a/backend/apps/agentic/runtime/base.py +++ b/backend/apps/agentic/runtime/base.py @@ -14,7 +14,6 @@ class BasePlaybook: DESC = "" TAGS = [] PROMPT_SLUG = "" - REQUIRED_PROMPTS = [] SCRIPT_PATH = None def __init__(self, *, playbook_run=None): diff --git a/backend/apps/agentic/services/custom.py b/backend/apps/agentic/services/custom.py index 1d3726b..2e9eec1 100644 --- a/backend/apps/agentic/services/custom.py +++ b/backend/apps/agentic/services/custom.py @@ -1,12 +1,14 @@ from pathlib import Path +import redis from django.conf import settings -from apps.agentic.runtime.module import scan_module_definitions +from apps.agentic.runtime.module import AGENTIC_MODULE_CONSUMER_GROUP, scan_module_definitions from apps.agentic.services.playbooks import scan_playbook_definitions +from apps.common.redis_stream import RedisStreamClient from integrations.siem.registry import reload_registry, scan_registry_configs -PROMPT_LANGUAGES = ("en", "zh") +MAX_STREAM_MESSAGES = 20 def _source_for_path(path): @@ -22,10 +24,10 @@ def _source_for_path(path): def _module_record(definition): return { "name": definition.name, + "description": getattr(definition.module_class, "DESC", ""), "stream_name": definition.stream_name, "thread_num": definition.thread_num, "path": str(definition.path), - "source": _source_for_path(definition.path), } @@ -42,74 +44,131 @@ def _playbook_record(definition): def _siem_record(item): - path = item["path"] + return item + + +def _decode(value): + return value.decode() if isinstance(value, bytes) else value + + +def _info_get(info, key, default=None): + return info.get(key, info.get(key.encode(), default)) + + +def _stream_group_record(group): return { - **item, - "source": _source_for_path(path), + "name": _decode(_info_get(group, "name", "")), + "consumers": _info_get(group, "consumers", 0), + "pending": _info_get(group, "pending", 0), + "last_delivered_id": _decode(_info_get(group, "last-delivered-id", "")), } -def _prompt_record(definition, prompt_name, language, path): +def _entry_id(entry): + if isinstance(entry, (list, tuple)) and entry: + return _decode(entry[0]) + return "" + + +def _stream_health(stream_name, *, redis_client=None): + try: + client = redis_client or RedisStreamClient() + info = client.stream_info(stream_name) + groups = client.stream_groups(stream_name) + except redis.ResponseError as exc: + if "no such key" in str(exc).lower(): + return { + "available": False, + "length": 0, + "first_id": "", + "last_id": "", + "groups": [], + "warning": "Stream does not exist yet.", + } + raise + return { - "playbook": definition.name, - "prompt": prompt_name, - "language": language, - "path": str(path), - "source": "custom", + "available": True, + "length": _info_get(info, "length", 0), + "first_id": _entry_id(_info_get(info, "first-entry")), + "last_id": _entry_id(_info_get(info, "last-entry")), + "groups": [_stream_group_record(group) for group in groups], + "warning": "", } -def _scan_playbook_prompts(playbooks): - items = [] - errors = [] - for definition in playbooks: - if _source_for_path(definition.path) != "custom": - continue - required_prompts = getattr(definition.script_class, "REQUIRED_PROMPTS", []) or [] - for prompt_name in required_prompts: - for language in PROMPT_LANGUAGES: - path = definition.script_class.prompt_path(prompt_name, language=language) - if path.exists(): - items.append(_prompt_record(definition, prompt_name, language, path)) - else: - errors.append({ - "path": str(path), - "error": f"Missing custom playbook prompt: {definition.name} {prompt_name}_{language}", - }) - return items, errors +def _module_record_with_stream_health(definition, *, redis_client=None): + record = _module_record(definition) + try: + record["stream_health"] = _stream_health(definition.stream_name, redis_client=redis_client) + except redis.RedisError as exc: + record["stream_health"] = { + "available": False, + "length": 0, + "first_id": "", + "last_id": "", + "groups": [], + "warning": f"{type(exc).__name__}: {exc}", + } + return record -def refresh_custom_definitions(): - reload_registry() - modules, module_errors = scan_module_definitions() - playbooks, playbook_errors = scan_playbook_definitions() - siem_indices, siem_errors = scan_registry_configs() - prompt_items, prompt_errors = _scan_playbook_prompts(playbooks) - sections = { - "modules": { - "items": [_module_record(item) for item in modules], - "errors": module_errors, - }, - "playbooks": { - "items": [_playbook_record(item) for item in playbooks], - "errors": playbook_errors, - }, - "siem": { - "items": [_siem_record(item) for item in siem_indices], - "errors": siem_errors, - }, - "prompts": { - "items": prompt_items, - "errors": prompt_errors, +def _section_result(section, items, errors): + return { + "section": section, + "success": not errors, + "counts": { + "items": len(items), + "errors": len(errors), }, + "items": items, + "errors": errors, } - result = dict(sections) - result["success"] = not any(section["errors"] for section in sections.values()) - result["counts"] = { - "modules": len(result["modules"]["items"]), - "playbooks": len(result["playbooks"]["items"]), - "siem": len(result["siem"]["items"]), - "prompts": len(result["prompts"]["items"]), - "errors": sum(len(section["errors"]) for section in sections.values()), + + +def list_module_definitions_with_health(): + modules, errors = scan_module_definitions() + items = [ + _module_record_with_stream_health(definition) + for definition in modules + ] + return _section_result("modules", items, errors) + + +def list_playbook_definition_records(): + playbooks, errors = scan_playbook_definitions() + items = [_playbook_record(item) for item in playbooks] + return _section_result("playbooks", items, errors) + + +def list_siem_definition_records(*, reload=False): + if reload: + reload_registry() + siem_indices, errors = scan_registry_configs() + items = [_siem_record(item) for item in siem_indices] + return _section_result("siem", items, errors) + + +def known_module_streams(): + modules, _errors = scan_module_definitions() + return {definition.stream_name for definition in modules} + + +def read_module_stream_recent(stream_name, limit): + limit = max(1, min(int(limit or 5), MAX_STREAM_MESSAGES)) + messages = RedisStreamClient().read_stream_recent(stream_name, limit) + return { + "stream_name": stream_name, + "consumer_group": AGENTIC_MODULE_CONSUMER_GROUP, + "limit": limit, + "messages": messages, + } + + +def read_module_stream_message(stream_name, message_id): + message = RedisStreamClient().read_stream_message_by_id(stream_name, message_id) + return { + "stream_name": stream_name, + "message_id": message_id, + "message": message, } - return result diff --git a/backend/apps/common/redis_stream.py b/backend/apps/common/redis_stream.py index 0bb9c33..ca1b15e 100644 --- a/backend/apps/common/redis_stream.py +++ b/backend/apps/common/redis_stream.py @@ -80,6 +80,10 @@ class RedisStreamClient: messages = self.redis.xrange(stream, min="-", max="+", count=count) return [_decode_stream_message(message_id, fields, stream=stream) for message_id, fields in messages] + def read_stream_recent(self, stream, count): + messages = self.redis.xrevrange(stream, max="+", min="-", count=count) + return [_decode_stream_message(message_id, fields, stream=stream) for message_id, fields in messages] + def read_stream_message_by_id(self, stream, message_id): messages = self.redis.xrange(stream, min=message_id, max=message_id, count=1) if not messages: @@ -87,5 +91,11 @@ class RedisStreamClient: found_id, fields = messages[0] return _decode_stream_message(found_id, fields, stream=stream) + def stream_info(self, stream): + return self.redis.xinfo_stream(stream) + + def stream_groups(self, stream): + return self.redis.xinfo_groups(stream) + def delete_stream(self, stream): return bool(self.redis.delete(stream)) diff --git a/backend/apps/settings/custom_views.py b/backend/apps/settings/custom_views.py new file mode 100644 index 0000000..3f9b4b5 --- /dev/null +++ b/backend/apps/settings/custom_views.py @@ -0,0 +1,119 @@ +import redis +from django.contrib.contenttypes.models import ContentType +from rest_framework import permissions, status, views +from rest_framework.response import Response + +from apps.accounts.permissions import IsAdmin +from apps.agentic.services.custom import ( + MAX_STREAM_MESSAGES, + known_module_streams, + list_module_definitions_with_health, + list_playbook_definition_records, + list_siem_definition_records, + read_module_stream_message, + read_module_stream_recent, +) +from apps.audit.models import AuditLog +from .models import RuntimeConfig + + +def _audit_refresh(section, request, result): + instance = RuntimeConfig.get_current() + AuditLog.objects.create( + content_type=ContentType.objects.get_for_model(RuntimeConfig), + object_id=str(instance.pk), + action="refresh", + actor=request.user if getattr(request.user, "is_authenticated", False) else None, + metadata={ + "section": section, + "success": result["success"], + "counts": result["counts"], + }, + ) + + +class CustomDefinitionsModuleView(views.APIView): + permission_classes = [permissions.IsAuthenticated, IsAdmin] + + def get(self, request): + return Response(list_module_definitions_with_health(), status=status.HTTP_200_OK) + + def post(self, request): + result = list_module_definitions_with_health() + _audit_refresh("modules", request, result) + return Response(result, status=status.HTTP_200_OK) + + +class CustomDefinitionsPlaybookView(views.APIView): + permission_classes = [permissions.IsAuthenticated, IsAdmin] + + def get(self, request): + return Response(list_playbook_definition_records(), status=status.HTTP_200_OK) + + def post(self, request): + result = list_playbook_definition_records() + _audit_refresh("playbooks", request, result) + return Response(result, status=status.HTTP_200_OK) + + +class CustomDefinitionsSiemView(views.APIView): + permission_classes = [permissions.IsAuthenticated, IsAdmin] + + def get(self, request): + return Response(list_siem_definition_records(), status=status.HTTP_200_OK) + + def post(self, request): + result = list_siem_definition_records(reload=True) + _audit_refresh("siem", request, result) + return Response(result, status=status.HTTP_200_OK) + + +def _stream_name(request): + stream_name = str(request.query_params.get("stream_name") or "").strip() + if not stream_name: + return None, Response({"stream_name": ["This query parameter is required."]}, status=status.HTTP_400_BAD_REQUEST) + if stream_name not in known_module_streams(): + return None, Response({"detail": "Unknown module stream."}, status=status.HTTP_400_BAD_REQUEST) + return stream_name, None + + +def _stream_limit(request): + raw_limit = request.query_params.get("limit", 5) + try: + return max(1, min(int(raw_limit), MAX_STREAM_MESSAGES)), None + except (TypeError, ValueError): + return None, Response({"limit": ["Limit must be an integer."]}, status=status.HTTP_400_BAD_REQUEST) + + +class CustomModuleStreamMessagesView(views.APIView): + permission_classes = [permissions.IsAuthenticated, IsAdmin] + + def get(self, request): + stream_name, error = _stream_name(request) + if error is not None: + return error + limit, error = _stream_limit(request) + if error is not None: + return error + try: + result = read_module_stream_recent(stream_name, limit) + except redis.RedisError as exc: + return Response({"detail": f"{type(exc).__name__}: {exc}"}, status=status.HTTP_503_SERVICE_UNAVAILABLE) + return Response(result, status=status.HTTP_200_OK) + + +class CustomModuleStreamMessageView(views.APIView): + permission_classes = [permissions.IsAuthenticated, IsAdmin] + + def get(self, request): + stream_name, error = _stream_name(request) + if error is not None: + return error + message_id = str(request.query_params.get("message_id") or "").strip() + if not message_id: + return Response({"message_id": ["This query parameter is required."]}, status=status.HTTP_400_BAD_REQUEST) + try: + result = read_module_stream_message(stream_name, message_id) + except redis.RedisError as exc: + return Response({"detail": f"{type(exc).__name__}: {exc}"}, status=status.HTTP_503_SERVICE_UNAVAILABLE) + return Response(result, status=status.HTTP_200_OK) diff --git a/backend/apps/settings/urls.py b/backend/apps/settings/urls.py index 25e3a91..487043d 100644 --- a/backend/apps/settings/urls.py +++ b/backend/apps/settings/urls.py @@ -1,12 +1,18 @@ from django.urls import include, path from rest_framework.routers import DefaultRouter +from .custom_views import ( + CustomDefinitionsModuleView, + CustomDefinitionsPlaybookView, + CustomDefinitionsSiemView, + CustomModuleStreamMessageView, + CustomModuleStreamMessagesView, +) from .views import ( LLMProviderConfigViewSet, LdapConfigView, LdapTestView, RuntimeConfigView, - RuntimeCustomDefinitionsRefreshView, SiemElkConfigView, SiemElkTestView, SiemSplunkConfigView, @@ -29,6 +35,10 @@ urlpatterns = [ path("settings/ldap/", LdapConfigView.as_view(), name="ldap-config"), path("settings/ldap/test/", LdapTestView.as_view(), name="ldap-test"), path("settings/runtime/", RuntimeConfigView.as_view(), name="runtime-config"), - path("settings/runtime/custom-definitions/refresh/", RuntimeCustomDefinitionsRefreshView.as_view(), name="runtime-custom-definitions-refresh"), + path("custom/modules/", CustomDefinitionsModuleView.as_view(), name="custom-definitions-modules"), + path("custom/modules/stream/messages/", CustomModuleStreamMessagesView.as_view(), name="custom-module-stream-messages"), + path("custom/modules/stream/message/", CustomModuleStreamMessageView.as_view(), name="custom-module-stream-message"), + path("custom/playbooks/", CustomDefinitionsPlaybookView.as_view(), name="custom-definitions-playbooks"), + path("custom/siem/", CustomDefinitionsSiemView.as_view(), name="custom-definitions-siem"), path("settings/", include(router.urls)), ] diff --git a/backend/apps/settings/views.py b/backend/apps/settings/views.py index f589cd2..c7a1774 100644 --- a/backend/apps/settings/views.py +++ b/backend/apps/settings/views.py @@ -8,7 +8,6 @@ from rest_framework.filters import OrderingFilter, SearchFilter from rest_framework.response import Response from apps.accounts.permissions import IsAdmin -from apps.agentic.services.custom import refresh_custom_definitions from apps.audit.models import AuditLog from apps.common.advanced_filters import AdvancedFilterBackend from .models import ( @@ -389,20 +388,3 @@ class RuntimeConfigView(views.APIView): transaction.on_commit(lambda: invalidate("runtime")) return Response(RuntimeConfigSerializer(instance).data) - -class RuntimeCustomDefinitionsRefreshView(views.APIView): - permission_classes = [permissions.IsAuthenticated, IsAdmin] - - def post(self, request): - result = refresh_custom_definitions() - instance = RuntimeConfig.get_current() - _write_audit( - instance, - "refresh", - request.user, - metadata={ - "success": result["success"], - "counts": result["counts"], - }, - ) - return Response(result, status=status.HTTP_200_OK) diff --git a/backend/custom/playbooks/case_summary.py b/backend/custom/playbooks/case_summary.py index 16ada43..bd8e55d 100644 --- a/backend/custom/playbooks/case_summary.py +++ b/backend/custom/playbooks/case_summary.py @@ -13,7 +13,6 @@ class Playbook(BasePlaybook): DESC = "Generate a concise analyst-facing summary for the linked Case." TAGS = ["Custom", "LLM", "Case"] PROMPT_SLUG = "case_summary" - REQUIRED_PROMPTS = ["System"] def run(self): if self.case is None: diff --git a/backend/integrations/siem/registry.py b/backend/integrations/siem/registry.py index 8582f9d..c928dea 100644 --- a/backend/integrations/siem/registry.py +++ b/backend/integrations/siem/registry.py @@ -57,7 +57,16 @@ def scan_registry_configs(): except Exception as exc: errors.append({"path": str(yaml_file), "error": f"{type(exc).__name__}: {exc}"}) continue - indices.append({"name": index_info.name, "backend": index_info.backend, "path": str(yaml_file)}) + fields = [field.model_dump() for field in index_info.fields] + indices.append({ + "name": index_info.name, + "backend": index_info.backend, + "description": index_info.description, + "path": str(yaml_file), + "field_count": len(fields), + "key_field_count": sum(1 for field in index_info.fields if field.is_key_field), + "fields": fields, + }) return indices, errors diff --git a/docs/superpowers/specs/2026-06-28-custom-definitions-design.md b/docs/superpowers/specs/2026-06-28-custom-definitions-design.md index f8da612..99a6c44 100644 --- a/docs/superpowers/specs/2026-06-28-custom-definitions-design.md +++ b/docs/superpowers/specs/2026-06-28-custom-definitions-design.md @@ -25,7 +25,7 @@ Prompt 文件不再作为 Custom Definitions 的管理对象。Playbook 可以 ## 目标 -- 让管理员可以从独立入口查看所有已加载 definition，并区分 `official` / `custom` 来源。 +- 让管理员可以从独立入口查看所有已加载 definition。Playbook 需要区分 `official` / `custom` 来源；Modules 和 SIEM YAML 没有 official 来源概念，不展示 source。 - 将验证功能按 tab 拆分，避免一个聚合结果难以定位问题。 - 让 Module 页面展示对应 Redis Stream 的基础运行信息，并支持只读查看最近消息或指定消息。 - 让 Playbook 页面展示可运行定义，但不在 Custom 页面直接执行 playbook。 @@ -89,7 +89,6 @@ Module section 返回： - `name` - `description` -- `source` - `path` - `stream_name` - `thread_num` @@ -139,7 +138,6 @@ SIEM section 返回： - `name` - `backend` - `description` -- `source` - `path` - `field_count` - `key_field_count` @@ -163,13 +161,11 @@ Toolbar： - `Refresh / Validate` - `Reload` -- source filter - search 主表字段： - Module name -- source - description - stream_name - thread_num @@ -208,7 +204,6 @@ Toolbar： 动作： -- `Copy name` - Playbook 运行记录跳转留到后续单独设计，本次不实现 URL 过滤或 deep link。 ### SIEM YAML tab @@ -218,7 +213,6 @@ Toolbar： - `Refresh / Validate` - `Reload` - backend filter -- source filter - search 主表字段： @@ -226,7 +220,6 @@ Toolbar： - index name - backend - description -- source - field count - key field count - path @@ -235,7 +228,7 @@ Toolbar： - name - type -- key +- key field - description - sample values @@ -292,7 +285,7 @@ Toolbar： - Runtime 页不再显示 Custom Definitions。 - 每个 tab 可单独 Refresh / Validate 并展示本 section errors。 - Modules 详情抽屉可读取最近 stream 消息和指定 message id。 -- Playbooks tab 只展示和复制，不直接执行 playbook。 +- Playbooks tab 只展示，不直接执行 playbook。 - SIEM YAML 详情抽屉展示 fields 表。 文档： diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 5f8d654..24bfc67 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -11,6 +11,7 @@ import PlaybookList from './pages/PlaybookList' import KnowledgeList from './pages/KnowledgeList' import Dashboard from './pages/Dashboard' import SystemSettings from './pages/SystemSettings' +import CustomDefinitions from './pages/CustomDefinitions' import {useAuthStore} from './stores/auth' import {hasPermission, type PermissionKey} from './utils/permissions' import {getMe} from './api/auth' @@ -56,6 +57,7 @@ export default function App() { } /> } /> } /> + } /> } /> } /> diff --git a/frontend/src/components/MainLayout.tsx b/frontend/src/components/MainLayout.tsx index 257e324..f0941b7 100644 --- a/frontend/src/components/MainLayout.tsx +++ b/frontend/src/components/MainLayout.tsx @@ -7,7 +7,7 @@ import { MenuUnfoldOutlined, UserOutlined, } from '@ant-design/icons' -import {BookOpenText, BrainCircuit, BriefcaseBusiness, Fingerprint, LayoutDashboard, Settings, Siren, WandSparkles} from 'lucide-react' +import {BookOpenText, BrainCircuit, BriefcaseBusiness, Fingerprint, LayoutDashboard, Puzzle, Settings, Siren, WandSparkles} from 'lucide-react' import {useAuthStore} from '../stores/auth' import {getResourceConfig} from '../config/resources' import type {ResourceConfig} from '../types/records' @@ -28,6 +28,7 @@ const breadcrumbMap: Record = { enrichments: 'Enrichments', playbooks: 'Playbooks', knowledge: 'Knowledge', + custom: 'Custom', dashboard: 'Dashboard', system: 'Setting', } @@ -63,7 +64,7 @@ export default function MainLayout() { }, [location.pathname]) const selectedKey = (() => { - const allKeys = ['/dashboard', '/cases', '/alerts', '/artifacts', '/enrichments', '/playbooks', '/knowledge', '/system'] + const allKeys = ['/dashboard', '/cases', '/alerts', '/artifacts', '/enrichments', '/playbooks', '/knowledge', '/custom', '/system'] if (allKeys.includes(location.pathname)) return location.pathname return '/' + (location.pathname.split('/').filter(Boolean)[0] || 'cases') })() @@ -77,6 +78,7 @@ export default function MainLayout() { { key: '/enrichments', icon: , label: 'Enrichments' }, { key: '/playbooks', icon: , label: 'Playbooks' }, { key: '/knowledge', icon: , label: 'Knowledge' }, + hasPermission(user, 'admin') ? { key: '/custom', icon: , label: 'Custom' } : null, hasPermission(user, 'admin') ? { key: '/system', icon: , label: 'Setting' } : null, ].filter((item): item is NonNullable => Boolean(item)) diff --git a/frontend/src/pages/CustomDefinitions.tsx b/frontend/src/pages/CustomDefinitions.tsx new file mode 100644 index 0000000..9482608 --- /dev/null +++ b/frontend/src/pages/CustomDefinitions.tsx @@ -0,0 +1,594 @@ +import {useCallback, useEffect, useMemo, useState} from 'react' +import {Alert, Button, Descriptions, Drawer, Empty, Flex, Input, InputNumber, List, message, Select, Space, Table, Tabs, Tag, Typography} from 'antd' +import {ReloadOutlined} from '@ant-design/icons' +import type {ColumnsType} from 'antd/es/table' +import {Boxes, BrainCircuit, DatabaseZap} from 'lucide-react' +import client from '../api/client' +import JsonViewer from '../components/JsonViewer' +import IconTabLabel from '../components/IconTabLabel' +import {comfortableTagProps} from '../utils/tagStyles' + +type SourceType = 'official' | 'custom' +type SourceFilter = 'all' | SourceType + +interface DefinitionError { + path: string + error: string +} + +interface SectionResult { + section: string + success: boolean + counts: { + items: number + errors: number + } + items: T[] + errors: DefinitionError[] +} + +interface StreamGroup { + name: string + consumers: number + pending: number + last_delivered_id: string +} + +interface StreamHealth { + available: boolean + length: number + first_id: string + last_id: string + groups: StreamGroup[] + warning: string +} + +interface ModuleDefinition { + name: string + description: string + path: string + stream_name: string + thread_num: number + stream_health: StreamHealth +} + +interface PlaybookDefinition { + name: string + description: string + tags: string[] + source: SourceType + path: string +} + +interface SiemField { + name: string + type: string + description: string + is_key_field: boolean + sample_values: unknown[] +} + +interface SiemDefinition { + name: string + backend: 'ELK' | 'Splunk' + description: string + path: string + field_count: number + key_field_count: number + fields: SiemField[] +} + +interface StreamMessage { + message_id: string + data: unknown +} + +interface StreamMessagesResponse { + stream_name: string + consumer_group: string + limit: number + messages: StreamMessage[] +} + +interface StreamMessageResponse { + stream_name: string + message_id: string + message: StreamMessage | Record +} + +function apiErrorMessage(error: unknown, fallback: string) { + const data = (error as { response?: { data?: unknown } }).response?.data + if (typeof data === 'string') return data + if (data && typeof data === 'object') { + const detail = (data as { detail?: unknown }).detail + if (typeof detail === 'string') return detail + return JSON.stringify(data) + } + return fallback +} + +function sourceOptions(): Array<{ label: string; value: SourceFilter }> { + return [ + { label: 'All sources', value: 'all' }, + { label: 'Official', value: 'official' }, + { label: 'Custom', value: 'custom' }, + ] +} + +function SourceTag({ source }: { source: SourceType }) { + return {source} +} + +function PathText({ path }: { path: string }) { + return ( + + {path} + + ) +} + +function filterBySource(items: T[], source: SourceFilter) { + return source === 'all' ? items : items.filter((item) => item.source === source) +} + +const PLAYBOOK_TAG_COLORS: Record = { + System: 'gold', + LLM: 'purple', + Case: 'green', + Knowledge: 'magenta', + CMDB: 'geekblue', + 'Threat Intel': 'volcano', + Enrichment: 'cyan', + Custom: 'blue', +} + +const SIEM_BACKEND_COLORS: Record = { + ELK: 'orange', + Splunk: 'green', +} + +function includesSearch(values: unknown[], search: string) { + const keyword = search.trim().toLowerCase() + if (!keyword) return true + return values.some((value) => String(value || '').toLowerCase().includes(keyword)) +} + +function DefinitionErrors({ errors }: { errors: DefinitionError[] }) { + if (!errors.length) return null + return ( + + {errors.map((error) => ( + + ))} + + ) +} + +function useDefinitionSection(endpoint: string, label: string) { + const [result, setResult] = useState | null>(null) + const [loading, setLoading] = useState(false) + const [refreshing, setRefreshing] = useState(false) + + const load = useCallback(async () => { + setLoading(true) + try { + const { data } = await client.get>(endpoint) + setResult(data) + } catch (error) { + message.error(apiErrorMessage(error, `Failed to load ${label}`)) + setResult(null) + } finally { + setLoading(false) + } + }, [endpoint, label]) + + const refresh = useCallback(async () => { + setRefreshing(true) + try { + const { data } = await client.post>(endpoint) + setResult(data) + if (data.success) { + message.success(`${label} refreshed`) + } else { + message.warning(`${label} refreshed with ${data.counts.errors} error(s)`) + } + } catch (error) { + message.error(apiErrorMessage(error, `Failed to refresh ${label}`)) + } finally { + setRefreshing(false) + } + }, [endpoint, label]) + + useEffect(() => { + // eslint-disable-next-line react-hooks/set-state-in-effect + load() + }, [load]) + + return { result, loading, refreshing, load, refresh } +} + +function SectionToolbar({ + search, + onSearchChange, + source, + onSourceChange, + onReload, + onRefresh, + loading, + refreshing, + extra, +}: { + search: string + onSearchChange: (value: string) => void + source?: SourceFilter + onSourceChange?: (value: SourceFilter) => void + onReload: () => void + onRefresh: () => void + loading: boolean + refreshing: boolean + extra?: React.ReactNode +}) { + return ( + + + + {source && onSourceChange ? value={source} options={sourceOptions()} onChange={onSourceChange} style={{ width: 140 }} /> : null} + {extra} + onSearchChange(event.target.value)} style={{ width: 280 }} /> + + ) +} + +function ModuleDrawer({ module, open, onClose }: { module: ModuleDefinition | null; open: boolean; onClose: () => void }) { + const [messages, setMessages] = useState([]) + const [messageLimit, setMessageLimit] = useState(5) + const [messageId, setMessageId] = useState('') + const [selectedMessage, setSelectedMessage] = useState | null>(null) + const [loadingMessages, setLoadingMessages] = useState(false) + const [messageError, setMessageError] = useState('') + const streamName = module?.stream_name || '' + + const loadRecent = useCallback(async () => { + if (!streamName) return + setLoadingMessages(true) + setMessageError('') + try { + const { data } = await client.get('/custom/modules/stream/messages/', { + params: { stream_name: streamName, limit: messageLimit }, + }) + setMessages(data.messages) + } catch (error) { + setMessages([]) + setMessageError(apiErrorMessage(error, 'Failed to load stream messages')) + } finally { + setLoadingMessages(false) + } + }, [messageLimit, streamName]) + + useEffect(() => { + if (!open || !streamName) return + // eslint-disable-next-line react-hooks/set-state-in-effect + setMessages([]) + setSelectedMessage(null) + setMessageId('') + loadRecent() + }, [loadRecent, open, streamName]) + + const loadById = async () => { + if (!module?.stream_name || !messageId.trim()) return + setLoadingMessages(true) + setMessageError('') + try { + const { data } = await client.get('/custom/modules/stream/message/', { + params: { stream_name: module.stream_name, message_id: messageId.trim() }, + }) + setSelectedMessage(data.message) + } catch (error) { + setSelectedMessage(null) + setMessageError(apiErrorMessage(error, 'Failed to load stream message')) + } finally { + setLoadingMessages(false) + } + } + + return ( + + {module ? ( + + {module.stream_name} }, + { key: 'threads', label: 'Threads', children: module.thread_num }, + { key: 'path', label: 'Path', children: }, + ]} /> + + + {module.stream_health.warning ? : null} + {module.stream_health.groups.length ? ( + + size="small" + rowKey="name" + pagination={false} + dataSource={module.stream_health.groups} + columns={[ + { title: 'Group', dataIndex: 'name' }, + { title: 'Consumers', dataIndex: 'consumers', width: 120 }, + { title: 'Pending', dataIndex: 'pending', width: 120 }, + { title: 'Last delivered', dataIndex: 'last_delivered_id', width: 220 }, + ]} + /> + ) : null} + +
+ Recent messages + + setMessageLimit(Number(value || 5))} /> + + +
+ {messageError ? : null} + }} + renderItem={(item) => ( + + + {item.message_id} + + + + )} + /> + +
+ Read by message ID + setMessageId(event.target.value)} + onSearch={loadById} + style={{ marginTop: 8 }} + placeholder="1700000000000-0" + /> +
+ {selectedMessage ? : null} + + ) : null} + + ) +} + +function ModulesTab() { + const { result, loading, refreshing, load, refresh } = useDefinitionSection('/custom/modules/', 'Modules') + const [search, setSearch] = useState('') + const [selected, setSelected] = useState(null) + const rows = useMemo(() => ( + (result?.items || []).filter((item) => includesSearch([ + item.name, + item.description, + item.stream_name, + item.path, + ], search)) + ), [result?.items, search]) + + const columns = useMemo>(() => [ + { title: 'Module', dataIndex: 'name', width: 340, render: (name: string) => {name} }, + { title: 'Description', dataIndex: 'description', ellipsis: true }, + { title: 'Stream', dataIndex: 'stream_name', width: 380, ellipsis: true }, + { title: 'Threads', dataIndex: 'thread_num', width: 100 }, + { title: 'Length', width: 100, render: (_, record) => record.stream_health.length }, + { title: 'Last ID', width: 180, render: (_, record) => record.stream_health.last_id || '—' }, + { title: 'Path', dataIndex: 'path', width: 280, render: (path: string) => }, + ], []) + + return ( + <> + + + + size="small" + rowKey="path" + loading={loading} + columns={columns} + dataSource={rows} + onRow={(record) => ({ onClick: () => setSelected(record), style: { cursor: 'pointer' } })} + locale={{ emptyText: loading ? : }} + scroll={{ x: 1600 }} + /> + setSelected(null)} /> + + ) +} + +function PlaybooksTab() { + const { result, loading, refreshing, load, refresh } = useDefinitionSection('/custom/playbooks/', 'Playbooks') + const [source, setSource] = useState('all') + const [search, setSearch] = useState('') + const [selected, setSelected] = useState(null) + const rows = useMemo(() => ( + filterBySource(result?.items || [], source).filter((item) => includesSearch([ + item.name, + item.description, + item.path, + ...item.tags, + ], search)) + ), [result?.items, search, source]) + + const columns = useMemo>(() => [ + { title: 'Playbook', dataIndex: 'name', width: 260, render: (name: string) => {name} }, + { title: 'Source', dataIndex: 'source', width: 110, render: (value: SourceType) => }, + { + title: 'Tags', + dataIndex: 'tags', + width: 260, + render: (tags: string[]) => ( + + {(tags || []).map((tag) => {tag})} + + ), + }, + { title: 'Description', dataIndex: 'description', ellipsis: true }, + { title: 'Path', dataIndex: 'path', width: 300, render: (path: string) => }, + ], []) + + return ( + <> + + + + size="small" + rowKey="path" + loading={loading} + columns={columns} + dataSource={rows} + onRow={(record) => ({ onClick: () => setSelected(record), style: { cursor: 'pointer' } })} + locale={{ emptyText: loading ? : }} + scroll={{ x: 1300 }} + /> + setSelected(null)} size="min(760px, calc(100vw - 48px))"> + {selected ? ( + + + + {(selected.tags || []).map((tag) => {tag})} + + {selected.description || 'No description.'} + }, + ]} /> + + ) : null} + + + ) +} + +function SiemTab() { + const { result, loading, refreshing, load, refresh } = useDefinitionSection('/custom/siem/', 'SIEM YAML') + const [backend, setBackend] = useState<'all' | 'ELK' | 'Splunk'>('all') + const [search, setSearch] = useState('') + const [selected, setSelected] = useState(null) + const rows = useMemo(() => ( + (result?.items || []) + .filter((item) => backend === 'all' || item.backend === backend) + .filter((item) => includesSearch([item.name, item.backend, item.description, item.path], search)) + ), [backend, result?.items, search]) + + const columns = useMemo>(() => [ + { title: 'Index', dataIndex: 'name', width: 220, render: (name: string) => {name} }, + { title: 'Backend', dataIndex: 'backend', width: 120, render: (value: string) => {value} }, + { title: 'Description', dataIndex: 'description', ellipsis: true }, + { title: 'Fields', dataIndex: 'field_count', width: 100 }, + { title: 'Key fields', dataIndex: 'key_field_count', width: 120 }, + { title: 'Path', dataIndex: 'path', width: 300, render: (path: string) => }, + ], []) + + const fieldColumns = useMemo>(() => [ + { title: 'Name', dataIndex: 'name', width: 220 }, + { title: 'Type', dataIndex: 'type', width: 120 }, + { title: 'Key field', dataIndex: 'is_key_field', width: 110, render: (value: boolean) => value ? Key field : '—' }, + { title: 'Description', dataIndex: 'description' }, + { + title: 'Sample values', + dataIndex: 'sample_values', + width: 260, + render: (values: unknown[]) => ( + + {JSON.stringify(values || [])} + + ), + }, + ], []) + + return ( + <> + + value={backend} + onChange={(value) => setBackend(value)} + style={{ width: 140 }} + options={[ + { label: 'All backends', value: 'all' }, + { label: 'ELK', value: 'ELK' }, + { label: 'Splunk', value: 'Splunk' }, + ]} + /> + )} + /> + + + size="small" + rowKey="path" + loading={loading} + columns={columns} + dataSource={rows} + onRow={(record) => ({ onClick: () => setSelected(record), style: { cursor: 'pointer' } })} + locale={{ emptyText: loading ? : }} + scroll={{ x: 1400 }} + /> + setSelected(null)} size="min(1080px, calc(100vw - 48px))"> + {selected ? ( + + {selected.backend} }, + { key: 'description', label: 'Description', children: selected.description || '—' }, + { key: 'path', label: 'Path', children: }, + ]} /> + + size="small" + rowKey="name" + columns={fieldColumns} + dataSource={selected.fields || []} + pagination={false} + scroll={{ x: 900, y: 520 }} + /> + + ) : null} + + + ) +} + +export default function CustomDefinitions() { + return ( +
+ Modules, children: }, + { key: 'playbooks', label: Playbooks, children: }, + { key: 'siem', label: SIEM YAML, children: }, + ]} + /> +
+ ) +} diff --git a/frontend/src/pages/RuntimeSettings.tsx b/frontend/src/pages/RuntimeSettings.tsx index 40896b9..05b7d9b 100644 --- a/frontend/src/pages/RuntimeSettings.tsx +++ b/frontend/src/pages/RuntimeSettings.tsx @@ -1,5 +1,5 @@ import {useCallback, useEffect, useState} from 'react' -import {Alert, Button, Card, Col, Divider, Form, InputNumber, List, message, Row, Select, Space, Tag, Tooltip, Typography} from 'antd' +import {Button, Card, Col, Divider, Form, InputNumber, message, Row, Select, Space, Tooltip, Typography} from 'antd' import {QuestionCircleOutlined} from '@ant-design/icons' import client from '../api/client' @@ -9,43 +9,6 @@ interface RuntimeConfig { updated_at?: string } -interface CustomDefinitionItem { - name?: string - source: 'official' | 'custom' - path: string - stream_name?: string - backend?: string - description?: string - playbook?: string - prompt?: string - language?: string -} - -interface CustomDefinitionError { - path: string - error: string -} - -interface CustomDefinitionSection { - items: CustomDefinitionItem[] - errors: CustomDefinitionError[] -} - -interface CustomDefinitionRefreshResult { - success: boolean - counts: { - modules: number - playbooks: number - siem: number - prompts: number - errors: number - } - modules: CustomDefinitionSection - playbooks: CustomDefinitionSection - siem: CustomDefinitionSection - prompts: CustomDefinitionSection -} - function initialValues(): RuntimeConfig { return { prompt_language: 'en', @@ -77,50 +40,10 @@ function helpLabel(label: string, help: string) { ) } -function CustomDefinitionSectionView({ title, section }: { title: string; section: CustomDefinitionSection }) { - return ( -
- {title} - ( - - - - {item.name || item.playbook || item.prompt} - {item.source} - {item.stream_name ? {item.stream_name} : null} - {item.backend ? {item.backend} : null} - {item.prompt ? {item.prompt} : null} - {item.language ? {item.language} : null} - - {item.path} - - - )} - /> - {section.errors.map((error) => ( - - ))} -
- ) -} - export default function RuntimeSettings() { const [form] = Form.useForm() const [loading, setLoading] = useState(false) const [saving, setSaving] = useState(false) - const [refreshingDefinitions, setRefreshingDefinitions] = useState(false) - const [definitionResult, setDefinitionResult] = useState(null) const loadConfig = useCallback(async () => { setLoading(true) @@ -153,23 +76,6 @@ export default function RuntimeSettings() { } } - const refreshDefinitions = async () => { - setRefreshingDefinitions(true) - try { - const { data } = await client.post('/settings/runtime/custom-definitions/refresh/') - setDefinitionResult(data) - if (data.success) { - message.success('Custom definitions refreshed') - } else { - message.warning(`Custom definitions refreshed with ${data.counts.errors} error(s)`) - } - } catch (error: unknown) { - message.error(apiErrorMessage(error, 'Failed to refresh custom definitions')) - } finally { - setRefreshingDefinitions(false) - } - } - return (
@@ -207,44 +113,6 @@ export default function RuntimeSettings() { - - Custom Definitions - - - Refresh and validate Module, Playbook, and SIEM YAML definitions after updating files in the custom directory. - Dependency or helper module changes still require reinstalling custom packages and restarting related containers. - - - - - {definitionResult ? ( -
- - - - - - - - - - - - - - - -
- ) : null}