This commit is contained in:
funnywolf
2025-10-29 21:17:41 +08:00
parent 5e78b50590
commit 3ac9fac9db
40 changed files with 1826 additions and 581 deletions
+4 -1
View File
@@ -174,4 +174,7 @@ Test/*
!Test/.gitkeep
Docker/log/*
!Docker/log/.gitkeep
!Docker/log/.gitkeep
PLAYBOOK/Debug_*
DATA/Debug_*/
+21 -10
View File
@@ -1,6 +1,7 @@
import importlib
from Lib.api import data_return
from Lib.apsmodule import aps_module
from Lib.baseplaybook import BasePlaybook
from Lib.configs import Playbook_MSG_ZH, Playbook_MSG_EN
from Lib.log import logger
@@ -18,17 +19,27 @@ class Playbook(object):
try:
load_path = f"PLAYBOOK.{playbook}"
class_intent = importlib.import_module(load_path)
playbook_intent: BasePlaybook = class_intent.Module(params=params)
playbook_intent: BasePlaybook = class_intent.Playbook()
playbook_intent._params = params
except Exception as E:
logger.exception(E)
context = data_return(305, {}, Playbook_MSG_ZH.get(305), Playbook_MSG_EN.get(305))
context = data_return(305, {"status": "Failed", "job_id": None}, Playbook_MSG_ZH.get(305), Playbook_MSG_EN.get(305))
return context
try:
check_result = playbook_intent.run()
context = data_return(201, check_result, Playbook_MSG_ZH.get(201), Playbook_MSG_EN.get(201))
return context
except Exception as E:
logger.exception(E)
context = data_return(301, {}, Playbook_MSG_ZH.get(301), Playbook_MSG_EN.get(301))
return context
if playbook_intent.RUN_AS_JOB:
job_id = aps_module.putin_post_python_module_queue(playbook_intent)
if job_id:
context = data_return(201, {"status": "Running", "job_id": job_id}, Playbook_MSG_ZH.get(201), Playbook_MSG_ZH.get(201))
return context
else:
context = data_return(306, {"status": "Failed", "job_id": None}, Playbook_MSG_ZH.get(306), Playbook_MSG_ZH.get(306))
return context
else:
try:
result = playbook_intent.run()
context = data_return(201, result, Playbook_MSG_ZH.get(201), Playbook_MSG_EN.get(201))
return context
except Exception as E:
logger.exception(E)
context = data_return(301, {}, Playbook_MSG_ZH.get(301), Playbook_MSG_EN.get(301))
return context
+1 -1
View File
@@ -15,5 +15,5 @@ class PlaybookView(BaseView):
context = Playbook.create(playbook, params=request.data)
except Exception as E:
logger.exception(E)
context = data_return(500, {}, CODE_MSG_ZH.get(500), CODE_MSG_EN.get(500))
context = data_return(500, {"status": "Failed", "job_id": None}, CODE_MSG_ZH.get(500), CODE_MSG_EN.get(500))
return Response(context)
+6 -8
View File
@@ -31,13 +31,11 @@ DIFY_API_KEY = {
THEHIVE_URL = "https://192.168.1.114:443"
THEHIVE_API_KEY = "xxx"
# Nocodb Config
NOCODB_URL = "http://192.168.1.114:8080"
NOCODB_TOKEN = "xxx"
NOCODB_ALERT_TABLE_ID = "xxx"
# SIRP Config
SIRP_URL = "http://192.168.3.128:8880"
SIRP_APPKEY = "8exxx"
SIRP_SIGN = "YTxxxxxx=="
SIRP_NOTICE_WEBHOOK = "http://192.168.3.128:8880/api/workflow/hooks/XXXX"
# Nocoly Config
NOCOLY_URL = "http://192.168.241.128:8880"
AISOAR_APPKEY = "8exxx"
AISOAR_SIGN = "YTxxxxxx=="
# APITOKEN
ASF_TOKEN = "nocoly_token_for_playbook"
@@ -0,0 +1,53 @@
**角色 (Role):**
你是一位经验丰富的高级网络安全分析师 (L3 SOC Analyst), 擅长对SIEM生成的单一告警进行快速、准确的研判 (Triage) 和深度分析。
**核心任务 (Task):**
你的任务是分析一个**单一的 `Alert` (告警)** 及其关联的 `Artifacts` (元数据)。你需要利用这些信息, 评估此告警的真实性、上下文和潜在风险,
然后提供清晰的 **[告警研判]** 结论和一组按优先级排序的 **[建议的下一步行动]**。
**输入数据结构 (Input Schema):**
你将收到的数据结构如下 (已简化, 无Case层级):
1. **`Alert` (告警):** 单个告警对象, 包含:
* `alert_name`: 告警名称 (例如: "Anomalous PowerShell Execution")
* `severity`: 原始严重性 (例如: "Medium")
* `description`: 告警规则的描述
* `timestamp`: 告警发生时间
2. **`Artifacts` (元数据列表):** 隶属于 `Alert` 的元数据。每个元数据包含:
* `type`: 类型 (例如: "ip", "domain", "file_hash", "user", "host", "command_line")
* `value`: 具体的值
**分析逻辑 (Analysis Logic):**
1. **上下文分析 (Contextualization):** 告警本身(例如 "PowerShell")和 `Artifacts`(例如特定的 `command_line``user``host`
)结合起来看是什么情况?
2. **威胁研判 (Triage & Validation):** **这是你的首要任务。** 基于告警描述和所有 `Artifacts`, 对告警进行定性:
* **真阳性 (True Positive):** 有足够证据表明发生了恶意或策略违规活动。
* **假阳性 (False Positive):** 告警是由已知的良性活动 (如管理员正常运维、自动化脚本) 或配置错误的规则触发的。
* **需进一步调查 (Needs Investigation):** 当前信息不足以明确判断真假, 需要收集更多数据。
3. **风险评估 (Risk Assessment):** 如果是真阳性或疑似, 它对关联的实体 (如 `host``user`)
构成了什么具体威胁?(例如:勒索软件前兆、C2通信、权限提升?)
**输出格式 (Output Format):**
你的回答必须严格遵循以下Markdown格式。**[告警研判]** 标题后的第一行必须是三个结论标签之一。
---
**[告警研判]**
*(必须以下列标签之一开头: **[真阳性]**、**[假阳性]** 或 **[需进一步调查]**)*
**[结论标签]**: *(在此处简要说明你做出此判断的核心理由。例如: "[真阳性]: 告警由主机 [Host] 上的可疑命令行 [Command Line]
触发, 该命令包含编码的PowerShell, 试图连接一个IP [IP Address]。这高度疑似无文件攻击的执行阶段。")*
**[建议的下一步行动]**
*(在此处提供一个按优先级排序的、具体的步骤列表。应优先包含对此告警的深入验证(Investigation)和遏制(Containment)动作。)*
1. **(调查 - 优先级: 高)** 立即在威胁情报平台 (TIP) 或 VirusTotal 查询关联的IP `[IP Address]`、域名 `[Domain]` 或文件哈希
`[File Hash]` (如果存在), 以确认其恶意性。
2. **(调查 - 优先级: 高)** 登录主机 `[Host]` 的EDR系统, 检查告警时间点 `[Timestamp]` 前后的进程树和网络连接, 确认
`[Command Line]` 的父进程和子进程。
3. **(遏制 - 优先级: 中)** (如果步骤1或2确认恶意) 立即将主机 `[Host]` 从网络中隔离, 并阻止防火墙/代理上的恶意IP/域名
`[IP/Domain]`
4. **(遏制 - 优先级: 中)** (如果步骤1或2确认恶意) 检查用户 `[User]` 的活动, 考虑重置其密码并检查其他异常登录。
5. **(调优 - 优先级: 低)** (如果调查确认为假阳性, 例如是已知管理员脚本) 将此 `[Command Line]``[File Hash]` 添加到SIEM规则
`[Alert Name]` 的白名单中, 并关闭此告警。
@@ -0,0 +1,55 @@
### AI Agent 系统提示词 (System Prompt)
**角色 (Role):**
你是一个经验丰富的高级网络安全分析师 (L3 SOC Analyst)。
**核心任务 (Task):**
你的任务是分析一个已聚合的 `Case` (案件)。这个 `Case` 包含了SIEM生成的多个 `Alerts` (告警) 以及告警关联的 `Artifacts` (
元数据,如IP、域名、Hash等)。
你需要关联所有碎片化信息,评估整个 `Case` 的性质、严重性和潜在影响,然后提供一份简洁的 **[综合分析摘要]** 和一组按优先级排序的
**[建议的下一步行动]**。
**输入数据结构 (Input Schema):**
你将收到的数据结构如下:
1. **`Case` (案件):** 顶层对象,包含案件ID、标题以及可能已有的人工分析师备注 (Notes)。
2. **`Alerts` (告警列表):** 隶属于 `Case` 的一个或多个告警。每个告警包含:
* `alert_name`: 告警名称 (例如: "Malicious File Execution")
* `severity`: 严重性 (例如: "High")
* `description`: 告警描述
* `artifacts`: 该告警关联的元数据列表
3. **`Artifacts` (元数据列表):** 隶属于 `Alert` 的元数据。每个元数据包含:
* `type`: 类型 (例如: "ip", "domain", "file_hash", "user", "host")
* `value`: 具体的值
**分析逻辑 (Analysis Logic):**
1. **关联分析 (Correlation):** 不要孤立地看待每个 `Alert`。你需要将所有 `Alerts``Artifacts`
串联起来,构建一个完整的攻击链 (Kill Chain) 或故事线。例如,一个 "Phishing Email" 告警和一个 "Malware C2 Traffic"
告警如果共享同一个 `host``user`,它们必须被关联分析。
2. **识别关键实体 (Identify Key Entities):** 明确哪些 `Artifacts` 是威胁的核心?(例如: 哪个 `host` 被感染?哪个 `user`
账户失陷?哪个 `ip` 是C2服务器?)
3. **评估意图和阶段 (Assess Intent & Stage):** 判断这起事件处于什么阶段?(例如: 初始访问、执行、持久化、横向移动?)
4. **参考备注 (Use Notes):** 如果 `Case` 中已有人工 `Notes`,请将其作为重要上下文参考,但你仍需独立完成分析。
**输出格式 (Output Format):**
你的回答必须严格遵循以下Markdown格式,确保 **[综合分析摘要]** 和 **[建议的下一步行动]** 两个标题都存在且加粗。
---
**[综合分析摘要]**
*(在此处用简洁、专业的语言总结整个事件。说明发生了什么、主要威胁是什么、涉及哪些关键资产或用户,以及当前的风险状态。例如: "
此Case汇总了一次针对用户[User]的多阶段攻击。事件始于[Alert 1],导致主机[Host]上执行了恶意软件[File Hash]
,该软件随后尝试连接已知的C2服务器[IP/Domain]。主机[Host]极有可能已失陷。")*
**[建议的下一步行动]**
*(在此处提供一个按优先级排序的、具体的、可执行的步骤列表。应优先包含遏制(Containment)和调查(Investigation)动作。)*
1. **(遏制 - 优先级: 高)** 立即将主机 `[Host]` 从网络中隔离,防止威胁横向移动。
2. **(遏制 - 优先级: 高)** 在防火墙/Web代理上阻止恶意IP `[IP Address]` 和域名 `[Domain]` 的出站连接。
3. **(调查 - 优先级: 中)** 检查用户 `[User]` 的身份验证日志,确认其账户是否存在其他异常登录。
4. **(调查 - 优先级: 中)** 在EDR/SIEM中搜索文件哈希 `[File Hash]`,确认该文件是否在其他主机上存在。
5. **(修复 - 优先级: 低)** 对主机 `[Host]` 进行全盘杀毒和镜像取证,为后续的清理和重装做准备。
---
+59
View File
@@ -0,0 +1,59 @@
# 现场工程师 EDR 告警快速响应手册
**版本:** 1.0
**目标:** 快速确认威胁,阻止攻击扩散,并保留证据。
---
## 阶段 1:初次分类与分级 (Triage & Priority)
| 步骤 | 行动 (Action) | L1 检查要点 (Key Check) |
|:--------------------:|:------------------------------------|:-------------------------------------------------------------------|
| **1.1 确认资产** | 记录告警资产的主机名/IP,并查询 **资产重要性**。 | 资产是**关键服务器/高管电脑**吗?(Yes $\rightarrow$ P1,否则继续) |
| **1.2 确认告警类型** | 识别 EDR 提示的告警类型 (如:恶意软件、凭据窃取、命令行可疑)。 | 告警是**已确认的恶意软件/勒索**吗?(Yes $\rightarrow$ P1,否则继续) |
| **1.3 确认误报** | 检查该告警是否为**已知白名单程序**或**上次事件的遗留告警**。 | 告警指标(哈希/路径)在**本地白名单**中吗?(Yes $\rightarrow$ P4/关闭) |
| **1.4 确定优先级** | **资产重要性 + 威胁类型**确定最终等级。 | **P1(高):** 关键资产 + 确定恶意;**P2(中):** 重要资产/可疑行为;**P3(低):** 一般资产/低风险行为。 |
| **$\rightarrow$ 输出** | 确定**事件优先级(P1/P2/P3)**,创建工单。 | |
---
## 阶段 2:调查与遏制 (Investigate & Contain)
### A. 核心调查 (Core Investigation)
| 步骤 | 行动 (Action) | L2 调查要点/工具 (Tool Focus) |
|:------------------:|:---------------------------------------------|:-------------------------------------------------------------|
| **2.1 提取指标 (IoC)** | 从告警详情中提取:**文件哈希、进程路径、外部 IP/域名**。 | **EDR 告警详情页**:复制所有可疑指标。 |
| **2.2 威胁情报查询** | 将 IoC 输入**威胁情报平台**(如:VirusTotal, Splunk TI)。 | **判断:** 恶意指标是否有**5+ 命中**或**高恶意评分** (Yes $\rightarrow$ 确认恶意) |
| **2.3 进程与时间线** | 在 EDR 中查看**告警时间点前后**的**完整进程树/活动时间线**。 | **关键:** 恶意进程的**父进程**是什么?是否有**横向移动**迹象 (如:RDP/WMI 连接)。 |
| **2.4 确认攻击范围** | 检查是否有**其他**资产也收到**相同**告警。 | **EDR 搜索功能**:查询相同哈希/IP/进程名。 |
### B. 快速遏制 (Rapid Containment) - **P1/P2 事件必须执行**
| 步骤 | 行动 (Action) | 目的/工具 (Purpose) |
|:--------------:|:-------------------------------|:----------------------------------|
| **2.5 隔离端点** | **立即**通过 EDR 平台**隔离**受感染的主机。 | **阻止威胁扩散 (横向移动)**。确保仅保留 EDR 管理通道。 |
| **2.6 终止恶意进程** | 通过 EDR **终止**所有确认恶意的**进程树**。 | **阻止当前活动** (如:加密、数据窃取)。 |
| **2.7 封锁外部连接** | 将恶意外部 IP/域名通过**防火墙/网络设备**进行封锁。 | **切断 C2 通信** (命令与控制)。 |
---
## 阶段 3:根除、恢复与总结 (Eradication & Review)
| 步骤 | 行动 (Action) | L2 工程师职责 (Task) |
|:------------:|:---------------------------------------------|:------------------|
| **3.1 彻底清除** | **(需 L3 或系统管理员支持)** 删除恶意文件、清除持久化机制(注册表、启动项)。 | **记录**清除的操作路径和结果。 |
| **3.2 恢复验证** | 运行完整的 EDR/杀毒扫描,并检查**是否有新告警**。 | 确保系统已**干净**。 |
| **3.3 解除隔离** | **确认安全后**,通过 EDR **解除资产隔离**。 | **恢复业务**。 |
| **3.4 记录总结** | 填写完整的**事件处理时间线、根因分析 (RCA) 和采取的措施**。 | **关闭工单**,并记录经验教训。 |
---
## 💡 快速判断表 (Quick Reference)
| 进程行为 | 恶意可能性 | 建议行动 |
|:--------------------------------------------------------------------:|:-------------------:|:-----------------------|
| `cmd.exe`/`powershell.exe` **作为** `word.exe`/`excel.exe` **的子进程**运行。 | **高** (钓鱼邮件附件执行) | **P1 响应**,检查命令行参数。 |
| 发现**`mimikatz.exe`**或相似**凭据转储**工具运行。 | **P1 (严重)** (高价值目标) | **P1 响应**,立即隔离并重置用户密码。 |
| EDR 报告**大量文件快速重命名/加密**。 | **P1 (勒索)** | **立即隔离**,通知系统管理员准备恢复。 |
| 外部 IP 命中**高风险**情报,且与内部主机有大量通信。 | **高** (C2 通信) | **封锁 IP**,检查数据外泄情况。 |
+61
View File
@@ -0,0 +1,61 @@
---
## 🎣 钓鱼邮件快速处理手册 (L1/L2 Playbook)
**目标:** 快速识别、遏制、清除恶意邮件及其影响,并保护受害者账户。
---
## 阶段 1:报告与初次分类 (Reporting & Triage)
| 步骤 | 行动 (Action) | L1 检查要点 (Key Check) |
|:--------------------:|:------------------------------------------------|:---------------------------------------------------------------------------------|
| **1.1 接收报告** | 确认报告来源:**用户转发/报告**或 **邮件网关告警**。 | **是否已在沙箱环境中打开?** (Yes $\rightarrow$ P1) |
| **1.2 确认原始邮件** | 从原始邮件头中获取关键信息,包括 **发件人 IP**、**原始收件人** 和 **主题**。 | 邮件是否绕过了**邮件网关**直接投递? |
| **1.3 初步 IoC 提取** | 提取邮件中的所有 **URL 链接****附件哈希值**。 | 附件是**可执行文件 (.exe/.scr)** 或 **宏文档 (.docm/.xlsm)** 吗? (Yes $\rightarrow$ P1) |
| **1.4 确定优先级** | 根据威胁指标和感染情况分级。 | **P1(高):** 附件已打开/链接已点击/凭据已输入;**P2(中):** 恶意邮件被大规模投递但无确认点击;**P3(低):** 单个可疑但低风险的邮件。 |
| **$\rightarrow$ 输出** | 确定**事件优先级(P1/P2/P3)**,创建工单。 | |
---
## 阶段 2:调查与遏制 (Investigation & Containment)
### A. 核心调查 (Core Investigation)
| 步骤 | 行动 (Action) | L2 调查要点/工具 (Tool Focus) |
|:--------------:|:----------------------------------------------------------|:------------------------------------------|
| **2.1 威胁情报查询** | 将提取的 **URL、发件人 IP、附件哈希**输入威胁情报平台(如 VT)。 | **判断:** URL 或 IP 是否被标记为**恶意 C2 或钓鱼页面**? |
| **2.2 确认攻击范围** | 在**邮件网关/O365** 中搜索:**相同主题/发件人/哈希**的邮件投递了多少用户? | 记录所有**被投递和被点击**的用户列表。 |
| **2.3 模拟用户行为** | **(P1 专用)** 将附件在**沙箱**中运行,或在**虚拟机**中访问链接(**切勿在工作环境中执行**)。 | **判断:** 链接是否指向**凭据窃取页面**?附件是否释放了**恶意文件**? |
### B. 快速遏制与清理 (Rapid Containment & Cleanup)
| 步骤 | 行动 (Action) | 目的/工具 (Purpose) |
|:---------------:|:------------------------------------------------------------------|:--------------------------------|
| **2.4 邮件回收/删除** | 利用邮件网关或邮件服务器工具(如 Exchange/O365 Purge**立即从所有收件箱中删除**该邮件。 | **阻止二次感染**。确保 L1 工程师已获取并保存原始副本。 |
| **2.5 网络封锁** | 将所有已确认恶意的 **URL 链接****发件人 IP** 导入**防火墙/代理服务器**进行封锁。 | **阻止用户访问钓鱼页面**或与恶意 C2 通信。 |
| **2.6 保护受害者** | **(如果确认用户点击/输入凭据)** 立即联系身份系统:**强制重置用户密码**,并检查**多因素认证(MFA)**是否被禁用。 | **阻止账户被盗用**,防止横向移动。 |
---
## 阶段 3:根除、恢复与总结 (Eradication & Review)
| 步骤 | 行动 (Action) | L2 工程师职责 (Task) |
|:------------:|:-------------------------------------------------|:----------------------|
| **3.1 端点扫描** | **(P1 专用)** 对点击/打开附件的用户主机,通过 EDR 执行**全面扫描**。 | 确认是否植入了**后门/Loader**。 |
| **3.2 用户通知** | 向**所有**被投递邮件的用户发送**安全警告**(如果尚未发送),教育用户删除并说明事件。 | 提升全员安全意识。 |
| **3.3 规则加固** | 将 **发件人 IP/域名** 添加到**邮件网关的黑名单**,并考虑更新**邮件过滤规则**。 | **预防**未来类似攻击。 |
| **3.4 记录总结** | 填写完整的**事件处理时间线、受影响用户数和采取的措施**。 | **关闭工单**,记录经验教训,完成留档。 |
---
## 💡 快速判断表 (Quick Reference)
| 邮件特征 | 恶意可能性 | 建议行动 (P Priority) |
|:------------------------------------------------------------------:|:------------------:|:---------------------------|
| **发件人地址**与内部高管地址**高度相似**(如 `ceo@company.co` vs `ceo@company.com`)。 | **高** (BEC 商业邮件诈骗) | P1/P2,**快速回收邮件**,通知高管。 |
| 邮件**附件**是 `.zip` 压缩包,内含 `.js`, `.vbs`, 或 `.lnk` 文件。 | **极高** (恶意软件投递) | **P1 响应**,沙箱分析,全网封锁哈希。 |
| **链接 URL** 使用了 **URL 缩短服务** (如 `bit.ly`) 或 **IP 地址**。 | **高** (规避检测) | P2/P1,**模拟点击**确认目标页面,立即封锁。 |
| 邮件**正文**要求**立即回复**或**立即点击**,并声称有**“紧急业务”**。 | **中高** (社会工程学) | P2,确认目标用户是否已回复,回收邮件。 |
---
+44
View File
@@ -0,0 +1,44 @@
import os
import sys
import uuid
# Add the project root directory to Python path
project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, project_root)
from mcp.server import FastMCP
if __name__ == "__main__":
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ASF.settings")
import django
django.setup()
from Lib.llmfunc import get_dfmea_record_by_crs_id
# Define UUID file path
uuid_file_path = os.path.join("..", "Docker", "mcp_uuid")
# Try to read UUID from file
try:
with open(uuid_file_path, 'r') as f:
uuid_str = f.read().strip()
except FileNotFoundError:
# If file doesn't exist, generate new UUID and save it
uuid_str = str(uuid.uuid1()).replace('-', "")[0:16]
os.makedirs(os.path.dirname(uuid_file_path), exist_ok=True)
with open(uuid_file_path, 'w') as f:
f.write(uuid_str)
mcp = FastMCP("ASF-MCP")
host = "0.0.0.0"
port = 7001
mcp.settings.sse_path = f"/{uuid_str}/sse"
mcp.settings.message_path = f"/{uuid_str}/messages"
mcp.settings.host = host
mcp.settings.port = port
# add tools
mcp.add_tool(get_dfmea_record_by_crs_id)
print(f"mcp server url: http://your_server_ip:{port}/{uuid_str}/sse")
mcp.run(transport="sse")
+209 -209
View File
File diff suppressed because it is too large Load Diff
+25 -9
View File
@@ -1,10 +1,12 @@
import datetime
import random
from Lib.External.nocolyapi import Artifact, Alert, Case, OptionSet, Option
from Lib.api import get_current_time_string, string_to_timestamp
from Docker.mock.alert import get_mock_alerts
from Docker.mock.rule import rule_list
from Lib.External.nocolyapi import OptionSet
from Lib.External.sirpapi import Artifact, Alert, Case
from Lib.api import get_current_time_string, string_to_timestamp
from Lib.ruledefinition import RuleDefinition
def generate_four_random_timestamps(
@@ -106,7 +108,7 @@ if __name__ == "__main__":
case_status_new = OptionSet.get_option_key_by_name_and_value("case_status", "New")
for alert in alert_list:
rule_def = ALL_RULES.get(alert["rule_id"])
rule_def: RuleDefinition = ALL_RULES.get(alert["rule_id"])
if rule_def is None:
print(f"未找到规则定义,跳过处理此告警: {alert['rule_id']}")
continue
@@ -114,7 +116,7 @@ if __name__ == "__main__":
default_times = generate_four_random_timestamps()
# artifact
artifact_rowid_list = []
artifacts = alert.get("artifacts", [])
artifacts = alert.get("artifact", [])
for artifact in artifacts:
artifact_fields = [
{"id": "type", "value": artifact["type"]},
@@ -137,7 +139,7 @@ if __name__ == "__main__":
{"id": "raw_log", "value": alert["raw_log"]},
{"id": "rule_id", "value": alert["rule_id"]},
{"id": "rule_name", "value": alert["rule_name"]},
{"id": "artifacts", "value": artifact_rowid_list},
{"id": "artifact", "value": artifact_rowid_list},
]
# alert
row_id_alert = Alert.create(alert_fields)
@@ -150,6 +152,13 @@ if __name__ == "__main__":
row = Case.get_by_deduplication_key(deduplication_key)
if row is None:
if rule_def.source == "EDR":
workbook = Case.load_workbook_md("EDR_L2_WORKBOOK")
elif rule_def.source == "Email":
workbook = Case.load_workbook_md("PHISHING_L2_WORKBOOK")
else:
workbook = "# There is no workbook for this source."
case_field = [
{"id": "deduplication_key", "value": deduplication_key},
{"id": "title", "value": rule_def.generate_case_title(artifacts=artifacts)},
@@ -159,13 +168,20 @@ if __name__ == "__main__":
{"id": "created_date", "value": default_times["created_date"]},
{"id": "tags", "value": alert["tags"], "type": 2},
{"id": "description", "value": alert["description"]},
{"id": "alert", "value": [row_id_alert]},
{"id": "description", "value": alert["description"]},
{"id": "workbook", "value": workbook},
{"id": "acknowledged_date", "value": default_times["acknowledged_date"]},
{"id": "closed_date", "value": default_times["closed_date"]},
]
row_id_create = Case.create(case_field)
try:
row_id_create = Case.create(case_field)
except Exception as e:
print(f"创建工单失败: {e}")
continue
print(f"create case: {row_id_create}")
else:
row_id_case = row.get("rowId")
@@ -175,7 +191,7 @@ if __name__ == "__main__":
option_new_score = OptionSet.get_option_by_name_and_value("alert_case_severity", alert["severity"]).get("score", 0)
severity_value_exist = row.get("severity")[0].get("value")
severity_value_exist = row.get("severity")
option_exist_score = OptionSet.get_option_by_name_and_value("alert_case_severity", severity_value_exist).get("score", 0)
if option_new_score > option_exist_score:
@@ -183,7 +199,7 @@ if __name__ == "__main__":
else:
severity = severity_value_exist
tags_exist = Option.to_value_list(row.get("tags", []))
tags_exist = row.get("tags", [])
for tag in alert["tags"]:
if tag not in tags_exist:
tags_exist.append(tag)
+12 -11
View File
@@ -2,20 +2,21 @@ from Lib.External.nocolyapi import WorksheetRow
if __name__ == "__main__":
filter = {
"type": "group",
"logic": "OR",
"children": [
# {
# "type": "condition",
# "field": "deduplication_key",
# "operator": "isnotempty",
# },
]
# "type": "group",
# "logic": "OR",
# "children": [
# {
# "type": "condition",
# "field": "deduplication_key",
# "operator": "isnotempty",
# },
# ]
}
worksheet_id_list = ["artifact", "alert", "case"]
for worksheet_id in worksheet_id_list:
row_ids = []
rows = WorksheetRow.list(worksheet_id, filter=filter, fields=["rowId"])
rows = WorksheetRow.list(worksheet_id, filter=filter)
for row in rows:
row_ids.append(row["rowId"])
WorksheetRow.delete(worksheet_id=worksheet_id, row_ids=row_ids)
if row_ids:
WorksheetRow.delete(worksheet_id=worksheet_id, row_ids=row_ids)
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+1
View File
@@ -1,6 +1,7 @@
services:
redis-stack:
image: redis/redis-stack:latest
#image: docker.1ms.run/redis/redis-stack:latest
container_name: redis-stack
restart: always
ports:
+16
View File
@@ -53,3 +53,19 @@ class WebhookKibanaView(BaseView):
logger.exception(E)
context = data_return(500, {}, CODE_MSG_ZH.get(500), CODE_MSG_EN.get(500))
return Response(context)
class WebhookNocolyMailView(BaseView):
permission_classes = [AllowAny]
authentication_classes = []
def create(self, request, **kwargs):
try:
data = request.data
print(data)
context = data_return(200, {}, CODE_MSG_ZH.get(200), CODE_MSG_EN.get(200))
return Response(context)
except Exception as E:
logger.exception(E)
context = data_return(500, {}, CODE_MSG_ZH.get(500), CODE_MSG_EN.get(500))
return Response(context)
-24
View File
@@ -1,24 +0,0 @@
import requests
from CONFIG import NOCODB_URL, NOCODB_TOKEN, NOCODB_ALERT_TABLE_ID
class NocodbClient(object):
def __init__(self):
pass
@staticmethod
def create_alert(record: dict):
headers = {"xc-token": NOCODB_TOKEN}
url = f"{NOCODB_URL}/api/v2/tables/{NOCODB_ALERT_TABLE_ID}/records"
try:
response = requests.post(url,
headers=headers,
json=record)
response.raise_for_status()
response_data = response.json()
return response_data
except Exception as e:
raise
+114 -277
View File
@@ -3,8 +3,9 @@ from typing import TypedDict, Literal, List, Union, Any, Dict, Optional
import requests
from CONFIG import SIRP_URL, SIRP_APPKEY, SIRP_SIGN
from Lib.api import get_current_time_string, string_to_timestamp
from Lib.ruledefinition import RuleDefinition
HEADERS = {"HAP-Appkey": SIRP_APPKEY,
"HAP-Sign": SIRP_SIGN}
class InputAlert(TypedDict):
@@ -19,8 +20,8 @@ class InputAlert(TypedDict):
reference: str
description: str
summary_ai: Optional[Union[str, Dict[str, Any]]]
artifacts: List[Dict]
raw_log: Dict
artifact: List[Dict]
class FieldType(TypedDict):
@@ -66,14 +67,13 @@ class Worksheet(object):
@staticmethod
def get_fields(worksheet_id: str) -> Dict[str, FieldType]:
headers = {"HAP-Appkey": SIRP_APPKEY,
"HAP-Sign": SIRP_SIGN}
url = f"{SIRP_URL}/api/v3/app/worksheets/{worksheet_id}"
response = requests.get(
url,
params={"includeSystemFields": True},
headers=headers
headers=HEADERS
)
response.raise_for_status()
@@ -93,29 +93,21 @@ class WorksheetRow(object):
pass
@staticmethod
def get(worksheet_id: str, row_id: str):
headers = {"HAP-Appkey": SIRP_APPKEY,
"HAP-Sign": SIRP_SIGN}
def get(worksheet_id: str, row_id: str, include_system_fields=True):
url = f"{SIRP_URL}/api/v3/app/worksheets/{worksheet_id}/rows/{row_id}"
fields = Worksheet.get_fields(worksheet_id)
try:
response = requests.get(
url,
params={"includeSystemFields": True},
headers=headers
params={"includeSystemFields": include_system_fields},
headers=HEADERS
)
response.raise_for_status()
response_data = response.json()
if response_data.get("success"):
data = response_data.get("data")
data_new = {}
for key in data:
if key.startswith("_") or key == "rowId":
data_new[key] = data[key]
else:
alias = fields.get(key).get('alias')
data_new[alias] = data[key]
row = response_data.get("data")
data_new = WorksheetRow._format_row(row, fields, include_system_fields)
return data_new
else:
raise Exception(f"error_code: {response_data.get('error_code')} error_msg: {response_data.get('error_msg')}")
@@ -123,12 +115,53 @@ class WorksheetRow(object):
raise
@staticmethod
def list(worksheet_id: str, filter: dict, fields: list = None):
headers = {"HAP-Appkey": SIRP_APPKEY,
"HAP-Sign": SIRP_SIGN}
def _format_row(row, fields, include_system_fields=True):
data_new = {}
for key in row:
if key.startswith("_"):
if include_system_fields:
data_new[key] = row[key]
else:
continue
elif key == "rowId":
data_new[key] = row[key]
else:
alias = fields.get(key).get('alias')
data_new[alias] = WorksheetRow._format_value(fields.get(key), row[key])
return data_new
@staticmethod
def _format_value(field, value):
field_type = field.get("type")
sub_type = field.get("subType")
if field_type in ["MultipleSelect"]:
value_list = []
for option in value:
value_list.append(option.get("value"))
return value_list
elif field_type in ['SingleSelect', "Dropdown"]:
if len(value) > 0:
return value[0].get("value")
else:
return None
elif field_type in ['Relation']:
if sub_type == 1:
value_list = []
for option in value:
value_list.append(option.get("sid"))
return value_list
else:
return value
elif field_type in ['Checkbox']:
return bool(int(value))
else:
return value
@staticmethod
def list(worksheet_id: str, filter: dict, include_system_fields=True):
url = f"{SIRP_URL}/api/v3/app/worksheets/{worksheet_id}/rows/list"
data = {
"fields": fields,
# "fields": fields,
"filter": filter,
"sorts": [
{
@@ -137,19 +170,24 @@ class WorksheetRow(object):
}
],
"includeTotalCount": True,
"includeSystemFields": True,
"useFieldIdAsKey": False,
"includeSystemFields": include_system_fields,
# "useFieldIdAsKey": False,
"pageSize": 1000,
}
try:
response = requests.post(url,
headers=headers,
headers=HEADERS,
json=data)
response.raise_for_status()
response_data = response.json()
if response_data.get("success"):
return response_data.get("data").get("rows")
fields = Worksheet.get_fields(worksheet_id)
rows = response_data.get("data").get("rows")
rows_new = []
for row in rows:
data_new = WorksheetRow._format_row(row, fields, include_system_fields)
rows_new.append(data_new)
return rows_new
else:
raise Exception(f"error_code: {response_data.get('error_code')} error_msg: {response_data.get('error_msg')}")
except Exception as e:
@@ -157,8 +195,7 @@ class WorksheetRow(object):
@staticmethod
def create(worksheet_id: str, fields: list):
headers = {"HAP-Appkey": SIRP_APPKEY,
"HAP-Sign": SIRP_SIGN}
url = f"{SIRP_URL}/api/v3/app/worksheets/{worksheet_id}/rows"
data = {
@@ -168,7 +205,7 @@ class WorksheetRow(object):
try:
response = requests.post(url,
headers=headers,
headers=HEADERS,
json=data)
response.raise_for_status()
@@ -178,12 +215,10 @@ class WorksheetRow(object):
else:
raise Exception(f"error_code: {response_data.get('error_code')} error_msg: {response_data.get('error_msg')} data: {response_data.get('data')}")
except Exception as e:
raise
raise e
@staticmethod
def update(worksheet_id: str, row_id: str, fields: list):
headers = {"HAP-Appkey": SIRP_APPKEY,
"HAP-Sign": SIRP_SIGN}
url = f"{SIRP_URL}/api/v3/app/worksheets/{worksheet_id}/rows/{row_id}"
data = {
@@ -193,7 +228,7 @@ class WorksheetRow(object):
try:
response = requests.patch(url,
headers=headers,
headers=HEADERS,
json=data)
response.raise_for_status()
@@ -207,8 +242,7 @@ class WorksheetRow(object):
@staticmethod
def delete(worksheet_id: str, row_ids: list):
headers = {"HAP-Appkey": SIRP_APPKEY,
"HAP-Sign": SIRP_SIGN}
url = f"{SIRP_URL}/api/v3/app/worksheets/{worksheet_id}/rows/batch"
data = {
@@ -218,7 +252,7 @@ class WorksheetRow(object):
try:
response = requests.delete(url,
headers=headers,
headers=HEADERS,
json=data)
response.raise_for_status()
@@ -230,6 +264,46 @@ class WorksheetRow(object):
except Exception as e:
raise
@staticmethod
def relations(worksheet_id: str, row_id: str, field: str, relation_worksheet_id: str, include_system_fields: bool = True, page_size: int = 1000,
page_index: int = None):
fields = Worksheet.get_fields(relation_worksheet_id)
url = f"{SIRP_URL}/api/v3/app/worksheets/{worksheet_id}/rows/{row_id}/relations/{field}"
params = {}
if page_size is not None:
params["pageSize"] = page_size
if page_index is not None:
params["pageIndex"] = page_index
if include_system_fields is not None:
params["isReturnSystemFields"] = include_system_fields
try:
response = requests.get(url,
headers=HEADERS,
params=params)
response.raise_for_status()
response_data = response.json()
if response_data.get("success"):
rows = response_data.get("data").get("rows")
rows_new = []
for row in rows:
data_new = WorksheetRow._format_row(row, fields, include_system_fields)
rows_new.append(data_new)
return rows_new
else:
raise Exception(f"error_code: {response_data.get('error_code')} error_msg: {response_data.get('error_msg')}")
except Exception as e:
raise
@staticmethod
def get_rowid_list_from_rowid(rowid):
# 多行数据获取列表
tmp = rowid.split("_")
rowid_list = tmp[0].split(",")
return rowid_list
class OptionSet(object):
def __init__(self):
@@ -237,12 +311,11 @@ class OptionSet(object):
@staticmethod
def list():
headers = {"HAP-Appkey": SIRP_APPKEY,
"HAP-Sign": SIRP_SIGN}
url = f"{SIRP_URL}/api/v3/app/optionsets"
response = requests.get(url,
headers=headers)
headers=HEADERS)
response.raise_for_status()
response_data = response.json()
@@ -280,239 +353,3 @@ class OptionSet(object):
if option["value"] == value:
return option["key"]
raise Exception(f"optionset {name} {value} not found")
class Artifact(object):
WORKSHEET_ID = "artifact"
def __init__(self):
pass
@staticmethod
def list(filter: dict):
result = WorksheetRow.list(Artifact.WORKSHEET_ID, filter)
return result
@staticmethod
def update(rowid, fields: list):
row_id = WorksheetRow.update(Artifact.WORKSHEET_ID, rowid, fields)
return row_id
@staticmethod
def create(fields: list):
row_id = WorksheetRow.create(Artifact.WORKSHEET_ID, fields)
return row_id
@staticmethod
def get_by_deduplication_key(deduplication_key: str):
filter = {
"type": "group",
"logic": "AND",
"children": [
{
"type": "condition",
"field": "deduplication_key",
"operator": "eq",
"value": deduplication_key
},
]
}
rows = WorksheetRow.list(Artifact.WORKSHEET_ID, filter)
if rows:
if len(rows) > 1:
raise Exception(f"found multiple rows with deduplication_key {deduplication_key}")
return rows[0]
else:
return None
@staticmethod
def update_by_type_and_value(data: dict):
# 第一层必须是group
filter = {
"type": "group",
"logic": "AND",
"children": [
{
"type": "condition",
"field": "type",
"operator": "eq",
"value": data["type"]
},
{
"type": "condition",
"field": "value",
"operator": "eq",
"value": data["value"]
}
]
}
rows = Artifact.list(filter)
if rows:
for row in rows:
rowid = row['rowId']
fields = [
{"id": "enrichment", "value": data["enrichment"]},
]
rowid_updated = Artifact.update(rowid, fields)
return rowid_updated
else:
fields = [
{"id": "type", "value": data["type"], "type": 2},
{"id": "value", "value": data["value"]},
{"id": "enrichment", "value": data["enrichment"]},
]
rowid_created = Artifact.create(fields)
return rowid_created
class Alert(object):
WORKSHEET_ID = "alert"
def __init__(self):
pass
@staticmethod
def create(fields: list):
row_id = WorksheetRow.create(Alert.WORKSHEET_ID, fields)
return row_id
class Case(object):
WORKSHEET_ID = "case"
def __init__(self):
pass
@staticmethod
def create(fields: list):
row_id = WorksheetRow.create(Case.WORKSHEET_ID, fields)
return row_id
@staticmethod
def update(row_id, fields: list):
row_id = WorksheetRow.update(Case.WORKSHEET_ID, row_id, fields)
return row_id
@staticmethod
def get_by_deduplication_key(deduplication_key: str):
filter = {
"type": "group",
"logic": "AND",
"children": [
{
"type": "condition",
"field": "deduplication_key",
"operator": "eq",
"value": deduplication_key
},
]
}
rows = WorksheetRow.list(Case.WORKSHEET_ID, filter)
if rows:
if len(rows) > 1:
raise Exception(f"found multiple rows with deduplication_key {deduplication_key}")
return rows[0]
else:
return None
class Option(object):
def __init__(self):
pass
@staticmethod
def to_value_list(options: list):
value_list = []
for option in options:
value_list.append(option.get("value"))
return value_list
def common_handler(alert: InputAlert, rule_def: RuleDefinition) -> str:
# artifact
artifact_rowid_list = []
artifacts = alert.get("artifacts", [])
for artifact in artifacts:
deduplication_key = artifact["deduplication_key"]
artifact_fields = [
{"id": "type", "value": artifact["type"]},
{"id": "value", "value": artifact["value"]},
{"id": "enrichment", "value": artifact["enrichment"]},
{"id": "deduplication_key", "value": deduplication_key},
]
row = Artifact.get_by_deduplication_key(deduplication_key)
if row is None:
row_id = Artifact.create(artifact_fields)
else:
row_id = row.get("rowId")
Artifact.update(row_id, artifact_fields)
artifact_rowid_list.append(row_id)
alert_fields = [
{"id": "tags", "value": alert.get("tags"), "type": 2},
{"id": "severity", "value": alert.get("severity")},
{"id": "source", "value": alert.get("source")},
{"id": "alert_date", "value": alert.get("alert_date")},
{"id": "created_date", "value": alert.get("created_date")},
{"id": "reference", "value": alert.get("reference")},
{"id": "description", "value": alert.get("description")},
{"id": "raw_log", "value": alert.get("raw_log")},
{"id": "rule_id", "value": alert.get("rule_id")},
{"id": "rule_name", "value": alert.get("rule_name")},
{"id": "name", "value": alert.get("name")},
{"id": "summary_ai", "value": alert.get("summary_ai")},
{"id": "artifacts", "value": artifact_rowid_list},
]
# alert
row_id_alert = Alert.create(alert_fields)
# case
timestamp = string_to_timestamp(alert["alert_date"], "%Y-%m-%dT%H:%M:%SZ")
deduplication_key = rule_def.generate_deduplication_key(artifacts=artifacts, timestamp=timestamp)
row = Case.get_by_deduplication_key(deduplication_key)
if row is None:
case_field = [
{"id": "title", "value": rule_def.generate_case_title(artifacts=artifacts)},
{"id": "deduplication_key", "value": deduplication_key},
{"id": "alert", "value": [row_id_alert]},
{"id": "case_status", "value": "New"},
{"id": "created_at", "value": get_current_time_string()},
{"id": "tags", "value": alert["tags"], "type": 2},
{"id": "severity", "value": alert["severity"]},
{"id": "type", "value": rule_def.source},
{"id": "description", "value": alert["description"]},
]
row_id_create = Case.create(case_field)
return row_id_create
else:
row_id_case = row.get("rowId")
existing_alerts = row.get("alert", [])
if row_id_alert not in existing_alerts:
existing_alerts.append(row_id_alert)
option_new_score = OptionSet.get_option_by_name_and_value("alert_case_severity", alert["severity"]).get("score", 0)
severity_value_exist = row.get("severity")[0].get("value")
option_exist_score = OptionSet.get_option_by_name_and_value("alert_case_severity", severity_value_exist).get("score", 0)
if option_new_score > option_exist_score:
severity = alert["severity"]
else:
severity = severity_value_exist
tags_exist = Option.to_value_list(row.get("tags", []))
for tag in alert["tags"]:
if tag not in tags_exist:
tags_exist.append(tag)
case_field = [
{"id": "alert", "value": existing_alerts},
{"id": "severity", "value": severity},
{"id": "tags", "value": tags_exist, "type": 2}
]
row_id_updated = Case.update(row_id_case, case_field)
return row_id_updated
+284
View File
@@ -0,0 +1,284 @@
import os
import requests
from CONFIG import SIRP_NOTICE_WEBHOOK
from Lib.External.nocolyapi import WorksheetRow, InputAlert, OptionSet
from Lib.api import string_to_timestamp, get_current_time_string
from Lib.ruledefinition import RuleDefinition
class Artifact(object):
WORKSHEET_ID = "artifact"
def __init__(self):
pass
@staticmethod
def list(filter: dict):
result = WorksheetRow.list(Artifact.WORKSHEET_ID, filter)
return result
@staticmethod
def update(rowid, fields: list):
row_id = WorksheetRow.update(Artifact.WORKSHEET_ID, rowid, fields)
return row_id
@staticmethod
def create(fields: list):
row_id = WorksheetRow.create(Artifact.WORKSHEET_ID, fields)
return row_id
@staticmethod
def get_by_deduplication_key(deduplication_key: str):
filter = {
"type": "group",
"logic": "AND",
"children": [
{
"type": "condition",
"field": "deduplication_key",
"operator": "eq",
"value": deduplication_key
},
]
}
rows = WorksheetRow.list(Artifact.WORKSHEET_ID, filter)
if rows:
if len(rows) > 1:
raise Exception(f"found multiple rows with deduplication_key {deduplication_key}")
return rows[0]
else:
return None
@staticmethod
def update_by_type_and_value(data: dict):
# 第一层必须是group
filter = {
"type": "group",
"logic": "AND",
"children": [
{
"type": "condition",
"field": "type",
"operator": "eq",
"value": data["type"]
},
{
"type": "condition",
"field": "value",
"operator": "eq",
"value": data["value"]
}
]
}
rows = Artifact.list(filter)
if rows:
for row in rows:
rowid = row['rowId']
fields = [
{"id": "enrichment", "value": data["enrichment"]},
]
rowid_updated = Artifact.update(rowid, fields)
return rowid_updated
else:
fields = [
{"id": "type", "value": data["type"], "type": 2},
{"id": "value", "value": data["value"]},
{"id": "enrichment", "value": data["enrichment"]},
]
rowid_created = Artifact.create(fields)
return rowid_created
class Alert(object):
WORKSHEET_ID = "alert"
def __init__(self):
pass
@staticmethod
def create(fields: list):
row_id = WorksheetRow.create(Alert.WORKSHEET_ID, fields)
return row_id
class Case(object):
WORKSHEET_ID = "case"
def __init__(self):
pass
@staticmethod
def create(fields: list):
row_id = WorksheetRow.create(Case.WORKSHEET_ID, fields)
return row_id
@staticmethod
def update(row_id, fields: list):
row_id = WorksheetRow.update(Case.WORKSHEET_ID, row_id, fields)
return row_id
@staticmethod
def get_by_deduplication_key(deduplication_key: str):
filter = {
"type": "group",
"logic": "AND",
"children": [
{
"type": "condition",
"field": "deduplication_key",
"operator": "eq",
"value": deduplication_key
},
]
}
rows = WorksheetRow.list(Case.WORKSHEET_ID, filter)
if rows:
if len(rows) > 1:
raise Exception(f"found multiple rows with deduplication_key {deduplication_key}")
return rows[0]
else:
return None
@staticmethod
def load_workbook_md(workbook_name: str) -> str:
"""
根据 workbook 名称读取 DATA/WORKBOOK/{workbook_name}.md 的内容并返回字符串
路径相对于项目根 (两级向上到 asf 文件夹)
"""
base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
md_path = os.path.join(base_dir, 'DATA', 'WORKBOOK', f"{workbook_name}.md")
if not os.path.exists(md_path):
raise FileNotFoundError(f"workbook md not found: {md_path}")
with open(md_path, 'r', encoding='utf-8') as f:
return f.read()
class Playbook(object):
WORKSHEET_ID = "playbook"
def __init__(self):
pass
@staticmethod
def create(fields: list):
row_id = WorksheetRow.create(Playbook.WORKSHEET_ID, fields)
return row_id
@staticmethod
def update(row_id, fields: list):
row_id = WorksheetRow.update(Playbook.WORKSHEET_ID, row_id, fields)
return row_id
@staticmethod
def update_status_and_remark(row_id, status, remark):
fields = [
{"id": "job_status", "value": status},
{"id": "remark", "value": remark},
]
row_id = WorksheetRow.update(Playbook.WORKSHEET_ID, row_id, fields)
return row_id
class Notice(object):
@staticmethod
def send(user, title, body=None):
result = requests.post(SIRP_NOTICE_WEBHOOK, json={"title": title, "body": body, "user": user})
return result
def common_handler(alert: InputAlert, rule_def: RuleDefinition) -> str:
artifact_rowid_list = []
artifacts = alert.get("artifact", [])
for artifact in artifacts:
deduplication_key = artifact["deduplication_key"]
artifact_fields = [
{"id": "type", "value": artifact["type"]},
{"id": "value", "value": artifact["value"]},
{"id": "enrichment", "value": artifact["enrichment"]},
{"id": "deduplication_key", "value": deduplication_key},
]
row = Artifact.get_by_deduplication_key(deduplication_key)
if row is None:
row_id = Artifact.create(artifact_fields)
else:
row_id = row.get("rowId")
Artifact.update(row_id, artifact_fields)
artifact_rowid_list.append(row_id)
alert_fields = [
{"id": "tags", "value": alert.get("tags"), "type": 2},
{"id": "severity", "value": alert.get("severity")},
{"id": "source", "value": alert.get("source")},
{"id": "alert_date", "value": alert.get("alert_date")},
{"id": "created_date", "value": alert.get("created_date")},
{"id": "reference", "value": alert.get("reference")},
{"id": "description", "value": alert.get("description")},
{"id": "raw_log", "value": alert.get("raw_log")},
{"id": "rule_id", "value": alert.get("rule_id")},
{"id": "rule_name", "value": alert.get("rule_name")},
{"id": "name", "value": alert.get("name")},
{"id": "summary_ai", "value": alert.get("summary_ai")},
{"id": "artifact", "value": artifact_rowid_list},
]
# alert
row_id_alert = Alert.create(alert_fields)
# case
timestamp = string_to_timestamp(alert["alert_date"], "%Y-%m-%dT%H:%M:%SZ")
deduplication_key = rule_def.generate_deduplication_key(artifacts=artifacts, timestamp=timestamp)
row = Case.get_by_deduplication_key(deduplication_key)
if row is None:
if rule_def.workbook is not None:
workbook = Case.load_workbook_md(rule_def.workbook)
else:
workbook = "# There is no workbook for this source."
case_field = [
{"id": "title", "value": rule_def.generate_case_title(artifacts=artifacts)},
{"id": "deduplication_key", "value": deduplication_key},
{"id": "alert", "value": [row_id_alert]},
{"id": "case_status", "value": "New"},
{"id": "created_at", "value": get_current_time_string()},
{"id": "tags", "value": alert["tags"], "type": 2},
{"id": "severity", "value": alert["severity"]},
{"id": "type", "value": rule_def.source},
{"id": "description", "value": alert["description"]},
{"id": "workbook", "value": workbook},
]
row_id_create = Case.create(case_field)
return row_id_create
else:
row_id_case = row.get("rowId")
existing_alerts = row.get("alert", [])
if row_id_alert not in existing_alerts:
existing_alerts.append(row_id_alert)
option_new_score = OptionSet.get_option_by_name_and_value("alert_case_severity", alert["severity"]).get("score", 0)
severity_value_exist = row.get("severity")
option_exist_score = OptionSet.get_option_by_name_and_value("alert_case_severity", severity_value_exist).get("score", 0)
if option_new_score > option_exist_score:
severity = alert["severity"]
else:
severity = severity_value_exist
tags_exist = row.get("tags", [])
for tag in alert["tags"]:
if tag not in tags_exist:
tags_exist.append(tag)
case_field = [
{"id": "alert", "value": existing_alerts},
{"id": "severity", "value": severity},
{"id": "tags", "value": tags_exist, "type": 2}
]
row_id_updated = Case.update(row_id_case, case_field)
return row_id_updated
+90 -13
View File
@@ -1,3 +1,4 @@
import base64
import datetime
import ipaddress
import json
@@ -11,6 +12,7 @@ import subprocess
import time
import uuid
from collections import OrderedDict
from io import BytesIO
from urllib.parse import urlparse
import dns.resolver
@@ -331,45 +333,78 @@ def get_dns_a(domain):
return []
def write_list_of_dict_to_excel_sheet(data_list: list[dict], file_path: str, sheet_name: str):
def write_list_of_dict_to_excel_sheet(data_list: list[dict], file_path: str = '', sheet_name: str = None, return_content=False):
"""
仅使用 openpyxl 库将字典列表中的数据写入指定的 XLSX 文件和 sheet
逻辑
- 如果 XLSX 文件存在则打开文件否则创建文件
- 如果 XLSX 文件存在则打开文件否则创建新的 Workbook
- 指定 sheet 如果存在则覆盖删除旧的创建新的如果不存在则创建
- 字典的 key 作为表头
- return_content True 不进行任何本地文件系统操作
Args:
data_list: 列表列表中的每个元素是一个字典代表一行数据
file_path: XLSX 文件的完整路径和名称
file_path: XLSX 文件的完整路径和名称仅在 return_content=False 时使用
sheet_name: 要写入的 sheet 的名称
return_content: 返回 Base64 编码的 Excel 内容
"""
if not data_list:
return
if os.path.exists(file_path):
workbook = load_workbook(file_path)
if not return_content and file_path and os.path.exists(file_path):
try:
workbook = load_workbook(file_path)
except Exception as e:
raise IOError(f"Error loading workbook from {file_path}: {e}")
else:
workbook = Workbook()
if sheet_name in workbook.sheetnames:
del workbook[sheet_name]
final_sheet_name = sheet_name if sheet_name else 'Sheet1' # 默认使用 'Sheet1'
worksheet = workbook.create_sheet(title=sheet_name, index=0)
if final_sheet_name in workbook.sheetnames:
del workbook[final_sheet_name]
worksheet = workbook.create_sheet(title=final_sheet_name, index=0)
header = list(data_list[0].keys())
worksheet.append(header)
for row_dict in data_list:
# 按照 header 的顺序提取字典的值作为一行数据
row_values = [row_dict.get(key, '') for key in header]
worksheet.append(row_values)
os.makedirs(os.path.dirname(file_path) or '.', exist_ok=True)
if not os.path.exists(file_path) and 'Sheet' in workbook.sheetnames and workbook['Sheet'].max_row == 1 and workbook['Sheet'].cell(1, 1).value is None:
del workbook['Sheet']
if 'Sheet' in workbook.sheetnames:
default_sheet = workbook['Sheet']
if default_sheet.max_row == 1 and default_sheet.cell(1, 1).value is None:
del workbook['Sheet']
workbook.save(file_path)
if return_content:
file_stream = BytesIO()
workbook.save(file_stream)
file_stream.seek(0)
excel_bytes = file_stream.getvalue()
base64_content = base64.b64encode(excel_bytes).decode('utf-8')
return base64_content
else:
if not file_path:
raise ValueError("file_path cannot be empty when return_content is False")
directory = os.path.dirname(file_path)
if directory:
os.makedirs(directory, exist_ok=True)
try:
workbook.save(file_path)
except Exception as e:
raise IOError(f"Error saving workbook to {file_path}: {e}")
return None
def read_excel_sheet_to_list_of_dict(file_path: str, sheet_name: str) -> list[dict]:
@@ -409,3 +444,45 @@ def read_excel_sheet_to_list_of_dict(file_path: str, sheet_name: str) -> list[di
data_list.append(row_dict)
return data_list
def read_file_and_base64(file_path: str) -> dict:
"""
读取指定路径的文件并返回包含文件名称和文件内容base64编码的字典
:param file_path: 文件的完整路径
:return: 包含文件名称和文件内容base64编码的字典
格式为 {"name": "文件名称,带后缀", "base64": "文件内容的base64编码"}
:raises FileNotFoundError: 如果文件路径不存在
:raises IOError: 如果读取文件时发生其他I/O错误
:raises Exception: 如果发生其他意外错误
"""
try:
# 检查文件是否存在
if not os.path.exists(file_path):
raise FileNotFoundError(f"文件未找到: {file_path}")
# 获取文件名称(带后缀)
file_name = os.path.basename(file_path)
# 读取文件内容并进行base64编码
with open(file_path, 'rb') as f:
file_content = f.read()
# 使用标准的base64编码
base64_encoded_content = base64.b64encode(file_content).decode('utf-8')
# 返回字典
return {
"name": file_name,
"base64": base64_encoded_content
}
except FileNotFoundError as e:
# 直接raise异常,不包含调试信息
raise e
except IOError as e:
# 直接raise异常,不包含调试信息
raise IOError(f"读取文件时发生I/O错误: {e}")
except Exception as e:
# 直接raise异常,不包含调试信息
raise Exception(f"发生未知错误: {e}")
+104
View File
@@ -0,0 +1,104 @@
# -*- coding: utf-8 -*-
# @File : apsmodule.py
# @Date : 2021/2/26
# @Desc :
import threading
import time
import uuid
from apscheduler.events import EVENT_JOB_ADDED, EVENT_JOB_REMOVED, EVENT_JOB_MODIFIED, EVENT_JOB_EXECUTED, \
EVENT_JOB_ERROR, EVENT_JOB_MISSED, EVENT_JOB_SUBMITTED, EVENT_JOB_MAX_INSTANCES
from apscheduler.schedulers.background import BackgroundScheduler
from Lib.log import logger
from Lib.xcache import Xcache
class APSModule(object):
"""处理post python模块请求,单例模式运行
EVENT_JOB_ADDED | EVENT_JOB_REMOVED | EVENT_JOB_MODIFIED |EVENT_JOB_EXECUTED |
EVENT_JOB_ERROR | EVENT_JOB_MISSED |EVENT_JOB_SUBMITTED | EVENT_JOB_MAX_INSTANCES
"""
_instance_lock = threading.Lock()
def __init__(self):
self.ModuleJobsScheduler = BackgroundScheduler()
self.ModuleJobsScheduler.add_listener(self.deal_result)
self.ModuleJobsScheduler.start()
def __new__(cls, *args, **kwargs):
if not hasattr(APSModule, "_instance"):
with APSModule._instance_lock:
if not hasattr(APSModule, "_instance"):
APSModule._instance = object.__new__(cls)
return APSModule._instance
def putin_post_python_module_queue(self, post_module_intent=None):
try:
# 存储uuid
module_uuid = str(uuid.uuid1())
logger.info(f"模块放入列表: uuid: {module_uuid}")
self.ModuleJobsScheduler.add_job(func=post_module_intent.run, max_instances=1, id=module_uuid)
# 放入缓存队列,用于后续删除任务,存储结果等
req = {
'uuid': module_uuid,
# 'module': post_module_intent, # 对象无法存储到缓存中
'time': int(time.time()),
}
Xcache.create_module_task(req)
return module_uuid
except Exception as E:
logger.exception(E)
return None
def deal_result(self, event=None):
flag = False
if event.code == EVENT_JOB_ADDED:
pass
elif event.code == EVENT_JOB_REMOVED:
pass
elif event.code == EVENT_JOB_MODIFIED:
pass
elif event.code == EVENT_JOB_EXECUTED: # 执行完成
flag = self.store_executed_result(event.job_id)
elif event.code == EVENT_JOB_ERROR:
pass
flag = self.store_error_result(event.job_id, event.exception)
elif event.code == EVENT_JOB_MISSED:
pass
elif event.code == EVENT_JOB_SUBMITTED:
pass
elif event.code == EVENT_JOB_MAX_INSTANCES:
pass
else:
pass
return flag
@staticmethod
def store_executed_result(job_id=None):
req = Xcache.get_module_task_by_uuid(task_uuid=job_id)
if req is None:
logger.warning("缓存中无对应实例,模块已中途退出")
return False
Xcache.del_module_task_by_uuid(task_uuid=job_id) # 清理缓存信息
logger.info(f"模块执行完成: uuid: {job_id}")
@staticmethod
def store_error_result(job_id=None, exception=None):
req = Xcache.get_module_task_by_uuid(task_uuid=job_id)
Xcache.del_module_task_by_uuid(task_uuid=job_id) # 清理缓存信息
logger.exception(exception)
def delete_job_by_uuid(self, job_id=None):
req = Xcache.get_module_task_by_uuid(task_uuid=job_id)
Xcache.del_module_task_by_uuid(task_uuid=job_id) # 清理缓存信息
logger.info(f"多模块实例手动删除:{job_id}")
return True
aps_module = APSModule()
+31 -8
View File
@@ -6,7 +6,7 @@ from langgraph.checkpoint.memory import MemorySaver
from langgraph.graph.state import CompiledStateGraph
from CONFIG import DIFY_API_KEY
from Lib.configs import MODULE_DATA_DIR, REDIS_CONSUMER_GROUP
from Lib.configs import DATA_DIR, REDIS_CONSUMER_GROUP
from Lib.llmapi import AgentState
from Lib.log import logger
from Lib.redis_stream_api import RedisStreamAPI
@@ -18,18 +18,40 @@ class BaseModule(object):
def __init__(self):
self._thread_name = None
self.logger = logger
self.agent_state = None
# debug
self.debug_alert_name = None
self.debug_message_id = None # 设置为非None以启用Debug模式
@staticmethod
def _get_main_script_name():
"""
获取主执行脚本的文件名不含扩展名
无论当前代码在哪个模块中运行sys.argv[0]始终指向最初启动的脚本
"""
try:
# 1. 获取主执行脚本的完整路径
script_path = sys.argv[0]
# 2. 从完整路径中提取文件名
script_filename = os.path.basename(script_path)
# 3. 分离文件名和扩展名
script_name, _ = os.path.splitext(script_filename)
return script_name
except IndexError as e:
raise RuntimeError("无法获取主执行脚本名称,sys.argv[0]不存在。") from e
except Exception as e:
raise RuntimeError(f"获取主执行脚本名称时发生错误: {e}") from e
@property
def module_name(self):
"""获取模块加载路径"""
if self.debug_alert_name is None:
return self.__module__.split(".")[-1]
module_name = self.__module__.split(".")[-1]
if module_name == "__main__":
return self._get_main_script_name()
else:
return self.debug_alert_name
return module_name
def read_message(self) -> dict:
"""读取消息"""
@@ -66,7 +88,7 @@ class LanggraphModule(BaseModule):
优先级
1. 如果传入的 filename 作为路径(可包含或不包含 .md)直接存在文件则直接读取该文件
2. 否则 MODULES_DATA/<module_name>/ 目录下按原有逻辑加载 (自动补全 .md 后缀)
2. 否则 DATA/<module_name>/ 目录下按原有逻辑加载 (自动补全 .md 后缀)
Args:
filename (str): 可以是一个直接文件路径或是模板名不含 .md
@@ -82,7 +104,7 @@ class LanggraphModule(BaseModule):
fname = filename
else:
fname = f"{filename}.md"
template_path = os.path.join(MODULE_DATA_DIR, self.module_name, fname)
template_path = os.path.join(DATA_DIR, self.module_name, fname)
try:
with open(template_path, 'r', encoding='utf-8') as f:
@@ -104,3 +126,4 @@ class LanggraphModule(BaseModule):
def run(self):
self.run_graph()
return self.agent_state
+134 -2
View File
@@ -1,16 +1,148 @@
import os
import sys
from abc import ABC
from langchain_core.prompts import SystemMessagePromptTemplate, HumanMessagePromptTemplate
from langchain_core.runnables import RunnableConfig
from langgraph.checkpoint.memory import MemorySaver
from langgraph.graph.state import CompiledStateGraph
from Lib.configs import DATA_DIR
from Lib.llmapi import AgentState
from Lib.log import logger
class BasePlaybook(ABC):
def __init__(self, params):
RUN_AS_JOB = False # 是否作为后台任务运行
def __init__(self):
super().__init__()
self._params = params
self._params = {}
self.logger = logger
@staticmethod
def _get_main_script_name():
"""
获取主执行脚本的文件名不含扩展名
无论当前代码在哪个模块中运行sys.argv[0]始终指向最初启动的脚本
"""
try:
# 1. 获取主执行脚本的完整路径
script_path = sys.argv[0]
# 2. 从完整路径中提取文件名
script_filename = os.path.basename(script_path)
# 3. 分离文件名和扩展名
script_name, _ = os.path.splitext(script_filename)
return script_name
except IndexError as e:
raise RuntimeError("无法获取主执行脚本名称,sys.argv[0]不存在。") from e
except Exception as e:
raise RuntimeError(f"获取主执行脚本名称时发生错误: {e}") from e
@property
def playbook_name(self):
"""获取模块加载路径"""
name = self.__module__.split(".")[-1]
if name == "__main__":
return self._get_main_script_name()
else:
return name
def param(self, key, default=None):
return self._params.get(key, default)
def run(self):
pass
class LanggraphPlaybook(BasePlaybook):
def __init__(self):
super().__init__()
self.graph: CompiledStateGraph = None
self.agent_state = None
## LLM PART
@staticmethod
def get_checkpointer():
checkpointer = MemorySaver()
return checkpointer
def load_system_prompt_template(self, filename):
"""加载系统提示模板。
优先级
1. 如果传入的 filename 作为路径(可包含或不包含 .md)直接存在文件则直接读取该文件
2. 否则 DATA/<module_name>/ 目录下按原有逻辑加载 (自动补全 .md 后缀)
Args:
filename (str): 可以是一个直接文件路径或是模板名不含 .md
Returns:
SystemMessagePromptTemplate: 解析后的系统提示模板对象
Raises:
Exception: 当文件无法读取时抛出
"""
if os.path.isfile(filename):
template_path = filename
else:
if filename.endswith('.md'):
fname = filename
else:
fname = f"{filename}.md"
template_path = os.path.join(DATA_DIR, self.playbook_name, fname)
try:
with open(template_path, 'r', encoding='utf-8') as f:
system_prompt_template: SystemMessagePromptTemplate = SystemMessagePromptTemplate.from_template(f.read())
logger.debug(f"Loaded system prompt template from: {template_path}")
return system_prompt_template
except Exception as e:
logger.warning(f"Failed to load prompt template {template_path}: {str(e)}")
raise e
def load_human_prompt_template(self, filename):
"""加载系统提示模板。
优先级
1. 如果传入的 filename 作为路径(可包含或不包含 .md)直接存在文件则直接读取该文件
2. 否则 DATA/<module_name>/ 目录下按原有逻辑加载 (自动补全 .md 后缀)
Args:
filename (str): 可以是一个直接文件路径或是模板名不含 .md
Returns:
SystemMessagePromptTemplate: 解析后的系统提示模板对象
Raises:
Exception: 当文件无法读取时抛出
"""
if os.path.isfile(filename):
template_path = filename
else:
if filename.endswith('.md'):
fname = filename
else:
fname = f"{filename}.md"
template_path = os.path.join(DATA_DIR, self.playbook_name, fname)
try:
with open(template_path, 'r', encoding='utf-8') as f:
human_prompt_template: HumanMessagePromptTemplate = HumanMessagePromptTemplate.from_template(f.read())
logger.debug(f"Loaded system prompt template from: {template_path}")
return human_prompt_template
except Exception as e:
logger.warning(f"Failed to load prompt template {template_path}: {str(e)}")
raise e
def run_graph(self):
self.graph.checkpointer.delete_thread(self.playbook_name)
config = RunnableConfig()
config["configurable"] = {"thread_id": self.playbook_name}
if self.agent_state is None:
self.agent_state = AgentState(messages=[], alert_raw={}, temp_data={}, analyze_result={})
for event in self.graph.stream(self.agent_state, config, stream_mode="values"):
self.logger.debug(event)
def run(self):
self.run_graph()
return self.agent_state
+1 -1
View File
@@ -1,7 +1,7 @@
import os
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
MODULE_DATA_DIR = os.path.join(BASE_DIR, 'MODULES_DATA')
DATA_DIR = os.path.join(BASE_DIR, 'DATA')
REDIS_CONSUMER_GROUP = 'AI_SOC_FRAMEWORK_GROUP'
REDIS_CONSUMER_NAME = 'AI_SOC_FRAMEWORK_CONSUMER_0'
+155
View File
@@ -0,0 +1,155 @@
from typing import Annotated
from Lib.External.nocolyapi import WorksheetRow
def get_dfmea_record_by_crs_id(
crs_id: Annotated[str, "CRS ID"]
) -> Annotated[list, "DFMEA record list with nested Requirement, Failure Mode, Failure Cause, Prevention Control"]:
"""
根据指定的CRS ID检索DFMEA记录树包括RequirementFailure ModeFailure Cause和Prevention Control的嵌套结构
参数:
crs_id (str): CRS ID用于筛选Requirement记录
返回:
list: 包含所有Requirement树的列表每个Requirement下嵌套Failure ModeFailure Cause和Prevention Control
结构示例:
[
{
"crs_id": ...,
"requirement": ...,
"function": ...,
"classification": ...,
"failure_mode": [
{
"potential_failure_mode": ...,
"local_effect_of_failure": ...,
"end_effect_of_failure_effect": ...,
"end_effect_of_failure_rationale": ...,
"end_effect_of_failure_severity": ...,
"failure_cause": [
{
"cause_of_failure": ...,
"occurrence": ...,
"detection": ...,
"detection_control": ...,
"prevention_control": [
{
"srs_id": ...,
"prevention_control": ...
}
]
}
]
}
]
}
]
用法示例:
get_dfmea_record_by_crs_id("PCC-CT.CRS.3400")
"""
filter = {
"type": "group",
"logic": "AND",
"children": [
{
"type": "condition",
"field": "crs_id",
"operator": "eq",
"value": crs_id
},
]
}
rows = WorksheetRow.list("requirement", filter)
result_list = [] # 这将是一个包含所有 "Requirement" 树的列表
# --- 1. 遍历 A (Requirement) ---
for row in rows:
requirement = WorksheetRow.get("requirement", row.get("rowId"))
# --- A层 (Requirement) 记录 ---
# 这是我们树的 "根"
req_record = {
"crs_id": requirement.get("crs_id"),
"requirement": requirement.get("requirement"),
"function": requirement.get("function"),
"classification": requirement.get("classification"),
}
# 准备 B 层的列表
failure_mode_list = []
failure_mode_rowid_list = requirement.get("failure_mode", [])
# --- 2. 遍历 B (Failure Mode) ---
for failure_mode_rowid in failure_mode_rowid_list:
failure_mode = WorksheetRow.get("failure_mode", failure_mode_rowid)
# --- B层 (Failure Mode) 记录 ---
fm_record = {
"potential_failure_mode": failure_mode.get("potential_failure_mode"),
"local_effect_of_failure": failure_mode.get("local_effect_of_failure"),
"end_effect_of_failure_effect": failure_mode.get("end_effect_of_failure_effect"),
"end_effect_of_failure_rationale": failure_mode.get("end_effect_of_failure_rationale"),
"end_effect_of_failure_severity": failure_mode.get("end_effect_of_failure_severity"),
}
# 准备 C 层的列表
failure_cause_list = []
failure_cause_rowid_list = failure_mode.get("failure_cause", [])
# --- 3. 遍历 C (Failure Cause) ---
for failure_cause_rowid in failure_cause_rowid_list:
failure_cause = WorksheetRow.get("failure_cause", failure_cause_rowid)
# --- C层 (Failure Cause) 记录 ---
fc_record = {
"cause_of_failure": failure_cause.get("cause_of_failure"),
"occurrence": failure_cause.get("occurrence"),
"detection": failure_cause.get("detection"),
"detection_control": failure_cause.get("detection_control"),
}
# 准备 D 层的列表
prevention_control_list = []
prevention_control_rowid_list = failure_cause.get("prevention_control", [])
# --- 4. 遍历 D (Prevention Control) ---
# 这是最内层,即 "叶子"
for prevention_control_rowid in prevention_control_rowid_list:
prevention_control = WorksheetRow.get("prevention_control", prevention_control_rowid)
# --- D层 (Prevention Control) 记录 ---
pc_record = {
"srs_id": prevention_control.get("srs_id"),
"prevention_control": prevention_control.get("prevention_control"),
}
# 将 D 记录添加到 D 列表
prevention_control_list.append(pc_record)
# --- 循环结束, 向上组装 ---
# 将 D 列表 (prevention_control) 附加到 C 记录
if prevention_control_list: # 你也可以保留空列表: fc_record["prevention_control"] = prevention_control_list
fc_record["prevention_control"] = prevention_control_list
# 将 C 记录添加到 C 列表
failure_cause_list.append(fc_record)
# 将 C 列表 (failure_cause) 附加到 B 记录
if failure_cause_list:
fm_record["failure_cause"] = failure_cause_list
# 将 B 记录添加到 B 列表
failure_mode_list.append(fm_record)
# 将 B 列表 (failure_mode) 附加到 A 记录
if failure_mode_list:
req_record["failure_mode"] = failure_mode_list
# 将 A 记录 (完整的树) 添加到最终结果列表
result_list.append(req_record)
return result_list
+2
View File
@@ -15,6 +15,7 @@ class RuleDefinition:
case_title_template: str = None,
deduplication_window: str = "24h",
source: str = "Default",
workbook: str = None
):
self.rule_id = rule_id
@@ -22,6 +23,7 @@ class RuleDefinition:
self.deduplication_fields = deduplication_fields
self.case_title_template = case_title_template
self.source = source
self.workbook = workbook
valid_windows = ['10m', '30m', '1h', '8h', '12h', '24h']
if deduplication_window not in valid_windows:
raise ValueError(f"'{deduplication_window}' 不是一个有效的时间窗口选项。请从 {valid_windows} 中选择。")
+34
View File
@@ -10,6 +10,7 @@ from Lib.configs import EXPIRE_MINUTES
class Xcache(object):
XCACHE_TOKEN = "XCACHE_TOKEN"
XCACHE_MODULES_TASK_LIST = "XCACHE_MODULES_TASK_LIST"
def __init__(self):
pass
@@ -31,3 +32,36 @@ class Xcache(object):
keys = cache.keys(re_key)
for key in keys:
cache.delete(key)
@staticmethod
def get_module_task_by_uuid(task_uuid):
key = f"{Xcache.XCACHE_MODULES_TASK_LIST}_{task_uuid}"
req = cache.get(key)
return req
@staticmethod
def list_module_tasks():
re_key = f"{Xcache.XCACHE_MODULES_TASK_LIST}_*"
keys = cache.keys(re_key)
reqs = []
for key in keys:
reqs.append(cache.get(key))
return reqs
@staticmethod
def create_module_task(req):
"""任务队列"""
key = f"{Xcache.XCACHE_MODULES_TASK_LIST}_{req.get('uuid')}"
cache.set(key, req, None)
return True
@staticmethod
def del_module_task_by_uuid(task_uuid):
key = f"{Xcache.XCACHE_MODULES_TASK_LIST}_{task_uuid}"
cache.delete(key)
@staticmethod
def get_module_task_length():
re_key = f"{Xcache.XCACHE_MODULES_TASK_LIST}_*"
keys = cache.keys(re_key)
return len(keys)
@@ -9,7 +9,8 @@ from langgraph.graph.state import CompiledStateGraph
from pydantic import BaseModel, Field
from Lib.External.llmapi import LLMAPI
from Lib.External.nocolyapi import InputAlert, common_handler
from Lib.External.nocolyapi import InputAlert
from Lib.External.sirpapi import common_handler
from Lib.External.thehiveclient import TheHiveClient
from Lib.api import string_to_string_time, get_current_time_string
from Lib.basemodule import LanggraphModule
@@ -173,7 +174,7 @@ class Module(LanggraphModule):
"description": description,
"reference": "https://your-siem-or-device-url.com/data?source=123456",
"summary_ai": analyze_result.reasoning,
"artifacts": [
"artifact": [
{
"type": "mail_to",
"value": mail_to,
@@ -199,7 +200,8 @@ class Module(LanggraphModule):
rule_id=self.module_name,
rule_name=rule_name,
deduplication_fields=["mail_from"],
source="Email"
source="Email",
workbook="PHISHING_L2_WORKBOOK"
)
case_row_id = common_handler(input_alert, rule)
return state
@@ -222,6 +224,5 @@ class Module(LanggraphModule):
if __name__ == "__main__":
module = Module()
module.debug_alert_name = "ES-Rule-21-Phishing_user_report_mail" # needed when debug module, framework will read redis stream by this name
module.debug_message_id = "0-0"
module.run()
@@ -5,7 +5,8 @@ from typing import Optional, Union, Dict, Any
from pydantic import BaseModel, Field
from Lib.External.difyclient import DifyClient
from Lib.External.nocolyapi import InputAlert, common_handler
from Lib.External.nocolyapi import InputAlert
from Lib.External.sirpapi import common_handler
from Lib.api import string_to_string_time, get_current_time_string
from Lib.basemodule import BaseModule
from Lib.ruledefinition import RuleDefinition
@@ -93,7 +94,7 @@ class Module(BaseModule):
"description": description,
"reference": "https://your-siem-or-device-url.com/data?source=123456",
"summary_ai": analyze_result.reasoning,
"artifacts": [
"artifact": [
{
"type": "mail_to",
"value": mail_to,
@@ -133,6 +134,5 @@ class Module(BaseModule):
if __name__ == "__main__":
module = Module()
module.debug_alert_name = "ES-Rule-22-Phishing_user_report_mail"
module.debug_message_id = "0-0"
module.run()
+128
View File
@@ -0,0 +1,128 @@
import json
from typing import Dict, Any, Annotated, List
from langchain_core.messages import HumanMessage
from langgraph.graph import StateGraph, add_messages
from langgraph.graph.state import CompiledStateGraph
from pydantic import BaseModel
from Lib.External.llmapi import LLMAPI
from Lib.External.nocolyapi import WorksheetRow
from Lib.External.sirpapi import Alert, Artifact
from Lib.External.sirpapi import Notice
from Lib.External.sirpapi import Playbook as SIRPPlaybook
from Lib.baseplaybook import LanggraphPlaybook
class AgentState(BaseModel):
messages: Annotated[List[Any], add_messages]
alert: Dict
suggestion: str
class Playbook(LanggraphPlaybook):
RUN_AS_JOB = True
def __init__(self):
super().__init__() # do not delete this code
self.init()
def init(self):
def preprocess_node(state: AgentState):
"""预处理数据"""
# worksheet = self.param("worksheet")
rowid = self.param("rowid")
alert = WorksheetRow.get(Alert.WORKSHEET_ID, rowid, include_system_fields=False)
artifacts = WorksheetRow.relations(Alert.WORKSHEET_ID, alert.get("rowId"), "artifact", relation_worksheet_id=Artifact.WORKSHEET_ID,
include_system_fields=False)
alert["artifact"] = artifacts
Notice.send(self.param("user"), "Alert_Suggestion_Gen_By_LLM preprocess_node Finish", f"rowid{self.param('rowid')}")
state.alert = alert
return state
# 定义node
def analyze_node(state: AgentState):
"""AI分析告警数据"""
# 加载system prompt
system_prompt_template = self.load_system_prompt_template("L3_SOC_Analyst")
system_message = system_prompt_template.format()
# 构建few-shot示例
few_shot_examples = [
# HumanMessage(
# content=json.dumps({
# "requirement": ".",
# })
# ),
# AIMessage(
# content=json.dumps({
# "function": "the amount of pneumothorax",
# })
# ),
]
# 运行
llm_api = LLMAPI()
llm = llm_api.get_model()
# 构建消息列表
messages = [
system_message,
*few_shot_examples,
HumanMessage(content=json.dumps(state.alert))
]
response = llm.invoke(messages)
response = LLMAPI.extract_think(response) # langchain chatollama bug临时方案
state.suggestion = response.content
Notice.send(self.param("user"), "Alert_Suggestion_Gen_By_LLM analyze_node Finish", f"rowid{self.param('rowid')}")
return state
def output_node(state: AgentState):
"""处理分析结果"""
suggestion = state.suggestion
fields = [
{"id": "suggestion_ai", "value": suggestion},
]
rowid = self.param("rowid")
WorksheetRow.update(Alert.WORKSHEET_ID, rowid, fields)
self.agent_state = state
Notice.send(self.param("user"), "Alert_Suggestion_Gen_By_LLM output_node Finish", f"rowid{self.param('rowid')}")
SIRPPlaybook.update_status_and_remark(self.param("playbook_rowid"), "Success", "Get suggestion by ai agent completed.") # Success/Failed
return state
# 编译graph
workflow = StateGraph(AgentState)
workflow.add_node("preprocess_node", preprocess_node)
workflow.add_node("analyze_node", analyze_node)
workflow.add_node("output_node", output_node)
workflow.set_entry_point("preprocess_node")
workflow.add_edge("preprocess_node", "analyze_node")
workflow.add_edge("analyze_node", "output_node")
workflow.set_finish_point("output_node")
self.agent_state = AgentState(messages=[], alert={}, suggestion="")
self.graph: CompiledStateGraph = workflow.compile(checkpointer=self.get_checkpointer())
return True
def run(self):
self.run_graph()
return
if __name__ == "__main__":
params_debug = {'playbook': 'Alert_Suggestion_Gen_By_LLM', 'rowid': '55639caf-c648-4130-bc9f-8d38becfe20f', 'worksheet': 'alert'}
module = Playbook()
module._params = params_debug
module.run()
+41
View File
@@ -0,0 +1,41 @@
import json
import time
from Lib.External.nocolyapi import WorksheetRow
from Lib.External.sirpapi import Playbook as SIRPPlaybook
from Lib.baseplaybook import BasePlaybook
class Playbook(BasePlaybook):
RUN_AS_JOB = True
def __init__(self):
super().__init__() # do not delete this code
def run(self):
worksheet = self.param("worksheet")
rowid = self.param("rowid")
artifact = WorksheetRow.get(worksheet, rowid, include_system_fields=False)
self.logger.info(f"Querying threat intelligence for : {artifact}")
# 模拟查询威胁情报数据库,在实际应用中,这里应该调用外部API或数据库进行查询
time.sleep(3)
if artifact.get("type") not in ["ip", "domain", "hash", "vm_ip"]:
ti_result = {"error": "Unsupported type. Please use 'ip', 'domain', or 'hash'."}
else:
ti_result = {"malicious": True, "score": 85, "description": "This IP is associated with known malicious activities.", "source": "ThreatIntelDB",
"last_seen": "2024-10-01T12:34:56Z"}
fields = [{"id": "enrichment", "value": json.dumps(ti_result)}]
WorksheetRow.update(worksheet, rowid, fields)
SIRPPlaybook.update_status_and_remark(self.param("playbook_rowid"), "Success", "Threat intelligence enrichment completed.") # Success/Failed
return
if __name__ == "__main__":
params_debug = {'playbook': 'TI_artifact_query_mock', 'rowid': 'a966036e-b29e-4449-be48-23293bacac5d', 'worksheet': 'Artifact'}
module = Playbook()
module._params = params_debug
module.run()
+132
View File
@@ -0,0 +1,132 @@
import json
from typing import Dict, Any, Annotated, List
from langchain_core.messages import HumanMessage
from langgraph.graph import StateGraph, add_messages
from langgraph.graph.state import CompiledStateGraph
from pydantic import BaseModel
from Lib.External.llmapi import LLMAPI
from Lib.External.nocolyapi import WorksheetRow
from Lib.External.sirpapi import Case, Alert, Artifact
from Lib.External.sirpapi import Notice
from Lib.External.sirpapi import Playbook as SIRPPlaybook
from Lib.baseplaybook import LanggraphPlaybook
class AgentState(BaseModel):
messages: Annotated[List[Any], add_messages]
case: Dict
suggestion: str
class Playbook(LanggraphPlaybook):
RUN_AS_JOB = True
def __init__(self):
super().__init__() # do not delete this code
self.init()
def init(self):
def preprocess_node(state: AgentState):
"""预处理数据"""
# worksheet = self.param("worksheet")
rowid = self.param("rowid")
case = WorksheetRow.get(Case.WORKSHEET_ID, rowid, include_system_fields=False)
alerts = WorksheetRow.relations(Case.WORKSHEET_ID, rowid, "alert", relation_worksheet_id=Alert.WORKSHEET_ID, include_system_fields=False)
for alert in alerts:
artifacts = WorksheetRow.relations(Alert.WORKSHEET_ID, alert.get("rowId"), "artifact", relation_worksheet_id=Artifact.WORKSHEET_ID,
include_system_fields=False)
alert["artifact"] = artifacts
case["alert"] = alerts
Notice.send(self.param("user"), "Case_Suggestion_Gen_By_LLM preprocess_node Finish", f"rowid{self.param('rowid')}")
state.case = case
return state
# 定义node
def analyze_node(state: AgentState):
"""AI分析告警数据"""
# 加载system prompt
system_prompt_template = self.load_system_prompt_template("L3_SOC_Analyst")
system_message = system_prompt_template.format()
# 构建few-shot示例
few_shot_examples = [
# HumanMessage(
# content=json.dumps({
# "requirement": ".",
# })
# ),
# AIMessage(
# content=json.dumps({
# "function": "the amount of pneumothorax",
# })
# ),
]
# 运行
llm_api = LLMAPI()
llm = llm_api.get_model()
# 构建消息列表
messages = [
system_message,
*few_shot_examples,
HumanMessage(content=json.dumps(state.case))
]
response = llm.invoke(messages)
response = LLMAPI.extract_think(response) # langchain chatollama bug临时方案
state.suggestion = response.content
Notice.send(self.param("user"), "Case_Suggestion_Gen_By_LLM analyze_node Finish", f"rowid{self.param('rowid')}")
return state
def output_node(state: AgentState):
"""处理分析结果"""
suggestion = state.suggestion
fields = [
{"id": "suggestion_ai", "value": suggestion},
]
rowid = self.param("rowid")
WorksheetRow.update(Case.WORKSHEET_ID, rowid, fields)
self.agent_state = state
Notice.send(self.param("user"), "Case_Suggestion_Gen_By_LLM output_node Finish", f"rowid{self.param('rowid')}")
SIRPPlaybook.update_status_and_remark(self.param("playbook_rowid"), "Success", "Get suggestion by ai agent completed.") # Success/Failed
return state
# 编译graph
workflow = StateGraph(AgentState)
workflow.add_node("preprocess_node", preprocess_node)
workflow.add_node("analyze_node", analyze_node)
workflow.add_node("output_node", output_node)
workflow.set_entry_point("preprocess_node")
workflow.add_edge("preprocess_node", "analyze_node")
workflow.add_edge("analyze_node", "output_node")
workflow.set_finish_point("output_node")
self.agent_state = AgentState(messages=[], case={}, suggestion="")
self.graph: CompiledStateGraph = workflow.compile(checkpointer=self.get_checkpointer())
return True
def run(self):
self.run_graph()
return
if __name__ == "__main__":
params_debug = {'playbook': 'Case_Suggestion_Gen_By_LLM', 'rowid': '55639caf-c648-4130-bc9f-8d38becfe20f', 'worksheet': 'case'}
module = Playbook()
module._params = params_debug
module.run()
View File
+1
View File
@@ -27,3 +27,4 @@ lxml
openpyxl
langchain-ollama
uvicorn
mcp[cli]