mirror of
https://github.com/FunnyWolf/agentic-soc-platform.git
synced 2026-08-22 13:12:56 +02:00
1029
This commit is contained in:
+4
-1
@@ -174,4 +174,7 @@ Test/*
|
||||
!Test/.gitkeep
|
||||
|
||||
Docker/log/*
|
||||
!Docker/log/.gitkeep
|
||||
!Docker/log/.gitkeep
|
||||
|
||||
PLAYBOOK/Debug_*
|
||||
DATA/Debug_*/
|
||||
@@ -1,6 +1,7 @@
|
||||
import importlib
|
||||
|
||||
from Lib.api import data_return
|
||||
from Lib.apsmodule import aps_module
|
||||
from Lib.baseplaybook import BasePlaybook
|
||||
from Lib.configs import Playbook_MSG_ZH, Playbook_MSG_EN
|
||||
from Lib.log import logger
|
||||
@@ -18,17 +19,27 @@ class Playbook(object):
|
||||
try:
|
||||
load_path = f"PLAYBOOK.{playbook}"
|
||||
class_intent = importlib.import_module(load_path)
|
||||
playbook_intent: BasePlaybook = class_intent.Module(params=params)
|
||||
playbook_intent: BasePlaybook = class_intent.Playbook()
|
||||
playbook_intent._params = params
|
||||
except Exception as E:
|
||||
logger.exception(E)
|
||||
context = data_return(305, {}, Playbook_MSG_ZH.get(305), Playbook_MSG_EN.get(305))
|
||||
context = data_return(305, {"status": "Failed", "job_id": None}, Playbook_MSG_ZH.get(305), Playbook_MSG_EN.get(305))
|
||||
return context
|
||||
|
||||
try:
|
||||
check_result = playbook_intent.run()
|
||||
context = data_return(201, check_result, Playbook_MSG_ZH.get(201), Playbook_MSG_EN.get(201))
|
||||
return context
|
||||
except Exception as E:
|
||||
logger.exception(E)
|
||||
context = data_return(301, {}, Playbook_MSG_ZH.get(301), Playbook_MSG_EN.get(301))
|
||||
return context
|
||||
if playbook_intent.RUN_AS_JOB:
|
||||
job_id = aps_module.putin_post_python_module_queue(playbook_intent)
|
||||
if job_id:
|
||||
context = data_return(201, {"status": "Running", "job_id": job_id}, Playbook_MSG_ZH.get(201), Playbook_MSG_ZH.get(201))
|
||||
return context
|
||||
else:
|
||||
context = data_return(306, {"status": "Failed", "job_id": None}, Playbook_MSG_ZH.get(306), Playbook_MSG_ZH.get(306))
|
||||
return context
|
||||
else:
|
||||
try:
|
||||
result = playbook_intent.run()
|
||||
context = data_return(201, result, Playbook_MSG_ZH.get(201), Playbook_MSG_EN.get(201))
|
||||
return context
|
||||
except Exception as E:
|
||||
logger.exception(E)
|
||||
context = data_return(301, {}, Playbook_MSG_ZH.get(301), Playbook_MSG_EN.get(301))
|
||||
return context
|
||||
|
||||
+1
-1
@@ -15,5 +15,5 @@ class PlaybookView(BaseView):
|
||||
context = Playbook.create(playbook, params=request.data)
|
||||
except Exception as E:
|
||||
logger.exception(E)
|
||||
context = data_return(500, {}, CODE_MSG_ZH.get(500), CODE_MSG_EN.get(500))
|
||||
context = data_return(500, {"status": "Failed", "job_id": None}, CODE_MSG_ZH.get(500), CODE_MSG_EN.get(500))
|
||||
return Response(context)
|
||||
|
||||
+6
-8
@@ -31,13 +31,11 @@ DIFY_API_KEY = {
|
||||
THEHIVE_URL = "https://192.168.1.114:443"
|
||||
THEHIVE_API_KEY = "xxx"
|
||||
|
||||
# Nocodb Config
|
||||
NOCODB_URL = "http://192.168.1.114:8080"
|
||||
NOCODB_TOKEN = "xxx"
|
||||
NOCODB_ALERT_TABLE_ID = "xxx"
|
||||
# SIRP Config
|
||||
SIRP_URL = "http://192.168.3.128:8880"
|
||||
SIRP_APPKEY = "8exxx"
|
||||
SIRP_SIGN = "YTxxxxxx=="
|
||||
SIRP_NOTICE_WEBHOOK = "http://192.168.3.128:8880/api/workflow/hooks/XXXX"
|
||||
|
||||
# Nocoly Config
|
||||
NOCOLY_URL = "http://192.168.241.128:8880"
|
||||
AISOAR_APPKEY = "8exxx"
|
||||
AISOAR_SIGN = "YTxxxxxx=="
|
||||
# APITOKEN
|
||||
ASF_TOKEN = "nocoly_token_for_playbook"
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
**角色 (Role):**
|
||||
你是一位经验丰富的高级网络安全分析师 (L3 SOC Analyst), 擅长对SIEM生成的单一告警进行快速、准确的研判 (Triage) 和深度分析。
|
||||
|
||||
**核心任务 (Task):**
|
||||
你的任务是分析一个**单一的 `Alert` (告警)** 及其关联的 `Artifacts` (元数据)。你需要利用这些信息, 评估此告警的真实性、上下文和潜在风险,
|
||||
然后提供清晰的 **[告警研判]** 结论和一组按优先级排序的 **[建议的下一步行动]**。
|
||||
|
||||
**输入数据结构 (Input Schema):**
|
||||
你将收到的数据结构如下 (已简化, 无Case层级):
|
||||
|
||||
1. **`Alert` (告警):** 单个告警对象, 包含:
|
||||
* `alert_name`: 告警名称 (例如: "Anomalous PowerShell Execution")
|
||||
* `severity`: 原始严重性 (例如: "Medium")
|
||||
* `description`: 告警规则的描述
|
||||
* `timestamp`: 告警发生时间
|
||||
2. **`Artifacts` (元数据列表):** 隶属于 `Alert` 的元数据。每个元数据包含:
|
||||
* `type`: 类型 (例如: "ip", "domain", "file_hash", "user", "host", "command_line")
|
||||
* `value`: 具体的值
|
||||
|
||||
**分析逻辑 (Analysis Logic):**
|
||||
|
||||
1. **上下文分析 (Contextualization):** 告警本身(例如 "PowerShell")和 `Artifacts`(例如特定的 `command_line`、`user`、`host`
|
||||
)结合起来看是什么情况?
|
||||
2. **威胁研判 (Triage & Validation):** **这是你的首要任务。** 基于告警描述和所有 `Artifacts`, 对告警进行定性:
|
||||
* **真阳性 (True Positive):** 有足够证据表明发生了恶意或策略违规活动。
|
||||
* **假阳性 (False Positive):** 告警是由已知的良性活动 (如管理员正常运维、自动化脚本) 或配置错误的规则触发的。
|
||||
* **需进一步调查 (Needs Investigation):** 当前信息不足以明确判断真假, 需要收集更多数据。
|
||||
3. **风险评估 (Risk Assessment):** 如果是真阳性或疑似, 它对关联的实体 (如 `host` 或 `user`)
|
||||
构成了什么具体威胁?(例如:勒索软件前兆、C2通信、权限提升?)
|
||||
|
||||
**输出格式 (Output Format):**
|
||||
你的回答必须严格遵循以下Markdown格式。**[告警研判]** 标题后的第一行必须是三个结论标签之一。
|
||||
|
||||
---
|
||||
|
||||
**[告警研判]**
|
||||
*(必须以下列标签之一开头: **[真阳性]**、**[假阳性]** 或 **[需进一步调查]**)*
|
||||
|
||||
**[结论标签]**: *(在此处简要说明你做出此判断的核心理由。例如: "[真阳性]: 告警由主机 [Host] 上的可疑命令行 [Command Line]
|
||||
触发, 该命令包含编码的PowerShell, 试图连接一个IP [IP Address]。这高度疑似无文件攻击的执行阶段。")*
|
||||
|
||||
**[建议的下一步行动]**
|
||||
*(在此处提供一个按优先级排序的、具体的步骤列表。应优先包含对此告警的深入验证(Investigation)和遏制(Containment)动作。)*
|
||||
|
||||
1. **(调查 - 优先级: 高)** 立即在威胁情报平台 (TIP) 或 VirusTotal 查询关联的IP `[IP Address]`、域名 `[Domain]` 或文件哈希
|
||||
`[File Hash]` (如果存在), 以确认其恶意性。
|
||||
2. **(调查 - 优先级: 高)** 登录主机 `[Host]` 的EDR系统, 检查告警时间点 `[Timestamp]` 前后的进程树和网络连接, 确认
|
||||
`[Command Line]` 的父进程和子进程。
|
||||
3. **(遏制 - 优先级: 中)** (如果步骤1或2确认恶意) 立即将主机 `[Host]` 从网络中隔离, 并阻止防火墙/代理上的恶意IP/域名
|
||||
`[IP/Domain]`。
|
||||
4. **(遏制 - 优先级: 中)** (如果步骤1或2确认恶意) 检查用户 `[User]` 的活动, 考虑重置其密码并检查其他异常登录。
|
||||
5. **(调优 - 优先级: 低)** (如果调查确认为假阳性, 例如是已知管理员脚本) 将此 `[Command Line]` 或 `[File Hash]` 添加到SIEM规则
|
||||
`[Alert Name]` 的白名单中, 并关闭此告警。
|
||||
@@ -0,0 +1,55 @@
|
||||
### AI Agent 系统提示词 (System Prompt)
|
||||
|
||||
**角色 (Role):**
|
||||
你是一个经验丰富的高级网络安全分析师 (L3 SOC Analyst)。
|
||||
|
||||
**核心任务 (Task):**
|
||||
你的任务是分析一个已聚合的 `Case` (案件)。这个 `Case` 包含了SIEM生成的多个 `Alerts` (告警) 以及告警关联的 `Artifacts` (
|
||||
元数据,如IP、域名、Hash等)。
|
||||
|
||||
你需要关联所有碎片化信息,评估整个 `Case` 的性质、严重性和潜在影响,然后提供一份简洁的 **[综合分析摘要]** 和一组按优先级排序的
|
||||
**[建议的下一步行动]**。
|
||||
|
||||
**输入数据结构 (Input Schema):**
|
||||
你将收到的数据结构如下:
|
||||
|
||||
1. **`Case` (案件):** 顶层对象,包含案件ID、标题以及可能已有的人工分析师备注 (Notes)。
|
||||
2. **`Alerts` (告警列表):** 隶属于 `Case` 的一个或多个告警。每个告警包含:
|
||||
* `alert_name`: 告警名称 (例如: "Malicious File Execution")
|
||||
* `severity`: 严重性 (例如: "High")
|
||||
* `description`: 告警描述
|
||||
* `artifacts`: 该告警关联的元数据列表
|
||||
3. **`Artifacts` (元数据列表):** 隶属于 `Alert` 的元数据。每个元数据包含:
|
||||
* `type`: 类型 (例如: "ip", "domain", "file_hash", "user", "host")
|
||||
* `value`: 具体的值
|
||||
|
||||
**分析逻辑 (Analysis Logic):**
|
||||
|
||||
1. **关联分析 (Correlation):** 不要孤立地看待每个 `Alert`。你需要将所有 `Alerts` 和 `Artifacts`
|
||||
串联起来,构建一个完整的攻击链 (Kill Chain) 或故事线。例如,一个 "Phishing Email" 告警和一个 "Malware C2 Traffic"
|
||||
告警如果共享同一个 `host` 或 `user`,它们必须被关联分析。
|
||||
2. **识别关键实体 (Identify Key Entities):** 明确哪些 `Artifacts` 是威胁的核心?(例如: 哪个 `host` 被感染?哪个 `user`
|
||||
账户失陷?哪个 `ip` 是C2服务器?)
|
||||
3. **评估意图和阶段 (Assess Intent & Stage):** 判断这起事件处于什么阶段?(例如: 初始访问、执行、持久化、横向移动?)
|
||||
4. **参考备注 (Use Notes):** 如果 `Case` 中已有人工 `Notes`,请将其作为重要上下文参考,但你仍需独立完成分析。
|
||||
|
||||
**输出格式 (Output Format):**
|
||||
你的回答必须严格遵循以下Markdown格式,确保 **[综合分析摘要]** 和 **[建议的下一步行动]** 两个标题都存在且加粗。
|
||||
|
||||
---
|
||||
|
||||
**[综合分析摘要]**
|
||||
*(在此处用简洁、专业的语言总结整个事件。说明发生了什么、主要威胁是什么、涉及哪些关键资产或用户,以及当前的风险状态。例如: "
|
||||
此Case汇总了一次针对用户[User]的多阶段攻击。事件始于[Alert 1],导致主机[Host]上执行了恶意软件[File Hash]
|
||||
,该软件随后尝试连接已知的C2服务器[IP/Domain]。主机[Host]极有可能已失陷。")*
|
||||
|
||||
**[建议的下一步行动]**
|
||||
*(在此处提供一个按优先级排序的、具体的、可执行的步骤列表。应优先包含遏制(Containment)和调查(Investigation)动作。)*
|
||||
|
||||
1. **(遏制 - 优先级: 高)** 立即将主机 `[Host]` 从网络中隔离,防止威胁横向移动。
|
||||
2. **(遏制 - 优先级: 高)** 在防火墙/Web代理上阻止恶意IP `[IP Address]` 和域名 `[Domain]` 的出站连接。
|
||||
3. **(调查 - 优先级: 中)** 检查用户 `[User]` 的身份验证日志,确认其账户是否存在其他异常登录。
|
||||
4. **(调查 - 优先级: 中)** 在EDR/SIEM中搜索文件哈希 `[File Hash]`,确认该文件是否在其他主机上存在。
|
||||
5. **(修复 - 优先级: 低)** 对主机 `[Host]` 进行全盘杀毒和镜像取证,为后续的清理和重装做准备。
|
||||
|
||||
---
|
||||
@@ -0,0 +1,59 @@
|
||||
# 现场工程师 EDR 告警快速响应手册
|
||||
|
||||
**版本:** 1.0
|
||||
**目标:** 快速确认威胁,阻止攻击扩散,并保留证据。
|
||||
|
||||
---
|
||||
|
||||
## 阶段 1:初次分类与分级 (Triage & Priority)
|
||||
|
||||
| 步骤 | 行动 (Action) | L1 检查要点 (Key Check) |
|
||||
|:--------------------:|:------------------------------------|:-------------------------------------------------------------------|
|
||||
| **1.1 确认资产** | 记录告警资产的主机名/IP,并查询 **资产重要性**。 | 资产是**关键服务器/高管电脑**吗?(Yes $\rightarrow$ P1,否则继续) |
|
||||
| **1.2 确认告警类型** | 识别 EDR 提示的告警类型 (如:恶意软件、凭据窃取、命令行可疑)。 | 告警是**已确认的恶意软件/勒索**吗?(Yes $\rightarrow$ P1,否则继续) |
|
||||
| **1.3 确认误报** | 检查该告警是否为**已知白名单程序**或**上次事件的遗留告警**。 | 告警指标(哈希/路径)在**本地白名单**中吗?(Yes $\rightarrow$ P4/关闭) |
|
||||
| **1.4 确定优先级** | **资产重要性 + 威胁类型**确定最终等级。 | **P1(高):** 关键资产 + 确定恶意;**P2(中):** 重要资产/可疑行为;**P3(低):** 一般资产/低风险行为。 |
|
||||
| **$\rightarrow$ 输出** | 确定**事件优先级(P1/P2/P3)**,创建工单。 | |
|
||||
|
||||
---
|
||||
|
||||
## 阶段 2:调查与遏制 (Investigate & Contain)
|
||||
|
||||
### A. 核心调查 (Core Investigation)
|
||||
|
||||
| 步骤 | 行动 (Action) | L2 调查要点/工具 (Tool Focus) |
|
||||
|:------------------:|:---------------------------------------------|:-------------------------------------------------------------|
|
||||
| **2.1 提取指标 (IoC)** | 从告警详情中提取:**文件哈希、进程路径、外部 IP/域名**。 | **EDR 告警详情页**:复制所有可疑指标。 |
|
||||
| **2.2 威胁情报查询** | 将 IoC 输入**威胁情报平台**(如:VirusTotal, Splunk TI)。 | **判断:** 恶意指标是否有**5+ 命中**或**高恶意评分**? (Yes $\rightarrow$ 确认恶意) |
|
||||
| **2.3 进程与时间线** | 在 EDR 中查看**告警时间点前后**的**完整进程树/活动时间线**。 | **关键:** 恶意进程的**父进程**是什么?是否有**横向移动**迹象 (如:RDP/WMI 连接)。 |
|
||||
| **2.4 确认攻击范围** | 检查是否有**其他**资产也收到**相同**告警。 | **EDR 搜索功能**:查询相同哈希/IP/进程名。 |
|
||||
|
||||
### B. 快速遏制 (Rapid Containment) - **P1/P2 事件必须执行**
|
||||
|
||||
| 步骤 | 行动 (Action) | 目的/工具 (Purpose) |
|
||||
|:--------------:|:-------------------------------|:----------------------------------|
|
||||
| **2.5 隔离端点** | **立即**通过 EDR 平台**隔离**受感染的主机。 | **阻止威胁扩散 (横向移动)**。确保仅保留 EDR 管理通道。 |
|
||||
| **2.6 终止恶意进程** | 通过 EDR **终止**所有确认恶意的**进程树**。 | **阻止当前活动** (如:加密、数据窃取)。 |
|
||||
| **2.7 封锁外部连接** | 将恶意外部 IP/域名通过**防火墙/网络设备**进行封锁。 | **切断 C2 通信** (命令与控制)。 |
|
||||
|
||||
---
|
||||
|
||||
## 阶段 3:根除、恢复与总结 (Eradication & Review)
|
||||
|
||||
| 步骤 | 行动 (Action) | L2 工程师职责 (Task) |
|
||||
|:------------:|:---------------------------------------------|:------------------|
|
||||
| **3.1 彻底清除** | **(需 L3 或系统管理员支持)** 删除恶意文件、清除持久化机制(注册表、启动项)。 | **记录**清除的操作路径和结果。 |
|
||||
| **3.2 恢复验证** | 运行完整的 EDR/杀毒扫描,并检查**是否有新告警**。 | 确保系统已**干净**。 |
|
||||
| **3.3 解除隔离** | **确认安全后**,通过 EDR **解除资产隔离**。 | **恢复业务**。 |
|
||||
| **3.4 记录总结** | 填写完整的**事件处理时间线、根因分析 (RCA) 和采取的措施**。 | **关闭工单**,并记录经验教训。 |
|
||||
|
||||
---
|
||||
|
||||
## 💡 快速判断表 (Quick Reference)
|
||||
|
||||
| 进程行为 | 恶意可能性 | 建议行动 |
|
||||
|:--------------------------------------------------------------------:|:-------------------:|:-----------------------|
|
||||
| `cmd.exe`/`powershell.exe` **作为** `word.exe`/`excel.exe` **的子进程**运行。 | **高** (钓鱼邮件附件执行) | **P1 响应**,检查命令行参数。 |
|
||||
| 发现**`mimikatz.exe`**或相似**凭据转储**工具运行。 | **P1 (严重)** (高价值目标) | **P1 响应**,立即隔离并重置用户密码。 |
|
||||
| EDR 报告**大量文件快速重命名/加密**。 | **P1 (勒索)** | **立即隔离**,通知系统管理员准备恢复。 |
|
||||
| 外部 IP 命中**高风险**情报,且与内部主机有大量通信。 | **高** (C2 通信) | **封锁 IP**,检查数据外泄情况。 |
|
||||
@@ -0,0 +1,61 @@
|
||||
---
|
||||
|
||||
## 🎣 钓鱼邮件快速处理手册 (L1/L2 Playbook)
|
||||
|
||||
**目标:** 快速识别、遏制、清除恶意邮件及其影响,并保护受害者账户。
|
||||
|
||||
---
|
||||
|
||||
## 阶段 1:报告与初次分类 (Reporting & Triage)
|
||||
|
||||
| 步骤 | 行动 (Action) | L1 检查要点 (Key Check) |
|
||||
|:--------------------:|:------------------------------------------------|:---------------------------------------------------------------------------------|
|
||||
| **1.1 接收报告** | 确认报告来源:**用户转发/报告**或 **邮件网关告警**。 | **是否已在沙箱环境中打开?** (Yes $\rightarrow$ P1) |
|
||||
| **1.2 确认原始邮件** | 从原始邮件头中获取关键信息,包括 **发件人 IP**、**原始收件人** 和 **主题**。 | 邮件是否绕过了**邮件网关**直接投递? |
|
||||
| **1.3 初步 IoC 提取** | 提取邮件中的所有 **URL 链接** 和 **附件哈希值**。 | 附件是**可执行文件 (.exe/.scr)** 或 **宏文档 (.docm/.xlsm)** 吗? (Yes $\rightarrow$ P1) |
|
||||
| **1.4 确定优先级** | 根据威胁指标和感染情况分级。 | **P1(高):** 附件已打开/链接已点击/凭据已输入;**P2(中):** 恶意邮件被大规模投递但无确认点击;**P3(低):** 单个可疑但低风险的邮件。 |
|
||||
| **$\rightarrow$ 输出** | 确定**事件优先级(P1/P2/P3)**,创建工单。 | |
|
||||
|
||||
---
|
||||
|
||||
## 阶段 2:调查与遏制 (Investigation & Containment)
|
||||
|
||||
### A. 核心调查 (Core Investigation)
|
||||
|
||||
| 步骤 | 行动 (Action) | L2 调查要点/工具 (Tool Focus) |
|
||||
|:--------------:|:----------------------------------------------------------|:------------------------------------------|
|
||||
| **2.1 威胁情报查询** | 将提取的 **URL、发件人 IP、附件哈希**输入威胁情报平台(如 VT)。 | **判断:** URL 或 IP 是否被标记为**恶意 C2 或钓鱼页面**? |
|
||||
| **2.2 确认攻击范围** | 在**邮件网关/O365** 中搜索:**相同主题/发件人/哈希**的邮件投递了多少用户? | 记录所有**被投递和被点击**的用户列表。 |
|
||||
| **2.3 模拟用户行为** | **(P1 专用)** 将附件在**沙箱**中运行,或在**虚拟机**中访问链接(**切勿在工作环境中执行**)。 | **判断:** 链接是否指向**凭据窃取页面**?附件是否释放了**恶意文件**? |
|
||||
|
||||
### B. 快速遏制与清理 (Rapid Containment & Cleanup)
|
||||
|
||||
| 步骤 | 行动 (Action) | 目的/工具 (Purpose) |
|
||||
|:---------------:|:------------------------------------------------------------------|:--------------------------------|
|
||||
| **2.4 邮件回收/删除** | 利用邮件网关或邮件服务器工具(如 Exchange/O365 Purge)**立即从所有收件箱中删除**该邮件。 | **阻止二次感染**。确保 L1 工程师已获取并保存原始副本。 |
|
||||
| **2.5 网络封锁** | 将所有已确认恶意的 **URL 链接** 和 **发件人 IP** 导入**防火墙/代理服务器**进行封锁。 | **阻止用户访问钓鱼页面**或与恶意 C2 通信。 |
|
||||
| **2.6 保护受害者** | **(如果确认用户点击/输入凭据)** 立即联系身份系统:**强制重置用户密码**,并检查**多因素认证(MFA)**是否被禁用。 | **阻止账户被盗用**,防止横向移动。 |
|
||||
|
||||
---
|
||||
|
||||
## 阶段 3:根除、恢复与总结 (Eradication & Review)
|
||||
|
||||
| 步骤 | 行动 (Action) | L2 工程师职责 (Task) |
|
||||
|:------------:|:-------------------------------------------------|:----------------------|
|
||||
| **3.1 端点扫描** | **(P1 专用)** 对点击/打开附件的用户主机,通过 EDR 执行**全面扫描**。 | 确认是否植入了**后门/Loader**。 |
|
||||
| **3.2 用户通知** | 向**所有**被投递邮件的用户发送**安全警告**(如果尚未发送),教育用户删除并说明事件。 | 提升全员安全意识。 |
|
||||
| **3.3 规则加固** | 将 **发件人 IP/域名** 添加到**邮件网关的黑名单**,并考虑更新**邮件过滤规则**。 | **预防**未来类似攻击。 |
|
||||
| **3.4 记录总结** | 填写完整的**事件处理时间线、受影响用户数和采取的措施**。 | **关闭工单**,记录经验教训,完成留档。 |
|
||||
|
||||
---
|
||||
|
||||
## 💡 快速判断表 (Quick Reference)
|
||||
|
||||
| 邮件特征 | 恶意可能性 | 建议行动 (P Priority) |
|
||||
|:------------------------------------------------------------------:|:------------------:|:---------------------------|
|
||||
| **发件人地址**与内部高管地址**高度相似**(如 `ceo@company.co` vs `ceo@company.com`)。 | **高** (BEC 商业邮件诈骗) | P1/P2,**快速回收邮件**,通知高管。 |
|
||||
| 邮件**附件**是 `.zip` 压缩包,内含 `.js`, `.vbs`, 或 `.lnk` 文件。 | **极高** (恶意软件投递) | **P1 响应**,沙箱分析,全网封锁哈希。 |
|
||||
| **链接 URL** 使用了 **URL 缩短服务** (如 `bit.ly`) 或 **IP 地址**。 | **高** (规避检测) | P2/P1,**模拟点击**确认目标页面,立即封锁。 |
|
||||
| 邮件**正文**要求**立即回复**或**立即点击**,并声称有**“紧急业务”**。 | **中高** (社会工程学) | P2,确认目标用户是否已回复,回收邮件。 |
|
||||
|
||||
---
|
||||
@@ -0,0 +1,44 @@
|
||||
import os
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
# Add the project root directory to Python path
|
||||
project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
from mcp.server import FastMCP
|
||||
|
||||
if __name__ == "__main__":
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ASF.settings")
|
||||
import django
|
||||
|
||||
django.setup()
|
||||
from Lib.llmfunc import get_dfmea_record_by_crs_id
|
||||
|
||||
# Define UUID file path
|
||||
uuid_file_path = os.path.join("..", "Docker", "mcp_uuid")
|
||||
# Try to read UUID from file
|
||||
try:
|
||||
with open(uuid_file_path, 'r') as f:
|
||||
uuid_str = f.read().strip()
|
||||
except FileNotFoundError:
|
||||
# If file doesn't exist, generate new UUID and save it
|
||||
uuid_str = str(uuid.uuid1()).replace('-', "")[0:16]
|
||||
os.makedirs(os.path.dirname(uuid_file_path), exist_ok=True)
|
||||
with open(uuid_file_path, 'w') as f:
|
||||
f.write(uuid_str)
|
||||
|
||||
mcp = FastMCP("ASF-MCP")
|
||||
host = "0.0.0.0"
|
||||
port = 7001
|
||||
mcp.settings.sse_path = f"/{uuid_str}/sse"
|
||||
mcp.settings.message_path = f"/{uuid_str}/messages"
|
||||
|
||||
mcp.settings.host = host
|
||||
mcp.settings.port = port
|
||||
|
||||
# add tools
|
||||
mcp.add_tool(get_dfmea_record_by_crs_id)
|
||||
|
||||
print(f"mcp server url: http://your_server_ip:{port}/{uuid_str}/sse")
|
||||
mcp.run(transport="sse")
|
||||
+209
-209
File diff suppressed because it is too large
Load Diff
@@ -1,10 +1,12 @@
|
||||
import datetime
|
||||
import random
|
||||
|
||||
from Lib.External.nocolyapi import Artifact, Alert, Case, OptionSet, Option
|
||||
from Lib.api import get_current_time_string, string_to_timestamp
|
||||
from Docker.mock.alert import get_mock_alerts
|
||||
from Docker.mock.rule import rule_list
|
||||
from Lib.External.nocolyapi import OptionSet
|
||||
from Lib.External.sirpapi import Artifact, Alert, Case
|
||||
from Lib.api import get_current_time_string, string_to_timestamp
|
||||
from Lib.ruledefinition import RuleDefinition
|
||||
|
||||
|
||||
def generate_four_random_timestamps(
|
||||
@@ -106,7 +108,7 @@ if __name__ == "__main__":
|
||||
case_status_new = OptionSet.get_option_key_by_name_and_value("case_status", "New")
|
||||
|
||||
for alert in alert_list:
|
||||
rule_def = ALL_RULES.get(alert["rule_id"])
|
||||
rule_def: RuleDefinition = ALL_RULES.get(alert["rule_id"])
|
||||
if rule_def is None:
|
||||
print(f"未找到规则定义,跳过处理此告警: {alert['rule_id']}")
|
||||
continue
|
||||
@@ -114,7 +116,7 @@ if __name__ == "__main__":
|
||||
default_times = generate_four_random_timestamps()
|
||||
# artifact
|
||||
artifact_rowid_list = []
|
||||
artifacts = alert.get("artifacts", [])
|
||||
artifacts = alert.get("artifact", [])
|
||||
for artifact in artifacts:
|
||||
artifact_fields = [
|
||||
{"id": "type", "value": artifact["type"]},
|
||||
@@ -137,7 +139,7 @@ if __name__ == "__main__":
|
||||
{"id": "raw_log", "value": alert["raw_log"]},
|
||||
{"id": "rule_id", "value": alert["rule_id"]},
|
||||
{"id": "rule_name", "value": alert["rule_name"]},
|
||||
{"id": "artifacts", "value": artifact_rowid_list},
|
||||
{"id": "artifact", "value": artifact_rowid_list},
|
||||
]
|
||||
# alert
|
||||
row_id_alert = Alert.create(alert_fields)
|
||||
@@ -150,6 +152,13 @@ if __name__ == "__main__":
|
||||
|
||||
row = Case.get_by_deduplication_key(deduplication_key)
|
||||
if row is None:
|
||||
if rule_def.source == "EDR":
|
||||
workbook = Case.load_workbook_md("EDR_L2_WORKBOOK")
|
||||
elif rule_def.source == "Email":
|
||||
workbook = Case.load_workbook_md("PHISHING_L2_WORKBOOK")
|
||||
else:
|
||||
workbook = "# There is no workbook for this source."
|
||||
|
||||
case_field = [
|
||||
{"id": "deduplication_key", "value": deduplication_key},
|
||||
{"id": "title", "value": rule_def.generate_case_title(artifacts=artifacts)},
|
||||
@@ -159,13 +168,20 @@ if __name__ == "__main__":
|
||||
{"id": "created_date", "value": default_times["created_date"]},
|
||||
{"id": "tags", "value": alert["tags"], "type": 2},
|
||||
{"id": "description", "value": alert["description"]},
|
||||
{"id": "alert", "value": [row_id_alert]},
|
||||
|
||||
{"id": "description", "value": alert["description"]},
|
||||
|
||||
{"id": "workbook", "value": workbook},
|
||||
|
||||
{"id": "acknowledged_date", "value": default_times["acknowledged_date"]},
|
||||
{"id": "closed_date", "value": default_times["closed_date"]},
|
||||
|
||||
]
|
||||
row_id_create = Case.create(case_field)
|
||||
try:
|
||||
row_id_create = Case.create(case_field)
|
||||
except Exception as e:
|
||||
print(f"创建工单失败: {e}")
|
||||
continue
|
||||
print(f"create case: {row_id_create}")
|
||||
else:
|
||||
row_id_case = row.get("rowId")
|
||||
@@ -175,7 +191,7 @@ if __name__ == "__main__":
|
||||
|
||||
option_new_score = OptionSet.get_option_by_name_and_value("alert_case_severity", alert["severity"]).get("score", 0)
|
||||
|
||||
severity_value_exist = row.get("severity")[0].get("value")
|
||||
severity_value_exist = row.get("severity")
|
||||
option_exist_score = OptionSet.get_option_by_name_and_value("alert_case_severity", severity_value_exist).get("score", 0)
|
||||
|
||||
if option_new_score > option_exist_score:
|
||||
@@ -183,7 +199,7 @@ if __name__ == "__main__":
|
||||
else:
|
||||
severity = severity_value_exist
|
||||
|
||||
tags_exist = Option.to_value_list(row.get("tags", []))
|
||||
tags_exist = row.get("tags", [])
|
||||
for tag in alert["tags"]:
|
||||
if tag not in tags_exist:
|
||||
tags_exist.append(tag)
|
||||
|
||||
@@ -2,20 +2,21 @@ from Lib.External.nocolyapi import WorksheetRow
|
||||
|
||||
if __name__ == "__main__":
|
||||
filter = {
|
||||
"type": "group",
|
||||
"logic": "OR",
|
||||
"children": [
|
||||
# {
|
||||
# "type": "condition",
|
||||
# "field": "deduplication_key",
|
||||
# "operator": "isnotempty",
|
||||
# },
|
||||
]
|
||||
# "type": "group",
|
||||
# "logic": "OR",
|
||||
# "children": [
|
||||
# {
|
||||
# "type": "condition",
|
||||
# "field": "deduplication_key",
|
||||
# "operator": "isnotempty",
|
||||
# },
|
||||
# ]
|
||||
}
|
||||
worksheet_id_list = ["artifact", "alert", "case"]
|
||||
for worksheet_id in worksheet_id_list:
|
||||
row_ids = []
|
||||
rows = WorksheetRow.list(worksheet_id, filter=filter, fields=["rowId"])
|
||||
rows = WorksheetRow.list(worksheet_id, filter=filter)
|
||||
for row in rows:
|
||||
row_ids.append(row["rowId"])
|
||||
WorksheetRow.delete(worksheet_id=worksheet_id, row_ids=row_ids)
|
||||
if row_ids:
|
||||
WorksheetRow.delete(worksheet_id=worksheet_id, row_ids=row_ids)
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,6 +1,7 @@
|
||||
services:
|
||||
redis-stack:
|
||||
image: redis/redis-stack:latest
|
||||
#image: docker.1ms.run/redis/redis-stack:latest
|
||||
container_name: redis-stack
|
||||
restart: always
|
||||
ports:
|
||||
|
||||
@@ -53,3 +53,19 @@ class WebhookKibanaView(BaseView):
|
||||
logger.exception(E)
|
||||
context = data_return(500, {}, CODE_MSG_ZH.get(500), CODE_MSG_EN.get(500))
|
||||
return Response(context)
|
||||
|
||||
|
||||
class WebhookNocolyMailView(BaseView):
|
||||
permission_classes = [AllowAny]
|
||||
authentication_classes = []
|
||||
|
||||
def create(self, request, **kwargs):
|
||||
try:
|
||||
data = request.data
|
||||
print(data)
|
||||
context = data_return(200, {}, CODE_MSG_ZH.get(200), CODE_MSG_EN.get(200))
|
||||
return Response(context)
|
||||
except Exception as E:
|
||||
logger.exception(E)
|
||||
context = data_return(500, {}, CODE_MSG_ZH.get(500), CODE_MSG_EN.get(500))
|
||||
return Response(context)
|
||||
|
||||
Vendored
-24
@@ -1,24 +0,0 @@
|
||||
import requests
|
||||
|
||||
from CONFIG import NOCODB_URL, NOCODB_TOKEN, NOCODB_ALERT_TABLE_ID
|
||||
|
||||
|
||||
class NocodbClient(object):
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def create_alert(record: dict):
|
||||
headers = {"xc-token": NOCODB_TOKEN}
|
||||
url = f"{NOCODB_URL}/api/v2/tables/{NOCODB_ALERT_TABLE_ID}/records"
|
||||
|
||||
try:
|
||||
response = requests.post(url,
|
||||
headers=headers,
|
||||
json=record)
|
||||
response.raise_for_status()
|
||||
|
||||
response_data = response.json()
|
||||
return response_data
|
||||
except Exception as e:
|
||||
raise
|
||||
Vendored
+114
-277
@@ -3,8 +3,9 @@ from typing import TypedDict, Literal, List, Union, Any, Dict, Optional
|
||||
import requests
|
||||
|
||||
from CONFIG import SIRP_URL, SIRP_APPKEY, SIRP_SIGN
|
||||
from Lib.api import get_current_time_string, string_to_timestamp
|
||||
from Lib.ruledefinition import RuleDefinition
|
||||
|
||||
HEADERS = {"HAP-Appkey": SIRP_APPKEY,
|
||||
"HAP-Sign": SIRP_SIGN}
|
||||
|
||||
|
||||
class InputAlert(TypedDict):
|
||||
@@ -19,8 +20,8 @@ class InputAlert(TypedDict):
|
||||
reference: str
|
||||
description: str
|
||||
summary_ai: Optional[Union[str, Dict[str, Any]]]
|
||||
artifacts: List[Dict]
|
||||
raw_log: Dict
|
||||
artifact: List[Dict]
|
||||
|
||||
|
||||
class FieldType(TypedDict):
|
||||
@@ -66,14 +67,13 @@ class Worksheet(object):
|
||||
|
||||
@staticmethod
|
||||
def get_fields(worksheet_id: str) -> Dict[str, FieldType]:
|
||||
headers = {"HAP-Appkey": SIRP_APPKEY,
|
||||
"HAP-Sign": SIRP_SIGN}
|
||||
|
||||
url = f"{SIRP_URL}/api/v3/app/worksheets/{worksheet_id}"
|
||||
|
||||
response = requests.get(
|
||||
url,
|
||||
params={"includeSystemFields": True},
|
||||
headers=headers
|
||||
headers=HEADERS
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
@@ -93,29 +93,21 @@ class WorksheetRow(object):
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def get(worksheet_id: str, row_id: str):
|
||||
headers = {"HAP-Appkey": SIRP_APPKEY,
|
||||
"HAP-Sign": SIRP_SIGN}
|
||||
def get(worksheet_id: str, row_id: str, include_system_fields=True):
|
||||
url = f"{SIRP_URL}/api/v3/app/worksheets/{worksheet_id}/rows/{row_id}"
|
||||
fields = Worksheet.get_fields(worksheet_id)
|
||||
try:
|
||||
response = requests.get(
|
||||
url,
|
||||
params={"includeSystemFields": True},
|
||||
headers=headers
|
||||
params={"includeSystemFields": include_system_fields},
|
||||
headers=HEADERS
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
response_data = response.json()
|
||||
if response_data.get("success"):
|
||||
data = response_data.get("data")
|
||||
data_new = {}
|
||||
for key in data:
|
||||
if key.startswith("_") or key == "rowId":
|
||||
data_new[key] = data[key]
|
||||
else:
|
||||
alias = fields.get(key).get('alias')
|
||||
data_new[alias] = data[key]
|
||||
row = response_data.get("data")
|
||||
data_new = WorksheetRow._format_row(row, fields, include_system_fields)
|
||||
return data_new
|
||||
else:
|
||||
raise Exception(f"error_code: {response_data.get('error_code')} error_msg: {response_data.get('error_msg')}")
|
||||
@@ -123,12 +115,53 @@ class WorksheetRow(object):
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def list(worksheet_id: str, filter: dict, fields: list = None):
|
||||
headers = {"HAP-Appkey": SIRP_APPKEY,
|
||||
"HAP-Sign": SIRP_SIGN}
|
||||
def _format_row(row, fields, include_system_fields=True):
|
||||
data_new = {}
|
||||
for key in row:
|
||||
if key.startswith("_"):
|
||||
if include_system_fields:
|
||||
data_new[key] = row[key]
|
||||
else:
|
||||
continue
|
||||
elif key == "rowId":
|
||||
data_new[key] = row[key]
|
||||
else:
|
||||
alias = fields.get(key).get('alias')
|
||||
data_new[alias] = WorksheetRow._format_value(fields.get(key), row[key])
|
||||
return data_new
|
||||
|
||||
@staticmethod
|
||||
def _format_value(field, value):
|
||||
field_type = field.get("type")
|
||||
sub_type = field.get("subType")
|
||||
if field_type in ["MultipleSelect"]:
|
||||
value_list = []
|
||||
for option in value:
|
||||
value_list.append(option.get("value"))
|
||||
return value_list
|
||||
elif field_type in ['SingleSelect', "Dropdown"]:
|
||||
if len(value) > 0:
|
||||
return value[0].get("value")
|
||||
else:
|
||||
return None
|
||||
elif field_type in ['Relation']:
|
||||
if sub_type == 1:
|
||||
value_list = []
|
||||
for option in value:
|
||||
value_list.append(option.get("sid"))
|
||||
return value_list
|
||||
else:
|
||||
return value
|
||||
elif field_type in ['Checkbox']:
|
||||
return bool(int(value))
|
||||
else:
|
||||
return value
|
||||
|
||||
@staticmethod
|
||||
def list(worksheet_id: str, filter: dict, include_system_fields=True):
|
||||
url = f"{SIRP_URL}/api/v3/app/worksheets/{worksheet_id}/rows/list"
|
||||
data = {
|
||||
"fields": fields,
|
||||
# "fields": fields,
|
||||
"filter": filter,
|
||||
"sorts": [
|
||||
{
|
||||
@@ -137,19 +170,24 @@ class WorksheetRow(object):
|
||||
}
|
||||
],
|
||||
"includeTotalCount": True,
|
||||
"includeSystemFields": True,
|
||||
"useFieldIdAsKey": False,
|
||||
"includeSystemFields": include_system_fields,
|
||||
# "useFieldIdAsKey": False,
|
||||
"pageSize": 1000,
|
||||
}
|
||||
try:
|
||||
response = requests.post(url,
|
||||
headers=headers,
|
||||
headers=HEADERS,
|
||||
json=data)
|
||||
response.raise_for_status()
|
||||
|
||||
response_data = response.json()
|
||||
if response_data.get("success"):
|
||||
return response_data.get("data").get("rows")
|
||||
fields = Worksheet.get_fields(worksheet_id)
|
||||
rows = response_data.get("data").get("rows")
|
||||
rows_new = []
|
||||
for row in rows:
|
||||
data_new = WorksheetRow._format_row(row, fields, include_system_fields)
|
||||
rows_new.append(data_new)
|
||||
return rows_new
|
||||
else:
|
||||
raise Exception(f"error_code: {response_data.get('error_code')} error_msg: {response_data.get('error_msg')}")
|
||||
except Exception as e:
|
||||
@@ -157,8 +195,7 @@ class WorksheetRow(object):
|
||||
|
||||
@staticmethod
|
||||
def create(worksheet_id: str, fields: list):
|
||||
headers = {"HAP-Appkey": SIRP_APPKEY,
|
||||
"HAP-Sign": SIRP_SIGN}
|
||||
|
||||
url = f"{SIRP_URL}/api/v3/app/worksheets/{worksheet_id}/rows"
|
||||
|
||||
data = {
|
||||
@@ -168,7 +205,7 @@ class WorksheetRow(object):
|
||||
|
||||
try:
|
||||
response = requests.post(url,
|
||||
headers=headers,
|
||||
headers=HEADERS,
|
||||
json=data)
|
||||
response.raise_for_status()
|
||||
|
||||
@@ -178,12 +215,10 @@ class WorksheetRow(object):
|
||||
else:
|
||||
raise Exception(f"error_code: {response_data.get('error_code')} error_msg: {response_data.get('error_msg')} data: {response_data.get('data')}")
|
||||
except Exception as e:
|
||||
raise
|
||||
raise e
|
||||
|
||||
@staticmethod
|
||||
def update(worksheet_id: str, row_id: str, fields: list):
|
||||
headers = {"HAP-Appkey": SIRP_APPKEY,
|
||||
"HAP-Sign": SIRP_SIGN}
|
||||
url = f"{SIRP_URL}/api/v3/app/worksheets/{worksheet_id}/rows/{row_id}"
|
||||
|
||||
data = {
|
||||
@@ -193,7 +228,7 @@ class WorksheetRow(object):
|
||||
|
||||
try:
|
||||
response = requests.patch(url,
|
||||
headers=headers,
|
||||
headers=HEADERS,
|
||||
json=data)
|
||||
response.raise_for_status()
|
||||
|
||||
@@ -207,8 +242,7 @@ class WorksheetRow(object):
|
||||
|
||||
@staticmethod
|
||||
def delete(worksheet_id: str, row_ids: list):
|
||||
headers = {"HAP-Appkey": SIRP_APPKEY,
|
||||
"HAP-Sign": SIRP_SIGN}
|
||||
|
||||
url = f"{SIRP_URL}/api/v3/app/worksheets/{worksheet_id}/rows/batch"
|
||||
|
||||
data = {
|
||||
@@ -218,7 +252,7 @@ class WorksheetRow(object):
|
||||
|
||||
try:
|
||||
response = requests.delete(url,
|
||||
headers=headers,
|
||||
headers=HEADERS,
|
||||
json=data)
|
||||
response.raise_for_status()
|
||||
|
||||
@@ -230,6 +264,46 @@ class WorksheetRow(object):
|
||||
except Exception as e:
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def relations(worksheet_id: str, row_id: str, field: str, relation_worksheet_id: str, include_system_fields: bool = True, page_size: int = 1000,
|
||||
page_index: int = None):
|
||||
fields = Worksheet.get_fields(relation_worksheet_id)
|
||||
|
||||
url = f"{SIRP_URL}/api/v3/app/worksheets/{worksheet_id}/rows/{row_id}/relations/{field}"
|
||||
|
||||
params = {}
|
||||
if page_size is not None:
|
||||
params["pageSize"] = page_size
|
||||
if page_index is not None:
|
||||
params["pageIndex"] = page_index
|
||||
if include_system_fields is not None:
|
||||
params["isReturnSystemFields"] = include_system_fields
|
||||
try:
|
||||
response = requests.get(url,
|
||||
headers=HEADERS,
|
||||
params=params)
|
||||
response.raise_for_status()
|
||||
|
||||
response_data = response.json()
|
||||
if response_data.get("success"):
|
||||
rows = response_data.get("data").get("rows")
|
||||
rows_new = []
|
||||
for row in rows:
|
||||
data_new = WorksheetRow._format_row(row, fields, include_system_fields)
|
||||
rows_new.append(data_new)
|
||||
return rows_new
|
||||
else:
|
||||
raise Exception(f"error_code: {response_data.get('error_code')} error_msg: {response_data.get('error_msg')}")
|
||||
except Exception as e:
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def get_rowid_list_from_rowid(rowid):
|
||||
# 多行数据获取列表
|
||||
tmp = rowid.split("_")
|
||||
rowid_list = tmp[0].split(",")
|
||||
return rowid_list
|
||||
|
||||
|
||||
class OptionSet(object):
|
||||
def __init__(self):
|
||||
@@ -237,12 +311,11 @@ class OptionSet(object):
|
||||
|
||||
@staticmethod
|
||||
def list():
|
||||
headers = {"HAP-Appkey": SIRP_APPKEY,
|
||||
"HAP-Sign": SIRP_SIGN}
|
||||
|
||||
url = f"{SIRP_URL}/api/v3/app/optionsets"
|
||||
|
||||
response = requests.get(url,
|
||||
headers=headers)
|
||||
headers=HEADERS)
|
||||
response.raise_for_status()
|
||||
|
||||
response_data = response.json()
|
||||
@@ -280,239 +353,3 @@ class OptionSet(object):
|
||||
if option["value"] == value:
|
||||
return option["key"]
|
||||
raise Exception(f"optionset {name} {value} not found")
|
||||
|
||||
|
||||
class Artifact(object):
|
||||
WORKSHEET_ID = "artifact"
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def list(filter: dict):
|
||||
result = WorksheetRow.list(Artifact.WORKSHEET_ID, filter)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def update(rowid, fields: list):
|
||||
row_id = WorksheetRow.update(Artifact.WORKSHEET_ID, rowid, fields)
|
||||
return row_id
|
||||
|
||||
@staticmethod
|
||||
def create(fields: list):
|
||||
row_id = WorksheetRow.create(Artifact.WORKSHEET_ID, fields)
|
||||
return row_id
|
||||
|
||||
@staticmethod
|
||||
def get_by_deduplication_key(deduplication_key: str):
|
||||
filter = {
|
||||
"type": "group",
|
||||
"logic": "AND",
|
||||
"children": [
|
||||
{
|
||||
"type": "condition",
|
||||
"field": "deduplication_key",
|
||||
"operator": "eq",
|
||||
"value": deduplication_key
|
||||
},
|
||||
]
|
||||
}
|
||||
rows = WorksheetRow.list(Artifact.WORKSHEET_ID, filter)
|
||||
if rows:
|
||||
if len(rows) > 1:
|
||||
raise Exception(f"found multiple rows with deduplication_key {deduplication_key}")
|
||||
return rows[0]
|
||||
else:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def update_by_type_and_value(data: dict):
|
||||
# 第一层必须是group
|
||||
filter = {
|
||||
"type": "group",
|
||||
"logic": "AND",
|
||||
"children": [
|
||||
{
|
||||
"type": "condition",
|
||||
"field": "type",
|
||||
"operator": "eq",
|
||||
"value": data["type"]
|
||||
},
|
||||
{
|
||||
"type": "condition",
|
||||
"field": "value",
|
||||
"operator": "eq",
|
||||
"value": data["value"]
|
||||
}
|
||||
]
|
||||
}
|
||||
rows = Artifact.list(filter)
|
||||
if rows:
|
||||
for row in rows:
|
||||
rowid = row['rowId']
|
||||
fields = [
|
||||
{"id": "enrichment", "value": data["enrichment"]},
|
||||
]
|
||||
rowid_updated = Artifact.update(rowid, fields)
|
||||
return rowid_updated
|
||||
else:
|
||||
fields = [
|
||||
{"id": "type", "value": data["type"], "type": 2},
|
||||
{"id": "value", "value": data["value"]},
|
||||
{"id": "enrichment", "value": data["enrichment"]},
|
||||
]
|
||||
rowid_created = Artifact.create(fields)
|
||||
return rowid_created
|
||||
|
||||
|
||||
class Alert(object):
|
||||
WORKSHEET_ID = "alert"
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def create(fields: list):
|
||||
row_id = WorksheetRow.create(Alert.WORKSHEET_ID, fields)
|
||||
return row_id
|
||||
|
||||
|
||||
class Case(object):
|
||||
WORKSHEET_ID = "case"
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def create(fields: list):
|
||||
row_id = WorksheetRow.create(Case.WORKSHEET_ID, fields)
|
||||
return row_id
|
||||
|
||||
@staticmethod
|
||||
def update(row_id, fields: list):
|
||||
row_id = WorksheetRow.update(Case.WORKSHEET_ID, row_id, fields)
|
||||
return row_id
|
||||
|
||||
@staticmethod
|
||||
def get_by_deduplication_key(deduplication_key: str):
|
||||
filter = {
|
||||
"type": "group",
|
||||
"logic": "AND",
|
||||
"children": [
|
||||
{
|
||||
"type": "condition",
|
||||
"field": "deduplication_key",
|
||||
"operator": "eq",
|
||||
"value": deduplication_key
|
||||
},
|
||||
]
|
||||
}
|
||||
rows = WorksheetRow.list(Case.WORKSHEET_ID, filter)
|
||||
if rows:
|
||||
if len(rows) > 1:
|
||||
raise Exception(f"found multiple rows with deduplication_key {deduplication_key}")
|
||||
return rows[0]
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
class Option(object):
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def to_value_list(options: list):
|
||||
value_list = []
|
||||
for option in options:
|
||||
value_list.append(option.get("value"))
|
||||
return value_list
|
||||
|
||||
|
||||
def common_handler(alert: InputAlert, rule_def: RuleDefinition) -> str:
|
||||
# artifact
|
||||
artifact_rowid_list = []
|
||||
artifacts = alert.get("artifacts", [])
|
||||
for artifact in artifacts:
|
||||
deduplication_key = artifact["deduplication_key"]
|
||||
artifact_fields = [
|
||||
{"id": "type", "value": artifact["type"]},
|
||||
{"id": "value", "value": artifact["value"]},
|
||||
{"id": "enrichment", "value": artifact["enrichment"]},
|
||||
{"id": "deduplication_key", "value": deduplication_key},
|
||||
]
|
||||
|
||||
row = Artifact.get_by_deduplication_key(deduplication_key)
|
||||
if row is None:
|
||||
row_id = Artifact.create(artifact_fields)
|
||||
else:
|
||||
row_id = row.get("rowId")
|
||||
Artifact.update(row_id, artifact_fields)
|
||||
|
||||
artifact_rowid_list.append(row_id)
|
||||
|
||||
alert_fields = [
|
||||
{"id": "tags", "value": alert.get("tags"), "type": 2},
|
||||
{"id": "severity", "value": alert.get("severity")},
|
||||
{"id": "source", "value": alert.get("source")},
|
||||
{"id": "alert_date", "value": alert.get("alert_date")},
|
||||
{"id": "created_date", "value": alert.get("created_date")},
|
||||
{"id": "reference", "value": alert.get("reference")},
|
||||
{"id": "description", "value": alert.get("description")},
|
||||
{"id": "raw_log", "value": alert.get("raw_log")},
|
||||
{"id": "rule_id", "value": alert.get("rule_id")},
|
||||
{"id": "rule_name", "value": alert.get("rule_name")},
|
||||
{"id": "name", "value": alert.get("name")},
|
||||
{"id": "summary_ai", "value": alert.get("summary_ai")},
|
||||
{"id": "artifacts", "value": artifact_rowid_list},
|
||||
]
|
||||
|
||||
# alert
|
||||
row_id_alert = Alert.create(alert_fields)
|
||||
|
||||
# case
|
||||
timestamp = string_to_timestamp(alert["alert_date"], "%Y-%m-%dT%H:%M:%SZ")
|
||||
deduplication_key = rule_def.generate_deduplication_key(artifacts=artifacts, timestamp=timestamp)
|
||||
|
||||
row = Case.get_by_deduplication_key(deduplication_key)
|
||||
if row is None:
|
||||
case_field = [
|
||||
{"id": "title", "value": rule_def.generate_case_title(artifacts=artifacts)},
|
||||
{"id": "deduplication_key", "value": deduplication_key},
|
||||
{"id": "alert", "value": [row_id_alert]},
|
||||
{"id": "case_status", "value": "New"},
|
||||
{"id": "created_at", "value": get_current_time_string()},
|
||||
{"id": "tags", "value": alert["tags"], "type": 2},
|
||||
{"id": "severity", "value": alert["severity"]},
|
||||
{"id": "type", "value": rule_def.source},
|
||||
{"id": "description", "value": alert["description"]},
|
||||
]
|
||||
row_id_create = Case.create(case_field)
|
||||
return row_id_create
|
||||
else:
|
||||
row_id_case = row.get("rowId")
|
||||
existing_alerts = row.get("alert", [])
|
||||
if row_id_alert not in existing_alerts:
|
||||
existing_alerts.append(row_id_alert)
|
||||
|
||||
option_new_score = OptionSet.get_option_by_name_and_value("alert_case_severity", alert["severity"]).get("score", 0)
|
||||
|
||||
severity_value_exist = row.get("severity")[0].get("value")
|
||||
option_exist_score = OptionSet.get_option_by_name_and_value("alert_case_severity", severity_value_exist).get("score", 0)
|
||||
|
||||
if option_new_score > option_exist_score:
|
||||
severity = alert["severity"]
|
||||
else:
|
||||
severity = severity_value_exist
|
||||
|
||||
tags_exist = Option.to_value_list(row.get("tags", []))
|
||||
for tag in alert["tags"]:
|
||||
if tag not in tags_exist:
|
||||
tags_exist.append(tag)
|
||||
|
||||
case_field = [
|
||||
{"id": "alert", "value": existing_alerts},
|
||||
{"id": "severity", "value": severity},
|
||||
{"id": "tags", "value": tags_exist, "type": 2}
|
||||
]
|
||||
row_id_updated = Case.update(row_id_case, case_field)
|
||||
return row_id_updated
|
||||
|
||||
Vendored
+284
@@ -0,0 +1,284 @@
|
||||
import os
|
||||
|
||||
import requests
|
||||
|
||||
from CONFIG import SIRP_NOTICE_WEBHOOK
|
||||
from Lib.External.nocolyapi import WorksheetRow, InputAlert, OptionSet
|
||||
from Lib.api import string_to_timestamp, get_current_time_string
|
||||
from Lib.ruledefinition import RuleDefinition
|
||||
|
||||
|
||||
class Artifact(object):
|
||||
WORKSHEET_ID = "artifact"
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def list(filter: dict):
|
||||
result = WorksheetRow.list(Artifact.WORKSHEET_ID, filter)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def update(rowid, fields: list):
|
||||
row_id = WorksheetRow.update(Artifact.WORKSHEET_ID, rowid, fields)
|
||||
return row_id
|
||||
|
||||
@staticmethod
|
||||
def create(fields: list):
|
||||
row_id = WorksheetRow.create(Artifact.WORKSHEET_ID, fields)
|
||||
return row_id
|
||||
|
||||
@staticmethod
|
||||
def get_by_deduplication_key(deduplication_key: str):
|
||||
filter = {
|
||||
"type": "group",
|
||||
"logic": "AND",
|
||||
"children": [
|
||||
{
|
||||
"type": "condition",
|
||||
"field": "deduplication_key",
|
||||
"operator": "eq",
|
||||
"value": deduplication_key
|
||||
},
|
||||
]
|
||||
}
|
||||
rows = WorksheetRow.list(Artifact.WORKSHEET_ID, filter)
|
||||
if rows:
|
||||
if len(rows) > 1:
|
||||
raise Exception(f"found multiple rows with deduplication_key {deduplication_key}")
|
||||
return rows[0]
|
||||
else:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def update_by_type_and_value(data: dict):
|
||||
# 第一层必须是group
|
||||
filter = {
|
||||
"type": "group",
|
||||
"logic": "AND",
|
||||
"children": [
|
||||
{
|
||||
"type": "condition",
|
||||
"field": "type",
|
||||
"operator": "eq",
|
||||
"value": data["type"]
|
||||
},
|
||||
{
|
||||
"type": "condition",
|
||||
"field": "value",
|
||||
"operator": "eq",
|
||||
"value": data["value"]
|
||||
}
|
||||
]
|
||||
}
|
||||
rows = Artifact.list(filter)
|
||||
if rows:
|
||||
for row in rows:
|
||||
rowid = row['rowId']
|
||||
fields = [
|
||||
{"id": "enrichment", "value": data["enrichment"]},
|
||||
]
|
||||
rowid_updated = Artifact.update(rowid, fields)
|
||||
return rowid_updated
|
||||
else:
|
||||
fields = [
|
||||
{"id": "type", "value": data["type"], "type": 2},
|
||||
{"id": "value", "value": data["value"]},
|
||||
{"id": "enrichment", "value": data["enrichment"]},
|
||||
]
|
||||
rowid_created = Artifact.create(fields)
|
||||
return rowid_created
|
||||
|
||||
|
||||
class Alert(object):
|
||||
WORKSHEET_ID = "alert"
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def create(fields: list):
|
||||
row_id = WorksheetRow.create(Alert.WORKSHEET_ID, fields)
|
||||
return row_id
|
||||
|
||||
|
||||
class Case(object):
|
||||
WORKSHEET_ID = "case"
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def create(fields: list):
|
||||
row_id = WorksheetRow.create(Case.WORKSHEET_ID, fields)
|
||||
return row_id
|
||||
|
||||
@staticmethod
|
||||
def update(row_id, fields: list):
|
||||
row_id = WorksheetRow.update(Case.WORKSHEET_ID, row_id, fields)
|
||||
return row_id
|
||||
|
||||
@staticmethod
|
||||
def get_by_deduplication_key(deduplication_key: str):
|
||||
filter = {
|
||||
"type": "group",
|
||||
"logic": "AND",
|
||||
"children": [
|
||||
{
|
||||
"type": "condition",
|
||||
"field": "deduplication_key",
|
||||
"operator": "eq",
|
||||
"value": deduplication_key
|
||||
},
|
||||
]
|
||||
}
|
||||
rows = WorksheetRow.list(Case.WORKSHEET_ID, filter)
|
||||
if rows:
|
||||
if len(rows) > 1:
|
||||
raise Exception(f"found multiple rows with deduplication_key {deduplication_key}")
|
||||
return rows[0]
|
||||
else:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def load_workbook_md(workbook_name: str) -> str:
|
||||
"""
|
||||
根据 workbook 名称读取 DATA/WORKBOOK/{workbook_name}.md 的内容并返回字符串。
|
||||
路径相对于项目根 (两级向上到 asf 文件夹)。
|
||||
"""
|
||||
base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
|
||||
md_path = os.path.join(base_dir, 'DATA', 'WORKBOOK', f"{workbook_name}.md")
|
||||
if not os.path.exists(md_path):
|
||||
raise FileNotFoundError(f"workbook md not found: {md_path}")
|
||||
with open(md_path, 'r', encoding='utf-8') as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
class Playbook(object):
|
||||
WORKSHEET_ID = "playbook"
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def create(fields: list):
|
||||
row_id = WorksheetRow.create(Playbook.WORKSHEET_ID, fields)
|
||||
return row_id
|
||||
|
||||
@staticmethod
|
||||
def update(row_id, fields: list):
|
||||
row_id = WorksheetRow.update(Playbook.WORKSHEET_ID, row_id, fields)
|
||||
return row_id
|
||||
|
||||
@staticmethod
|
||||
def update_status_and_remark(row_id, status, remark):
|
||||
fields = [
|
||||
{"id": "job_status", "value": status},
|
||||
{"id": "remark", "value": remark},
|
||||
]
|
||||
row_id = WorksheetRow.update(Playbook.WORKSHEET_ID, row_id, fields)
|
||||
return row_id
|
||||
|
||||
|
||||
class Notice(object):
|
||||
|
||||
@staticmethod
|
||||
def send(user, title, body=None):
|
||||
result = requests.post(SIRP_NOTICE_WEBHOOK, json={"title": title, "body": body, "user": user})
|
||||
return result
|
||||
|
||||
|
||||
def common_handler(alert: InputAlert, rule_def: RuleDefinition) -> str:
|
||||
artifact_rowid_list = []
|
||||
artifacts = alert.get("artifact", [])
|
||||
for artifact in artifacts:
|
||||
deduplication_key = artifact["deduplication_key"]
|
||||
artifact_fields = [
|
||||
{"id": "type", "value": artifact["type"]},
|
||||
{"id": "value", "value": artifact["value"]},
|
||||
{"id": "enrichment", "value": artifact["enrichment"]},
|
||||
{"id": "deduplication_key", "value": deduplication_key},
|
||||
]
|
||||
|
||||
row = Artifact.get_by_deduplication_key(deduplication_key)
|
||||
if row is None:
|
||||
row_id = Artifact.create(artifact_fields)
|
||||
else:
|
||||
row_id = row.get("rowId")
|
||||
Artifact.update(row_id, artifact_fields)
|
||||
|
||||
artifact_rowid_list.append(row_id)
|
||||
|
||||
alert_fields = [
|
||||
{"id": "tags", "value": alert.get("tags"), "type": 2},
|
||||
{"id": "severity", "value": alert.get("severity")},
|
||||
{"id": "source", "value": alert.get("source")},
|
||||
{"id": "alert_date", "value": alert.get("alert_date")},
|
||||
{"id": "created_date", "value": alert.get("created_date")},
|
||||
{"id": "reference", "value": alert.get("reference")},
|
||||
{"id": "description", "value": alert.get("description")},
|
||||
{"id": "raw_log", "value": alert.get("raw_log")},
|
||||
{"id": "rule_id", "value": alert.get("rule_id")},
|
||||
{"id": "rule_name", "value": alert.get("rule_name")},
|
||||
{"id": "name", "value": alert.get("name")},
|
||||
{"id": "summary_ai", "value": alert.get("summary_ai")},
|
||||
{"id": "artifact", "value": artifact_rowid_list},
|
||||
]
|
||||
|
||||
# alert
|
||||
row_id_alert = Alert.create(alert_fields)
|
||||
|
||||
# case
|
||||
timestamp = string_to_timestamp(alert["alert_date"], "%Y-%m-%dT%H:%M:%SZ")
|
||||
deduplication_key = rule_def.generate_deduplication_key(artifacts=artifacts, timestamp=timestamp)
|
||||
|
||||
row = Case.get_by_deduplication_key(deduplication_key)
|
||||
if row is None:
|
||||
if rule_def.workbook is not None:
|
||||
workbook = Case.load_workbook_md(rule_def.workbook)
|
||||
else:
|
||||
workbook = "# There is no workbook for this source."
|
||||
|
||||
case_field = [
|
||||
{"id": "title", "value": rule_def.generate_case_title(artifacts=artifacts)},
|
||||
{"id": "deduplication_key", "value": deduplication_key},
|
||||
{"id": "alert", "value": [row_id_alert]},
|
||||
{"id": "case_status", "value": "New"},
|
||||
{"id": "created_at", "value": get_current_time_string()},
|
||||
{"id": "tags", "value": alert["tags"], "type": 2},
|
||||
{"id": "severity", "value": alert["severity"]},
|
||||
{"id": "type", "value": rule_def.source},
|
||||
{"id": "description", "value": alert["description"]},
|
||||
{"id": "workbook", "value": workbook},
|
||||
]
|
||||
row_id_create = Case.create(case_field)
|
||||
return row_id_create
|
||||
else:
|
||||
row_id_case = row.get("rowId")
|
||||
existing_alerts = row.get("alert", [])
|
||||
if row_id_alert not in existing_alerts:
|
||||
existing_alerts.append(row_id_alert)
|
||||
|
||||
option_new_score = OptionSet.get_option_by_name_and_value("alert_case_severity", alert["severity"]).get("score", 0)
|
||||
|
||||
severity_value_exist = row.get("severity")
|
||||
option_exist_score = OptionSet.get_option_by_name_and_value("alert_case_severity", severity_value_exist).get("score", 0)
|
||||
|
||||
if option_new_score > option_exist_score:
|
||||
severity = alert["severity"]
|
||||
else:
|
||||
severity = severity_value_exist
|
||||
|
||||
tags_exist = row.get("tags", [])
|
||||
for tag in alert["tags"]:
|
||||
if tag not in tags_exist:
|
||||
tags_exist.append(tag)
|
||||
|
||||
case_field = [
|
||||
{"id": "alert", "value": existing_alerts},
|
||||
{"id": "severity", "value": severity},
|
||||
{"id": "tags", "value": tags_exist, "type": 2}
|
||||
]
|
||||
row_id_updated = Case.update(row_id_case, case_field)
|
||||
return row_id_updated
|
||||
+90
-13
@@ -1,3 +1,4 @@
|
||||
import base64
|
||||
import datetime
|
||||
import ipaddress
|
||||
import json
|
||||
@@ -11,6 +12,7 @@ import subprocess
|
||||
import time
|
||||
import uuid
|
||||
from collections import OrderedDict
|
||||
from io import BytesIO
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import dns.resolver
|
||||
@@ -331,45 +333,78 @@ def get_dns_a(domain):
|
||||
return []
|
||||
|
||||
|
||||
def write_list_of_dict_to_excel_sheet(data_list: list[dict], file_path: str, sheet_name: str):
|
||||
def write_list_of_dict_to_excel_sheet(data_list: list[dict], file_path: str = '', sheet_name: str = None, return_content=False):
|
||||
"""
|
||||
仅使用 openpyxl 库将字典列表中的数据写入指定的 XLSX 文件和 sheet。
|
||||
|
||||
逻辑:
|
||||
- 如果 XLSX 文件存在,则打开文件,否则创建文件。
|
||||
- 如果 XLSX 文件存在,则打开文件,否则创建新的 Workbook。
|
||||
- 指定 sheet 如果存在,则覆盖(删除旧的,创建新的);如果不存在,则创建。
|
||||
- 字典的 key 作为表头。
|
||||
- 当 return_content 为 True 时,不进行任何本地文件系统操作。
|
||||
|
||||
Args:
|
||||
data_list: 列表,列表中的每个元素是一个字典(代表一行数据)。
|
||||
file_path: XLSX 文件的完整路径和名称。
|
||||
file_path: XLSX 文件的完整路径和名称(仅在 return_content=False 时使用)。
|
||||
sheet_name: 要写入的 sheet 的名称。
|
||||
return_content: 返回 Base64 编码的 Excel 内容。
|
||||
"""
|
||||
if not data_list:
|
||||
return
|
||||
|
||||
if os.path.exists(file_path):
|
||||
workbook = load_workbook(file_path)
|
||||
if not return_content and file_path and os.path.exists(file_path):
|
||||
try:
|
||||
workbook = load_workbook(file_path)
|
||||
except Exception as e:
|
||||
|
||||
raise IOError(f"Error loading workbook from {file_path}: {e}")
|
||||
else:
|
||||
|
||||
workbook = Workbook()
|
||||
|
||||
if sheet_name in workbook.sheetnames:
|
||||
del workbook[sheet_name]
|
||||
final_sheet_name = sheet_name if sheet_name else 'Sheet1' # 默认使用 'Sheet1'
|
||||
|
||||
worksheet = workbook.create_sheet(title=sheet_name, index=0)
|
||||
if final_sheet_name in workbook.sheetnames:
|
||||
del workbook[final_sheet_name]
|
||||
|
||||
worksheet = workbook.create_sheet(title=final_sheet_name, index=0)
|
||||
|
||||
header = list(data_list[0].keys())
|
||||
worksheet.append(header)
|
||||
|
||||
for row_dict in data_list:
|
||||
# 按照 header 的顺序提取字典的值作为一行数据
|
||||
row_values = [row_dict.get(key, '') for key in header]
|
||||
worksheet.append(row_values)
|
||||
|
||||
os.makedirs(os.path.dirname(file_path) or '.', exist_ok=True)
|
||||
if not os.path.exists(file_path) and 'Sheet' in workbook.sheetnames and workbook['Sheet'].max_row == 1 and workbook['Sheet'].cell(1, 1).value is None:
|
||||
del workbook['Sheet']
|
||||
if 'Sheet' in workbook.sheetnames:
|
||||
default_sheet = workbook['Sheet']
|
||||
if default_sheet.max_row == 1 and default_sheet.cell(1, 1).value is None:
|
||||
del workbook['Sheet']
|
||||
|
||||
workbook.save(file_path)
|
||||
if return_content:
|
||||
|
||||
file_stream = BytesIO()
|
||||
workbook.save(file_stream)
|
||||
file_stream.seek(0)
|
||||
|
||||
excel_bytes = file_stream.getvalue()
|
||||
base64_content = base64.b64encode(excel_bytes).decode('utf-8')
|
||||
|
||||
return base64_content
|
||||
else:
|
||||
|
||||
if not file_path:
|
||||
raise ValueError("file_path cannot be empty when return_content is False")
|
||||
|
||||
directory = os.path.dirname(file_path)
|
||||
if directory:
|
||||
os.makedirs(directory, exist_ok=True)
|
||||
|
||||
try:
|
||||
workbook.save(file_path)
|
||||
except Exception as e:
|
||||
raise IOError(f"Error saving workbook to {file_path}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def read_excel_sheet_to_list_of_dict(file_path: str, sheet_name: str) -> list[dict]:
|
||||
@@ -409,3 +444,45 @@ def read_excel_sheet_to_list_of_dict(file_path: str, sheet_name: str) -> list[di
|
||||
data_list.append(row_dict)
|
||||
|
||||
return data_list
|
||||
|
||||
|
||||
def read_file_and_base64(file_path: str) -> dict:
|
||||
"""
|
||||
读取指定路径的文件,并返回包含文件名称和文件内容base64编码的字典。
|
||||
|
||||
:param file_path: 文件的完整路径。
|
||||
:return: 包含文件名称和文件内容base64编码的字典,
|
||||
格式为 {"name": "文件名称,带后缀", "base64": "文件内容的base64编码"}。
|
||||
:raises FileNotFoundError: 如果文件路径不存在。
|
||||
:raises IOError: 如果读取文件时发生其他I/O错误。
|
||||
:raises Exception: 如果发生其他意外错误。
|
||||
"""
|
||||
try:
|
||||
# 检查文件是否存在
|
||||
if not os.path.exists(file_path):
|
||||
raise FileNotFoundError(f"文件未找到: {file_path}")
|
||||
|
||||
# 获取文件名称(带后缀)
|
||||
file_name = os.path.basename(file_path)
|
||||
|
||||
# 读取文件内容并进行base64编码
|
||||
with open(file_path, 'rb') as f:
|
||||
file_content = f.read()
|
||||
# 使用标准的base64编码
|
||||
base64_encoded_content = base64.b64encode(file_content).decode('utf-8')
|
||||
|
||||
# 返回字典
|
||||
return {
|
||||
"name": file_name,
|
||||
"base64": base64_encoded_content
|
||||
}
|
||||
|
||||
except FileNotFoundError as e:
|
||||
# 直接raise异常,不包含调试信息
|
||||
raise e
|
||||
except IOError as e:
|
||||
# 直接raise异常,不包含调试信息
|
||||
raise IOError(f"读取文件时发生I/O错误: {e}")
|
||||
except Exception as e:
|
||||
# 直接raise异常,不包含调试信息
|
||||
raise Exception(f"发生未知错误: {e}")
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# @File : apsmodule.py
|
||||
# @Date : 2021/2/26
|
||||
# @Desc :
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from apscheduler.events import EVENT_JOB_ADDED, EVENT_JOB_REMOVED, EVENT_JOB_MODIFIED, EVENT_JOB_EXECUTED, \
|
||||
EVENT_JOB_ERROR, EVENT_JOB_MISSED, EVENT_JOB_SUBMITTED, EVENT_JOB_MAX_INSTANCES
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
|
||||
from Lib.log import logger
|
||||
from Lib.xcache import Xcache
|
||||
|
||||
|
||||
class APSModule(object):
|
||||
"""处理post python模块请求,单例模式运行
|
||||
EVENT_JOB_ADDED | EVENT_JOB_REMOVED | EVENT_JOB_MODIFIED |EVENT_JOB_EXECUTED |
|
||||
EVENT_JOB_ERROR | EVENT_JOB_MISSED |EVENT_JOB_SUBMITTED | EVENT_JOB_MAX_INSTANCES
|
||||
"""
|
||||
_instance_lock = threading.Lock()
|
||||
|
||||
def __init__(self):
|
||||
|
||||
self.ModuleJobsScheduler = BackgroundScheduler()
|
||||
self.ModuleJobsScheduler.add_listener(self.deal_result)
|
||||
self.ModuleJobsScheduler.start()
|
||||
|
||||
def __new__(cls, *args, **kwargs):
|
||||
if not hasattr(APSModule, "_instance"):
|
||||
with APSModule._instance_lock:
|
||||
if not hasattr(APSModule, "_instance"):
|
||||
APSModule._instance = object.__new__(cls)
|
||||
return APSModule._instance
|
||||
|
||||
def putin_post_python_module_queue(self, post_module_intent=None):
|
||||
try:
|
||||
# 存储uuid
|
||||
module_uuid = str(uuid.uuid1())
|
||||
|
||||
logger.info(f"模块放入列表: uuid: {module_uuid}")
|
||||
self.ModuleJobsScheduler.add_job(func=post_module_intent.run, max_instances=1, id=module_uuid)
|
||||
|
||||
# 放入缓存队列,用于后续删除任务,存储结果等
|
||||
|
||||
req = {
|
||||
'uuid': module_uuid,
|
||||
# 'module': post_module_intent, # 对象无法存储到缓存中
|
||||
'time': int(time.time()),
|
||||
}
|
||||
Xcache.create_module_task(req)
|
||||
|
||||
return module_uuid
|
||||
except Exception as E:
|
||||
logger.exception(E)
|
||||
return None
|
||||
|
||||
def deal_result(self, event=None):
|
||||
flag = False
|
||||
if event.code == EVENT_JOB_ADDED:
|
||||
pass
|
||||
elif event.code == EVENT_JOB_REMOVED:
|
||||
pass
|
||||
elif event.code == EVENT_JOB_MODIFIED:
|
||||
pass
|
||||
elif event.code == EVENT_JOB_EXECUTED: # 执行完成
|
||||
flag = self.store_executed_result(event.job_id)
|
||||
elif event.code == EVENT_JOB_ERROR:
|
||||
pass
|
||||
flag = self.store_error_result(event.job_id, event.exception)
|
||||
elif event.code == EVENT_JOB_MISSED:
|
||||
pass
|
||||
elif event.code == EVENT_JOB_SUBMITTED:
|
||||
pass
|
||||
elif event.code == EVENT_JOB_MAX_INSTANCES:
|
||||
pass
|
||||
else:
|
||||
pass
|
||||
return flag
|
||||
|
||||
@staticmethod
|
||||
def store_executed_result(job_id=None):
|
||||
req = Xcache.get_module_task_by_uuid(task_uuid=job_id)
|
||||
if req is None:
|
||||
logger.warning("缓存中无对应实例,模块已中途退出")
|
||||
return False
|
||||
Xcache.del_module_task_by_uuid(task_uuid=job_id) # 清理缓存信息
|
||||
logger.info(f"模块执行完成: uuid: {job_id}")
|
||||
|
||||
@staticmethod
|
||||
def store_error_result(job_id=None, exception=None):
|
||||
req = Xcache.get_module_task_by_uuid(task_uuid=job_id)
|
||||
Xcache.del_module_task_by_uuid(task_uuid=job_id) # 清理缓存信息
|
||||
logger.exception(exception)
|
||||
|
||||
def delete_job_by_uuid(self, job_id=None):
|
||||
req = Xcache.get_module_task_by_uuid(task_uuid=job_id)
|
||||
Xcache.del_module_task_by_uuid(task_uuid=job_id) # 清理缓存信息
|
||||
logger.info(f"多模块实例手动删除:{job_id}")
|
||||
return True
|
||||
|
||||
|
||||
aps_module = APSModule()
|
||||
+31
-8
@@ -6,7 +6,7 @@ from langgraph.checkpoint.memory import MemorySaver
|
||||
from langgraph.graph.state import CompiledStateGraph
|
||||
|
||||
from CONFIG import DIFY_API_KEY
|
||||
from Lib.configs import MODULE_DATA_DIR, REDIS_CONSUMER_GROUP
|
||||
from Lib.configs import DATA_DIR, REDIS_CONSUMER_GROUP
|
||||
from Lib.llmapi import AgentState
|
||||
from Lib.log import logger
|
||||
from Lib.redis_stream_api import RedisStreamAPI
|
||||
@@ -18,18 +18,40 @@ class BaseModule(object):
|
||||
def __init__(self):
|
||||
self._thread_name = None
|
||||
self.logger = logger
|
||||
|
||||
self.agent_state = None
|
||||
# debug
|
||||
self.debug_alert_name = None
|
||||
self.debug_message_id = None # 设置为非None以启用Debug模式
|
||||
|
||||
@staticmethod
|
||||
def _get_main_script_name():
|
||||
"""
|
||||
获取主执行脚本的文件名(不含扩展名)。
|
||||
无论当前代码在哪个模块中运行,sys.argv[0]始终指向最初启动的脚本。
|
||||
"""
|
||||
try:
|
||||
# 1. 获取主执行脚本的完整路径
|
||||
script_path = sys.argv[0]
|
||||
|
||||
# 2. 从完整路径中提取文件名
|
||||
script_filename = os.path.basename(script_path)
|
||||
|
||||
# 3. 分离文件名和扩展名
|
||||
script_name, _ = os.path.splitext(script_filename)
|
||||
|
||||
return script_name
|
||||
except IndexError as e:
|
||||
raise RuntimeError("无法获取主执行脚本名称,sys.argv[0]不存在。") from e
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"获取主执行脚本名称时发生错误: {e}") from e
|
||||
|
||||
@property
|
||||
def module_name(self):
|
||||
"""获取模块加载路径"""
|
||||
if self.debug_alert_name is None:
|
||||
return self.__module__.split(".")[-1]
|
||||
module_name = self.__module__.split(".")[-1]
|
||||
if module_name == "__main__":
|
||||
return self._get_main_script_name()
|
||||
else:
|
||||
return self.debug_alert_name
|
||||
return module_name
|
||||
|
||||
def read_message(self) -> dict:
|
||||
"""读取消息"""
|
||||
@@ -66,7 +88,7 @@ class LanggraphModule(BaseModule):
|
||||
|
||||
优先级:
|
||||
1. 如果传入的 filename 作为路径(可包含或不包含 .md)直接存在文件,则直接读取该文件。
|
||||
2. 否则,从 MODULES_DATA/<module_name>/ 目录下按原有逻辑加载 (自动补全 .md 后缀)。
|
||||
2. 否则,从 DATA/<module_name>/ 目录下按原有逻辑加载 (自动补全 .md 后缀)。
|
||||
|
||||
Args:
|
||||
filename (str): 可以是一个直接文件路径,或是模板名(不含 .md)。
|
||||
@@ -82,7 +104,7 @@ class LanggraphModule(BaseModule):
|
||||
fname = filename
|
||||
else:
|
||||
fname = f"{filename}.md"
|
||||
template_path = os.path.join(MODULE_DATA_DIR, self.module_name, fname)
|
||||
template_path = os.path.join(DATA_DIR, self.module_name, fname)
|
||||
|
||||
try:
|
||||
with open(template_path, 'r', encoding='utf-8') as f:
|
||||
@@ -104,3 +126,4 @@ class LanggraphModule(BaseModule):
|
||||
|
||||
def run(self):
|
||||
self.run_graph()
|
||||
return self.agent_state
|
||||
|
||||
+134
-2
@@ -1,16 +1,148 @@
|
||||
import os
|
||||
import sys
|
||||
from abc import ABC
|
||||
|
||||
from langchain_core.prompts import SystemMessagePromptTemplate, HumanMessagePromptTemplate
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
from langgraph.checkpoint.memory import MemorySaver
|
||||
from langgraph.graph.state import CompiledStateGraph
|
||||
|
||||
from Lib.configs import DATA_DIR
|
||||
from Lib.llmapi import AgentState
|
||||
from Lib.log import logger
|
||||
|
||||
|
||||
class BasePlaybook(ABC):
|
||||
def __init__(self, params):
|
||||
RUN_AS_JOB = False # 是否作为后台任务运行
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._params = params
|
||||
self._params = {}
|
||||
self.logger = logger
|
||||
|
||||
@staticmethod
|
||||
def _get_main_script_name():
|
||||
"""
|
||||
获取主执行脚本的文件名(不含扩展名)。
|
||||
无论当前代码在哪个模块中运行,sys.argv[0]始终指向最初启动的脚本。
|
||||
"""
|
||||
try:
|
||||
# 1. 获取主执行脚本的完整路径
|
||||
script_path = sys.argv[0]
|
||||
|
||||
# 2. 从完整路径中提取文件名
|
||||
script_filename = os.path.basename(script_path)
|
||||
|
||||
# 3. 分离文件名和扩展名
|
||||
script_name, _ = os.path.splitext(script_filename)
|
||||
|
||||
return script_name
|
||||
except IndexError as e:
|
||||
raise RuntimeError("无法获取主执行脚本名称,sys.argv[0]不存在。") from e
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"获取主执行脚本名称时发生错误: {e}") from e
|
||||
|
||||
@property
|
||||
def playbook_name(self):
|
||||
"""获取模块加载路径"""
|
||||
name = self.__module__.split(".")[-1]
|
||||
if name == "__main__":
|
||||
return self._get_main_script_name()
|
||||
else:
|
||||
return name
|
||||
|
||||
def param(self, key, default=None):
|
||||
return self._params.get(key, default)
|
||||
|
||||
def run(self):
|
||||
pass
|
||||
|
||||
|
||||
class LanggraphPlaybook(BasePlaybook):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.graph: CompiledStateGraph = None
|
||||
self.agent_state = None
|
||||
|
||||
## LLM PART
|
||||
@staticmethod
|
||||
def get_checkpointer():
|
||||
checkpointer = MemorySaver()
|
||||
return checkpointer
|
||||
|
||||
def load_system_prompt_template(self, filename):
|
||||
"""加载系统提示模板。
|
||||
|
||||
优先级:
|
||||
1. 如果传入的 filename 作为路径(可包含或不包含 .md)直接存在文件,则直接读取该文件。
|
||||
2. 否则,从 DATA/<module_name>/ 目录下按原有逻辑加载 (自动补全 .md 后缀)。
|
||||
|
||||
Args:
|
||||
filename (str): 可以是一个直接文件路径,或是模板名(不含 .md)。
|
||||
Returns:
|
||||
SystemMessagePromptTemplate: 解析后的系统提示模板对象。
|
||||
Raises:
|
||||
Exception: 当文件无法读取时抛出。
|
||||
"""
|
||||
if os.path.isfile(filename):
|
||||
template_path = filename
|
||||
else:
|
||||
if filename.endswith('.md'):
|
||||
fname = filename
|
||||
else:
|
||||
fname = f"{filename}.md"
|
||||
template_path = os.path.join(DATA_DIR, self.playbook_name, fname)
|
||||
|
||||
try:
|
||||
with open(template_path, 'r', encoding='utf-8') as f:
|
||||
system_prompt_template: SystemMessagePromptTemplate = SystemMessagePromptTemplate.from_template(f.read())
|
||||
logger.debug(f"Loaded system prompt template from: {template_path}")
|
||||
return system_prompt_template
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load prompt template {template_path}: {str(e)}")
|
||||
raise e
|
||||
|
||||
def load_human_prompt_template(self, filename):
|
||||
"""加载系统提示模板。
|
||||
|
||||
优先级:
|
||||
1. 如果传入的 filename 作为路径(可包含或不包含 .md)直接存在文件,则直接读取该文件。
|
||||
2. 否则,从 DATA/<module_name>/ 目录下按原有逻辑加载 (自动补全 .md 后缀)。
|
||||
|
||||
Args:
|
||||
filename (str): 可以是一个直接文件路径,或是模板名(不含 .md)。
|
||||
Returns:
|
||||
SystemMessagePromptTemplate: 解析后的系统提示模板对象。
|
||||
Raises:
|
||||
Exception: 当文件无法读取时抛出。
|
||||
"""
|
||||
if os.path.isfile(filename):
|
||||
template_path = filename
|
||||
else:
|
||||
if filename.endswith('.md'):
|
||||
fname = filename
|
||||
else:
|
||||
fname = f"{filename}.md"
|
||||
template_path = os.path.join(DATA_DIR, self.playbook_name, fname)
|
||||
|
||||
try:
|
||||
with open(template_path, 'r', encoding='utf-8') as f:
|
||||
human_prompt_template: HumanMessagePromptTemplate = HumanMessagePromptTemplate.from_template(f.read())
|
||||
logger.debug(f"Loaded system prompt template from: {template_path}")
|
||||
return human_prompt_template
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load prompt template {template_path}: {str(e)}")
|
||||
raise e
|
||||
|
||||
def run_graph(self):
|
||||
self.graph.checkpointer.delete_thread(self.playbook_name)
|
||||
config = RunnableConfig()
|
||||
config["configurable"] = {"thread_id": self.playbook_name}
|
||||
if self.agent_state is None:
|
||||
self.agent_state = AgentState(messages=[], alert_raw={}, temp_data={}, analyze_result={})
|
||||
for event in self.graph.stream(self.agent_state, config, stream_mode="values"):
|
||||
self.logger.debug(event)
|
||||
|
||||
def run(self):
|
||||
self.run_graph()
|
||||
return self.agent_state
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
import os
|
||||
|
||||
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
MODULE_DATA_DIR = os.path.join(BASE_DIR, 'MODULES_DATA')
|
||||
DATA_DIR = os.path.join(BASE_DIR, 'DATA')
|
||||
REDIS_CONSUMER_GROUP = 'AI_SOC_FRAMEWORK_GROUP'
|
||||
REDIS_CONSUMER_NAME = 'AI_SOC_FRAMEWORK_CONSUMER_0'
|
||||
|
||||
|
||||
+155
@@ -0,0 +1,155 @@
|
||||
from typing import Annotated
|
||||
|
||||
from Lib.External.nocolyapi import WorksheetRow
|
||||
|
||||
|
||||
def get_dfmea_record_by_crs_id(
|
||||
crs_id: Annotated[str, "CRS ID"]
|
||||
) -> Annotated[list, "DFMEA record list with nested Requirement, Failure Mode, Failure Cause, Prevention Control"]:
|
||||
"""
|
||||
根据指定的CRS ID,检索DFMEA记录树,包括Requirement、Failure Mode、Failure Cause和Prevention Control的嵌套结构。
|
||||
|
||||
参数:
|
||||
crs_id (str): CRS ID,用于筛选Requirement记录。
|
||||
|
||||
返回:
|
||||
list: 包含所有Requirement树的列表,每个Requirement下嵌套Failure Mode、Failure Cause和Prevention Control。
|
||||
|
||||
结构示例:
|
||||
[
|
||||
{
|
||||
"crs_id": ...,
|
||||
"requirement": ...,
|
||||
"function": ...,
|
||||
"classification": ...,
|
||||
"failure_mode": [
|
||||
{
|
||||
"potential_failure_mode": ...,
|
||||
"local_effect_of_failure": ...,
|
||||
"end_effect_of_failure_effect": ...,
|
||||
"end_effect_of_failure_rationale": ...,
|
||||
"end_effect_of_failure_severity": ...,
|
||||
"failure_cause": [
|
||||
{
|
||||
"cause_of_failure": ...,
|
||||
"occurrence": ...,
|
||||
"detection": ...,
|
||||
"detection_control": ...,
|
||||
"prevention_control": [
|
||||
{
|
||||
"srs_id": ...,
|
||||
"prevention_control": ...
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
用法示例:
|
||||
get_dfmea_record_by_crs_id("PCC-CT.CRS.3400")
|
||||
"""
|
||||
filter = {
|
||||
"type": "group",
|
||||
"logic": "AND",
|
||||
"children": [
|
||||
{
|
||||
"type": "condition",
|
||||
"field": "crs_id",
|
||||
"operator": "eq",
|
||||
"value": crs_id
|
||||
},
|
||||
]
|
||||
}
|
||||
rows = WorksheetRow.list("requirement", filter)
|
||||
result_list = [] # 这将是一个包含所有 "Requirement" 树的列表
|
||||
|
||||
# --- 1. 遍历 A (Requirement) ---
|
||||
for row in rows:
|
||||
requirement = WorksheetRow.get("requirement", row.get("rowId"))
|
||||
|
||||
# --- A层 (Requirement) 记录 ---
|
||||
# 这是我们树的 "根"
|
||||
req_record = {
|
||||
"crs_id": requirement.get("crs_id"),
|
||||
"requirement": requirement.get("requirement"),
|
||||
"function": requirement.get("function"),
|
||||
"classification": requirement.get("classification"),
|
||||
}
|
||||
|
||||
# 准备 B 层的列表
|
||||
failure_mode_list = []
|
||||
failure_mode_rowid_list = requirement.get("failure_mode", [])
|
||||
|
||||
# --- 2. 遍历 B (Failure Mode) ---
|
||||
for failure_mode_rowid in failure_mode_rowid_list:
|
||||
failure_mode = WorksheetRow.get("failure_mode", failure_mode_rowid)
|
||||
|
||||
# --- B层 (Failure Mode) 记录 ---
|
||||
fm_record = {
|
||||
"potential_failure_mode": failure_mode.get("potential_failure_mode"),
|
||||
"local_effect_of_failure": failure_mode.get("local_effect_of_failure"),
|
||||
"end_effect_of_failure_effect": failure_mode.get("end_effect_of_failure_effect"),
|
||||
"end_effect_of_failure_rationale": failure_mode.get("end_effect_of_failure_rationale"),
|
||||
"end_effect_of_failure_severity": failure_mode.get("end_effect_of_failure_severity"),
|
||||
}
|
||||
|
||||
# 准备 C 层的列表
|
||||
failure_cause_list = []
|
||||
failure_cause_rowid_list = failure_mode.get("failure_cause", [])
|
||||
|
||||
# --- 3. 遍历 C (Failure Cause) ---
|
||||
for failure_cause_rowid in failure_cause_rowid_list:
|
||||
failure_cause = WorksheetRow.get("failure_cause", failure_cause_rowid)
|
||||
|
||||
# --- C层 (Failure Cause) 记录 ---
|
||||
fc_record = {
|
||||
"cause_of_failure": failure_cause.get("cause_of_failure"),
|
||||
"occurrence": failure_cause.get("occurrence"),
|
||||
"detection": failure_cause.get("detection"),
|
||||
"detection_control": failure_cause.get("detection_control"),
|
||||
}
|
||||
|
||||
# 准备 D 层的列表
|
||||
prevention_control_list = []
|
||||
prevention_control_rowid_list = failure_cause.get("prevention_control", [])
|
||||
|
||||
# --- 4. 遍历 D (Prevention Control) ---
|
||||
# 这是最内层,即 "叶子"
|
||||
for prevention_control_rowid in prevention_control_rowid_list:
|
||||
prevention_control = WorksheetRow.get("prevention_control", prevention_control_rowid)
|
||||
|
||||
# --- D层 (Prevention Control) 记录 ---
|
||||
pc_record = {
|
||||
"srs_id": prevention_control.get("srs_id"),
|
||||
"prevention_control": prevention_control.get("prevention_control"),
|
||||
}
|
||||
|
||||
# 将 D 记录添加到 D 列表
|
||||
prevention_control_list.append(pc_record)
|
||||
|
||||
# --- 循环结束, 向上组装 ---
|
||||
|
||||
# 将 D 列表 (prevention_control) 附加到 C 记录
|
||||
if prevention_control_list: # 你也可以保留空列表: fc_record["prevention_control"] = prevention_control_list
|
||||
fc_record["prevention_control"] = prevention_control_list
|
||||
|
||||
# 将 C 记录添加到 C 列表
|
||||
failure_cause_list.append(fc_record)
|
||||
|
||||
# 将 C 列表 (failure_cause) 附加到 B 记录
|
||||
if failure_cause_list:
|
||||
fm_record["failure_cause"] = failure_cause_list
|
||||
|
||||
# 将 B 记录添加到 B 列表
|
||||
failure_mode_list.append(fm_record)
|
||||
|
||||
# 将 B 列表 (failure_mode) 附加到 A 记录
|
||||
if failure_mode_list:
|
||||
req_record["failure_mode"] = failure_mode_list
|
||||
|
||||
# 将 A 记录 (完整的树) 添加到最终结果列表
|
||||
result_list.append(req_record)
|
||||
return result_list
|
||||
@@ -15,6 +15,7 @@ class RuleDefinition:
|
||||
case_title_template: str = None,
|
||||
deduplication_window: str = "24h",
|
||||
source: str = "Default",
|
||||
workbook: str = None
|
||||
):
|
||||
|
||||
self.rule_id = rule_id
|
||||
@@ -22,6 +23,7 @@ class RuleDefinition:
|
||||
self.deduplication_fields = deduplication_fields
|
||||
self.case_title_template = case_title_template
|
||||
self.source = source
|
||||
self.workbook = workbook
|
||||
valid_windows = ['10m', '30m', '1h', '8h', '12h', '24h']
|
||||
if deduplication_window not in valid_windows:
|
||||
raise ValueError(f"'{deduplication_window}' 不是一个有效的时间窗口选项。请从 {valid_windows} 中选择。")
|
||||
|
||||
@@ -10,6 +10,7 @@ from Lib.configs import EXPIRE_MINUTES
|
||||
|
||||
class Xcache(object):
|
||||
XCACHE_TOKEN = "XCACHE_TOKEN"
|
||||
XCACHE_MODULES_TASK_LIST = "XCACHE_MODULES_TASK_LIST"
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
@@ -31,3 +32,36 @@ class Xcache(object):
|
||||
keys = cache.keys(re_key)
|
||||
for key in keys:
|
||||
cache.delete(key)
|
||||
|
||||
@staticmethod
|
||||
def get_module_task_by_uuid(task_uuid):
|
||||
key = f"{Xcache.XCACHE_MODULES_TASK_LIST}_{task_uuid}"
|
||||
req = cache.get(key)
|
||||
return req
|
||||
|
||||
@staticmethod
|
||||
def list_module_tasks():
|
||||
re_key = f"{Xcache.XCACHE_MODULES_TASK_LIST}_*"
|
||||
keys = cache.keys(re_key)
|
||||
reqs = []
|
||||
for key in keys:
|
||||
reqs.append(cache.get(key))
|
||||
return reqs
|
||||
|
||||
@staticmethod
|
||||
def create_module_task(req):
|
||||
"""任务队列"""
|
||||
key = f"{Xcache.XCACHE_MODULES_TASK_LIST}_{req.get('uuid')}"
|
||||
cache.set(key, req, None)
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def del_module_task_by_uuid(task_uuid):
|
||||
key = f"{Xcache.XCACHE_MODULES_TASK_LIST}_{task_uuid}"
|
||||
cache.delete(key)
|
||||
|
||||
@staticmethod
|
||||
def get_module_task_length():
|
||||
re_key = f"{Xcache.XCACHE_MODULES_TASK_LIST}_*"
|
||||
keys = cache.keys(re_key)
|
||||
return len(keys)
|
||||
|
||||
@@ -9,7 +9,8 @@ from langgraph.graph.state import CompiledStateGraph
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from Lib.External.llmapi import LLMAPI
|
||||
from Lib.External.nocolyapi import InputAlert, common_handler
|
||||
from Lib.External.nocolyapi import InputAlert
|
||||
from Lib.External.sirpapi import common_handler
|
||||
from Lib.External.thehiveclient import TheHiveClient
|
||||
from Lib.api import string_to_string_time, get_current_time_string
|
||||
from Lib.basemodule import LanggraphModule
|
||||
@@ -173,7 +174,7 @@ class Module(LanggraphModule):
|
||||
"description": description,
|
||||
"reference": "https://your-siem-or-device-url.com/data?source=123456",
|
||||
"summary_ai": analyze_result.reasoning,
|
||||
"artifacts": [
|
||||
"artifact": [
|
||||
{
|
||||
"type": "mail_to",
|
||||
"value": mail_to,
|
||||
@@ -199,7 +200,8 @@ class Module(LanggraphModule):
|
||||
rule_id=self.module_name,
|
||||
rule_name=rule_name,
|
||||
deduplication_fields=["mail_from"],
|
||||
source="Email"
|
||||
source="Email",
|
||||
workbook="PHISHING_L2_WORKBOOK"
|
||||
)
|
||||
case_row_id = common_handler(input_alert, rule)
|
||||
return state
|
||||
@@ -222,6 +224,5 @@ class Module(LanggraphModule):
|
||||
|
||||
if __name__ == "__main__":
|
||||
module = Module()
|
||||
module.debug_alert_name = "ES-Rule-21-Phishing_user_report_mail" # needed when debug module, framework will read redis stream by this name
|
||||
module.debug_message_id = "0-0"
|
||||
module.run()
|
||||
|
||||
@@ -5,7 +5,8 @@ from typing import Optional, Union, Dict, Any
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from Lib.External.difyclient import DifyClient
|
||||
from Lib.External.nocolyapi import InputAlert, common_handler
|
||||
from Lib.External.nocolyapi import InputAlert
|
||||
from Lib.External.sirpapi import common_handler
|
||||
from Lib.api import string_to_string_time, get_current_time_string
|
||||
from Lib.basemodule import BaseModule
|
||||
from Lib.ruledefinition import RuleDefinition
|
||||
@@ -93,7 +94,7 @@ class Module(BaseModule):
|
||||
"description": description,
|
||||
"reference": "https://your-siem-or-device-url.com/data?source=123456",
|
||||
"summary_ai": analyze_result.reasoning,
|
||||
"artifacts": [
|
||||
"artifact": [
|
||||
{
|
||||
"type": "mail_to",
|
||||
"value": mail_to,
|
||||
@@ -133,6 +134,5 @@ class Module(BaseModule):
|
||||
|
||||
if __name__ == "__main__":
|
||||
module = Module()
|
||||
module.debug_alert_name = "ES-Rule-22-Phishing_user_report_mail"
|
||||
module.debug_message_id = "0-0"
|
||||
module.run()
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
import json
|
||||
from typing import Dict, Any, Annotated, List
|
||||
|
||||
from langchain_core.messages import HumanMessage
|
||||
from langgraph.graph import StateGraph, add_messages
|
||||
from langgraph.graph.state import CompiledStateGraph
|
||||
from pydantic import BaseModel
|
||||
|
||||
from Lib.External.llmapi import LLMAPI
|
||||
from Lib.External.nocolyapi import WorksheetRow
|
||||
from Lib.External.sirpapi import Alert, Artifact
|
||||
from Lib.External.sirpapi import Notice
|
||||
from Lib.External.sirpapi import Playbook as SIRPPlaybook
|
||||
from Lib.baseplaybook import LanggraphPlaybook
|
||||
|
||||
|
||||
class AgentState(BaseModel):
|
||||
messages: Annotated[List[Any], add_messages]
|
||||
|
||||
alert: Dict
|
||||
suggestion: str
|
||||
|
||||
|
||||
class Playbook(LanggraphPlaybook):
|
||||
RUN_AS_JOB = True
|
||||
|
||||
def __init__(self):
|
||||
super().__init__() # do not delete this code
|
||||
self.init()
|
||||
|
||||
def init(self):
|
||||
def preprocess_node(state: AgentState):
|
||||
"""预处理数据"""
|
||||
# worksheet = self.param("worksheet")
|
||||
rowid = self.param("rowid")
|
||||
alert = WorksheetRow.get(Alert.WORKSHEET_ID, rowid, include_system_fields=False)
|
||||
artifacts = WorksheetRow.relations(Alert.WORKSHEET_ID, alert.get("rowId"), "artifact", relation_worksheet_id=Artifact.WORKSHEET_ID,
|
||||
include_system_fields=False)
|
||||
alert["artifact"] = artifacts
|
||||
|
||||
Notice.send(self.param("user"), "Alert_Suggestion_Gen_By_LLM preprocess_node Finish", f"rowid:{self.param('rowid')}")
|
||||
|
||||
state.alert = alert
|
||||
return state
|
||||
|
||||
# 定义node
|
||||
def analyze_node(state: AgentState):
|
||||
"""AI分析告警数据"""
|
||||
|
||||
# 加载system prompt
|
||||
system_prompt_template = self.load_system_prompt_template("L3_SOC_Analyst")
|
||||
|
||||
system_message = system_prompt_template.format()
|
||||
|
||||
# 构建few-shot示例
|
||||
few_shot_examples = [
|
||||
# HumanMessage(
|
||||
# content=json.dumps({
|
||||
# "requirement": ".",
|
||||
# })
|
||||
# ),
|
||||
# AIMessage(
|
||||
# content=json.dumps({
|
||||
# "function": "the amount of pneumothorax",
|
||||
# })
|
||||
# ),
|
||||
]
|
||||
|
||||
# 运行
|
||||
llm_api = LLMAPI()
|
||||
|
||||
llm = llm_api.get_model()
|
||||
|
||||
# 构建消息列表
|
||||
messages = [
|
||||
system_message,
|
||||
*few_shot_examples,
|
||||
HumanMessage(content=json.dumps(state.alert))
|
||||
]
|
||||
response = llm.invoke(messages)
|
||||
response = LLMAPI.extract_think(response) # langchain chatollama bug临时方案
|
||||
state.suggestion = response.content
|
||||
|
||||
Notice.send(self.param("user"), "Alert_Suggestion_Gen_By_LLM analyze_node Finish", f"rowid:{self.param('rowid')}")
|
||||
return state
|
||||
|
||||
def output_node(state: AgentState):
|
||||
"""处理分析结果"""
|
||||
|
||||
suggestion = state.suggestion
|
||||
fields = [
|
||||
{"id": "suggestion_ai", "value": suggestion},
|
||||
]
|
||||
rowid = self.param("rowid")
|
||||
WorksheetRow.update(Alert.WORKSHEET_ID, rowid, fields)
|
||||
|
||||
self.agent_state = state
|
||||
|
||||
Notice.send(self.param("user"), "Alert_Suggestion_Gen_By_LLM output_node Finish", f"rowid:{self.param('rowid')}")
|
||||
|
||||
SIRPPlaybook.update_status_and_remark(self.param("playbook_rowid"), "Success", "Get suggestion by ai agent completed.") # Success/Failed
|
||||
return state
|
||||
|
||||
# 编译graph
|
||||
workflow = StateGraph(AgentState)
|
||||
|
||||
workflow.add_node("preprocess_node", preprocess_node)
|
||||
workflow.add_node("analyze_node", analyze_node)
|
||||
workflow.add_node("output_node", output_node)
|
||||
|
||||
workflow.set_entry_point("preprocess_node")
|
||||
workflow.add_edge("preprocess_node", "analyze_node")
|
||||
workflow.add_edge("analyze_node", "output_node")
|
||||
workflow.set_finish_point("output_node")
|
||||
self.agent_state = AgentState(messages=[], alert={}, suggestion="")
|
||||
self.graph: CompiledStateGraph = workflow.compile(checkpointer=self.get_checkpointer())
|
||||
return True
|
||||
|
||||
def run(self):
|
||||
self.run_graph()
|
||||
return
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
params_debug = {'playbook': 'Alert_Suggestion_Gen_By_LLM', 'rowid': '55639caf-c648-4130-bc9f-8d38becfe20f', 'worksheet': 'alert'}
|
||||
module = Playbook()
|
||||
module._params = params_debug
|
||||
module.run()
|
||||
@@ -0,0 +1,41 @@
|
||||
import json
|
||||
import time
|
||||
|
||||
from Lib.External.nocolyapi import WorksheetRow
|
||||
from Lib.External.sirpapi import Playbook as SIRPPlaybook
|
||||
from Lib.baseplaybook import BasePlaybook
|
||||
|
||||
|
||||
class Playbook(BasePlaybook):
|
||||
RUN_AS_JOB = True
|
||||
|
||||
def __init__(self):
|
||||
super().__init__() # do not delete this code
|
||||
|
||||
def run(self):
|
||||
worksheet = self.param("worksheet")
|
||||
rowid = self.param("rowid")
|
||||
|
||||
artifact = WorksheetRow.get(worksheet, rowid, include_system_fields=False)
|
||||
self.logger.info(f"Querying threat intelligence for : {artifact}")
|
||||
|
||||
# 模拟查询威胁情报数据库,在实际应用中,这里应该调用外部API或数据库进行查询
|
||||
time.sleep(3)
|
||||
if artifact.get("type") not in ["ip", "domain", "hash", "vm_ip"]:
|
||||
ti_result = {"error": "Unsupported type. Please use 'ip', 'domain', or 'hash'."}
|
||||
else:
|
||||
ti_result = {"malicious": True, "score": 85, "description": "This IP is associated with known malicious activities.", "source": "ThreatIntelDB",
|
||||
"last_seen": "2024-10-01T12:34:56Z"}
|
||||
|
||||
fields = [{"id": "enrichment", "value": json.dumps(ti_result)}]
|
||||
WorksheetRow.update(worksheet, rowid, fields)
|
||||
|
||||
SIRPPlaybook.update_status_and_remark(self.param("playbook_rowid"), "Success", "Threat intelligence enrichment completed.") # Success/Failed
|
||||
return
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
params_debug = {'playbook': 'TI_artifact_query_mock', 'rowid': 'a966036e-b29e-4449-be48-23293bacac5d', 'worksheet': 'Artifact'}
|
||||
module = Playbook()
|
||||
module._params = params_debug
|
||||
module.run()
|
||||
@@ -0,0 +1,132 @@
|
||||
import json
|
||||
from typing import Dict, Any, Annotated, List
|
||||
|
||||
from langchain_core.messages import HumanMessage
|
||||
from langgraph.graph import StateGraph, add_messages
|
||||
from langgraph.graph.state import CompiledStateGraph
|
||||
from pydantic import BaseModel
|
||||
|
||||
from Lib.External.llmapi import LLMAPI
|
||||
from Lib.External.nocolyapi import WorksheetRow
|
||||
from Lib.External.sirpapi import Case, Alert, Artifact
|
||||
from Lib.External.sirpapi import Notice
|
||||
from Lib.External.sirpapi import Playbook as SIRPPlaybook
|
||||
from Lib.baseplaybook import LanggraphPlaybook
|
||||
|
||||
|
||||
class AgentState(BaseModel):
|
||||
messages: Annotated[List[Any], add_messages]
|
||||
|
||||
case: Dict
|
||||
suggestion: str
|
||||
|
||||
|
||||
class Playbook(LanggraphPlaybook):
|
||||
RUN_AS_JOB = True
|
||||
|
||||
def __init__(self):
|
||||
super().__init__() # do not delete this code
|
||||
self.init()
|
||||
|
||||
def init(self):
|
||||
def preprocess_node(state: AgentState):
|
||||
"""预处理数据"""
|
||||
# worksheet = self.param("worksheet")
|
||||
rowid = self.param("rowid")
|
||||
case = WorksheetRow.get(Case.WORKSHEET_ID, rowid, include_system_fields=False)
|
||||
|
||||
alerts = WorksheetRow.relations(Case.WORKSHEET_ID, rowid, "alert", relation_worksheet_id=Alert.WORKSHEET_ID, include_system_fields=False)
|
||||
for alert in alerts:
|
||||
artifacts = WorksheetRow.relations(Alert.WORKSHEET_ID, alert.get("rowId"), "artifact", relation_worksheet_id=Artifact.WORKSHEET_ID,
|
||||
include_system_fields=False)
|
||||
alert["artifact"] = artifacts
|
||||
case["alert"] = alerts
|
||||
|
||||
Notice.send(self.param("user"), "Case_Suggestion_Gen_By_LLM preprocess_node Finish", f"rowid:{self.param('rowid')}")
|
||||
|
||||
state.case = case
|
||||
return state
|
||||
|
||||
# 定义node
|
||||
def analyze_node(state: AgentState):
|
||||
"""AI分析告警数据"""
|
||||
|
||||
# 加载system prompt
|
||||
system_prompt_template = self.load_system_prompt_template("L3_SOC_Analyst")
|
||||
|
||||
system_message = system_prompt_template.format()
|
||||
|
||||
# 构建few-shot示例
|
||||
few_shot_examples = [
|
||||
# HumanMessage(
|
||||
# content=json.dumps({
|
||||
# "requirement": ".",
|
||||
# })
|
||||
# ),
|
||||
# AIMessage(
|
||||
# content=json.dumps({
|
||||
# "function": "the amount of pneumothorax",
|
||||
# })
|
||||
# ),
|
||||
]
|
||||
|
||||
# 运行
|
||||
llm_api = LLMAPI()
|
||||
|
||||
llm = llm_api.get_model()
|
||||
|
||||
# 构建消息列表
|
||||
messages = [
|
||||
system_message,
|
||||
*few_shot_examples,
|
||||
HumanMessage(content=json.dumps(state.case))
|
||||
]
|
||||
response = llm.invoke(messages)
|
||||
response = LLMAPI.extract_think(response) # langchain chatollama bug临时方案
|
||||
state.suggestion = response.content
|
||||
|
||||
Notice.send(self.param("user"), "Case_Suggestion_Gen_By_LLM analyze_node Finish", f"rowid:{self.param('rowid')}")
|
||||
return state
|
||||
|
||||
def output_node(state: AgentState):
|
||||
"""处理分析结果"""
|
||||
|
||||
suggestion = state.suggestion
|
||||
fields = [
|
||||
{"id": "suggestion_ai", "value": suggestion},
|
||||
]
|
||||
rowid = self.param("rowid")
|
||||
WorksheetRow.update(Case.WORKSHEET_ID, rowid, fields)
|
||||
|
||||
self.agent_state = state
|
||||
|
||||
Notice.send(self.param("user"), "Case_Suggestion_Gen_By_LLM output_node Finish", f"rowid:{self.param('rowid')}")
|
||||
|
||||
SIRPPlaybook.update_status_and_remark(self.param("playbook_rowid"), "Success", "Get suggestion by ai agent completed.") # Success/Failed
|
||||
return state
|
||||
|
||||
# 编译graph
|
||||
workflow = StateGraph(AgentState)
|
||||
|
||||
workflow.add_node("preprocess_node", preprocess_node)
|
||||
workflow.add_node("analyze_node", analyze_node)
|
||||
workflow.add_node("output_node", output_node)
|
||||
|
||||
workflow.set_entry_point("preprocess_node")
|
||||
workflow.add_edge("preprocess_node", "analyze_node")
|
||||
workflow.add_edge("analyze_node", "output_node")
|
||||
workflow.set_finish_point("output_node")
|
||||
self.agent_state = AgentState(messages=[], case={}, suggestion="")
|
||||
self.graph: CompiledStateGraph = workflow.compile(checkpointer=self.get_checkpointer())
|
||||
return True
|
||||
|
||||
def run(self):
|
||||
self.run_graph()
|
||||
return
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
params_debug = {'playbook': 'Case_Suggestion_Gen_By_LLM', 'rowid': '55639caf-c648-4130-bc9f-8d38becfe20f', 'worksheet': 'case'}
|
||||
module = Playbook()
|
||||
module._params = params_debug
|
||||
module.run()
|
||||
@@ -27,3 +27,4 @@ lxml
|
||||
openpyxl
|
||||
langchain-ollama
|
||||
uvicorn
|
||||
mcp[cli]
|
||||
Reference in New Issue
Block a user