add django

.gitignore

@@ -168,4 +168,5 @@ cython_debug/
 .idea/
 Test/*
 CONFIG.py
-Test
+Docker/log/*
+!Docker/log/.gitkeep

ASF/__init__.py

@@ -0,0 +1,8 @@
+# from __future__ import absolute_import, unicode_literals
+#
+#
+# # This will make sure the app is always imported when
+# # Django starts so that shared_task will use this app.
+# from .celery import app as celery_app
+#
+# __all__ = ('celery_app',)

ASF/asgi.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# @File  : asgi.py
+# @Date  : 2019/10/3
+# @Desc  :
+
+"""
+ASGI entrypoint. Configures Django and then runs the application
+defined in the ASGI_APPLICATION setting.
+"""
+import os
+
+import django
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ASF.settings')
+django.setup()
+from channels.auth import AuthMiddlewareStack
+from channels.routing import ProtocolTypeRouter, URLRouter
+from django.core.asgi import get_asgi_application
+
+websocket_urlpatterns = [
+
+]
+
+application = ProtocolTypeRouter({
+    "http": get_asgi_application(),  # Django 的 WSGI 应用处理 HTTP 请求
+    "websocket": AuthMiddlewareStack(
+        URLRouter(
+            websocket_urlpatterns  # WebSocket 处理
+        )
+    ),
+})

ASF/settings.py

@@ -0,0 +1,162 @@
+import os
+
+from django.conf import settings
+from django.core.management import utils
+
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+try:
+    SECRET_KEY = os.environ['SECRET_KEY']
+except:
+    SECRET_KEY = utils.get_random_secret_key()
+    os.environ['SECRET_KEY'] = SECRET_KEY
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = False
+
+ALLOWED_HOSTS = ['*']
+
+# Application definition
+
+MIDDLEWARE = [
+    'django.middleware.security.SecurityMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+]
+
+ROOT_URLCONF = 'ASF.urls'
+
+WSGI_APPLICATION = 'ASF.wsgi.application'
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+    },
+]
+
+LANGUAGE_CODE = 'en-us'
+
+TIME_ZONE = 'UTC'
+
+USE_I18N = True
+
+USE_L10N = True
+
+USE_TZ = False
+
+STATIC_URL = '/static/'
+
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'filters': {
+        'require_debug_true': {
+            '()': 'django.utils.log.RequireDebugTrue',
+        },
+    },
+    'formatters': {
+        'standard': {
+            'format': '[%(levelname)s][%(asctime).19s][%(filename)s][%(lineno)d][%(threadName)s] : %(message)s '
+        },
+    },
+    'handlers': {
+        'console': {
+            'level': 'INFO',
+            'class': 'logging.StreamHandler',
+            'formatter': 'standard',
+        },
+        'file': {
+            'level': 'INFO',
+            'class': 'logging.FileHandler',
+            'formatter': 'standard',
+            'filename': os.path.join(settings.BASE_DIR, 'Docker', 'log', 'django.log'),
+        },
+
+    },
+    'loggers': {
+        'django': {
+            'handlers': [
+                'console',
+                'file',
+            ],
+            'level': 'INFO',
+            'propagate': True
+        },
+
+    }
+}
+
+REST_FRAMEWORK = {
+    'DEFAULT_RENDERER_CLASSES': (
+        'rest_framework.renderers.JSONRenderer',
+    ),
+    'DEFAULT_PERMISSION_CLASSES': (
+
+        'rest_framework.permissions.IsAuthenticated',
+
+    ),
+    'DEFAULT_AUTHENTICATION_CLASSES': (
+        'Core.Handle.baseauth.BaseAuth',
+    )
+}
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
+
+ASGI_APPLICATION = 'ASF.asgi.application'
+
+DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
+    }
+}
+CHANNEL_LAYERS = {
+    "default": {
+        "BACKEND": "channels.layers.InMemoryChannelLayer",  # 使用内存后端
+    },
+}
+
+INSTALLED_APPS = [
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+
+    'rest_framework',
+    'rest_framework.authtoken',
+
+    'channels',
+    'Core',
+    'Forwarder.apps.ForwarderConfig'
+]
+
+from CONFIG import CACHES

ASF/urls.py

@@ -0,0 +1,19 @@
+from django.urls import re_path, include
+from rest_framework import routers
+
+from Core.views import BaseAuthView, CurrentUserView
+from Forwarder.views import WebhookSplunkView, WebhookKibanaView
+
+router = routers.DefaultRouter()
+router.register(r'api/login/account', BaseAuthView, basename="BaseAuth")
+router.register(r'api/currentUser', CurrentUserView, basename="CurrentUser")
+
+router.register(r'api/v1/webhook/splunk', WebhookSplunkView, basename="WebhookSplunkView")
+router.register(r'api/v1/webhook/kibana', WebhookKibanaView, basename="WebhookKibanaView")
+
+urlpatterns = [
+    re_path(r'^', include(router.urls)),
+]
+# from Lib.montior import MainMonitor
+#
+# MainMonitor().start()

ASF/wsgi.py

@@ -0,0 +1,16 @@
+"""
+WSGI config for BlackPost project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/2.1/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ASF.settings')
+
+application = get_wsgi_application()

CONFIG.example.py

@@ -30,3 +30,8 @@ THEHIVE_API_KEY = "xxx"
 NOCODB_URL = "http://192.168.1.114:8080"
 NOCODB_TOKEN = "xxx"
 NOCODB_ALERT_TABLE_ID = "xxx"
+
+# ollama config example
+# OPENAI_BASE_URL = 'http://localhost:11434/v1'
+# OPENAI_API_KEY='ollama'
+# OPENAI_MODEL = "qwen3:30b-a3b"

Core/Handle/__init__.py

@@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+# @File  : __init__.py.py
+# @Date  : 2021/2/25
+# @Desc  :

Core/Handle/baseauth.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+# @File  : baseauth.py
+# @Date  : 2021/2/25
+# @Desc  :
+import datetime
+
+from rest_framework import exceptions
+from rest_framework.authentication import TokenAuthentication
+
+from Lib.configs import EXPIRE_MINUTES
+from Lib.xcache import Xcache
+
+
+class BaseAuth(TokenAuthentication):
+    def authenticate_credentials(self, key=None):
+        # 搜索缓存的user token
+        cache_user = Xcache.alive_token(key)
+        if cache_user:
+            return cache_user, key
+
+        # 数据库中校验token
+        model = self.get_model()
+        try:
+            token = model.objects.select_related('user').get(key=key)
+        except model.DoesNotExist:
+            raise exceptions.AuthenticationFailed()
+
+        if not token.user.is_active:
+            raise exceptions.AuthenticationFailed()
+
+        # token超时清理
+        time_now = datetime.datetime.now()
+        if token.created < time_now - datetime.timedelta(minutes=EXPIRE_MINUTES):
+            token.delete()
+            raise exceptions.AuthenticationFailed()
+
+        # 缓存token
+        if token:
+            Xcache.set_token_user(key, token.user)
+        return token.user, token

Core/Handle/currentuser.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+# @File  : currentuser.py
+# @Date  : 2021/2/25
+# @Desc  :
+class CurrentUser(object):
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def list(user=None):
+        current_info = {
+            'name': user.username,
+            'currentAuthority': 'admin',
+            'userid': user.id,
+        }
+
+        return current_info

Core/Handle/user.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+# @File  : currentuser.py
+# @Date  : 2021/2/25
+# @Desc  :
+from django.contrib.auth.models import User
+
+from Core.serializers import UserAPISerializer
+
+
+class UserAPI(object):
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def list():
+        models = User.objects.all()
+        result = UserAPISerializer(models, many=True).data
+        return result
+
+    @staticmethod
+    def create_user(username, password):
+        if username.lower() == "root":
+            return False
+        try:
+            user = User.objects.get(username=username)
+            user.set_password(password)
+            user.save()
+            return True
+        except User.DoesNotExist:
+            try:
+                # 创建普通用户
+                user = User.objects.create_user(username=username, password=password)
+                user.save()
+                return True
+            except Exception as E:
+                return False
+
+    @staticmethod
+    def delete_user(username):
+        if username.lower() == "root":
+            return False
+        try:
+            user = User.objects.get(username=username)
+            user.delete()
+            return True
+        except User.DoesNotExist:
+            return False

Core/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class CoreConfig(AppConfig):
+    name = 'Core'

Core/models.py

@@ -0,0 +1,71 @@
+import ast
+
+from django.db import models
+
+
+# Create your models here.
+
+
+class DiyListField(models.TextField):
+    """数据库中用来存储list类型字段"""
+    description = "Stores a python list"
+
+    def __init__(self, *args, **kwargs):
+        super(DiyListField, self).__init__(*args, **kwargs)
+
+    def get_prep_value(self, value):  # 将python对象转为查询值
+        if value is None:
+            return value
+
+        return str(value)  # use str(value) in Python 3
+
+    @staticmethod
+    def from_db_value(value, expression, connection):
+        if not value:
+            value = []
+        if isinstance(value, list):
+            return value
+        # 直接将字符串转换成python内置的list
+        try:
+            return ast.literal_eval(value)
+        except Exception as E:
+            from Lib.log import logger
+            logger.exception(E)
+            logger.error(value)
+            return []
+
+    def value_to_string(self, obj):
+        value = self._get_val_from_obj(obj)
+        return self.get_db_prep_value(value)
+
+
+class DiyDictField(models.TextField):
+    """数据库中用来存储dict类型字段"""
+    description = "Stores a python dict"
+
+    def __init__(self, *args, **kwargs):
+        super(DiyDictField, self).__init__(*args, **kwargs)
+
+    def get_prep_value(self, value):  # 将python对象转为查询值
+        if value is None:
+            return value
+
+        return str(value)  # use str(value) in Python 3
+
+    def from_db_value(self, value, expression, connection):
+        if not value:
+            value = []
+        if isinstance(value, dict):
+            return value
+        # 直接将字符串转换成python内置的list
+        try:
+            return ast.literal_eval(value)
+        except Exception as E:
+            from Lib.log import logger
+            logger.exception(E)
+            logger.error(value)
+            return {}
+
+    def value_to_string(self, obj):
+        value = self._get_val_from_obj(obj)
+        return self.get_db_prep_value(value)

Core/module_loader.py

@@ -1,42 +0,0 @@
-import os
-
-from watchdog.events import FileSystemEventHandler
-from watchdog.observers import Observer
-
-from Lib.log import logger
-
-
-class ModuleChangeHandler(FileSystemEventHandler):
-    def __init__(self, engine):
-        self.engine = engine
-
-    def _is_valid_module(self, path: str) -> bool:
-        filename = os.path.basename(path)
-        return filename.endswith(".py") and not filename.startswith(("_", "."))
-
-    def on_created(self, event):
-        if not event.is_directory and self._is_valid_module(event.src_path):
-            logger.info(f"New module file detected: {event.src_path}")
-            module_name = os.path.basename(event.src_path).replace(".py", "")
-            self.engine.load_module(module_name, event.src_path)
-
-    def on_deleted(self, event):
-        if not event.is_directory and self._is_valid_module(event.src_path):
-            logger.info(f"Module file deleted: {event.src_path}")
-            module_name = os.path.basename(event.src_path).replace(".py", "")
-            self.engine.unload_module(module_name)
-
-    def on_modified(self, event):
-        if not event.is_directory and self._is_valid_module(event.src_path):
-            logger.info(f"Module file modified: {event.src_path}")
-            module_name = os.path.basename(event.src_path).replace(".py", "")
-            self.engine.reload_module(module_name, event.src_path)
-
-
-def start_watching(engine, path: str) -> Observer:
-    event_handler = ModuleChangeHandler(engine)
-    observer = Observer()
-    observer.schedule(event_handler, path, recursive=False)
-    observer.start()
-    logger.info(f"Starting to monitor directory: '{path}'")
-    return observer

Core/serializers.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+# @File  : serializers.py
+# @Date  : 2018/11/15
+# @Desc  :
+
+
+from rest_framework.serializers import Serializer, CharField, BooleanField
+
+
+class UserAPISerializer(Serializer):
+    username = CharField()
+    is_superuser = BooleanField()

Core/views.py

@@ -0,0 +1,63 @@
+import datetime
+
+from rest_framework.authtoken.models import Token
+from rest_framework.authtoken.serializers import AuthTokenSerializer
+from rest_framework.generics import UpdateAPIView, DestroyAPIView
+from rest_framework.permissions import AllowAny
+from rest_framework.response import Response
+from rest_framework.viewsets import ModelViewSet
+
+from Core.Handle.currentuser import CurrentUser
+from Lib.api import data_return
+from Lib.baseview import BaseView
+from Lib.log import logger
+
+
+class BaseAuthView(ModelViewSet, UpdateAPIView, DestroyAPIView):
+    queryset = []  # 设置类的queryset
+    serializer_class = AuthTokenSerializer  # 设置类的serializer_class
+    authentication_classes = []
+    permission_classes = [AllowAny]
+
+    def create(self, request, pk=None, **kwargs):
+
+        null_response = {"status": "error", "type": "account", "currentAuthority": "guest",
+                         "token": "forguest"}
+
+        # 检查是否为diypassword
+        # Get encrypted password and decrypt it
+        username = request.data.get('username')
+        password = request.data.get('password')
+
+        try:
+            serializer = AuthTokenSerializer(data={"username": username, "password": password})
+            if serializer.is_valid():
+                token, created = Token.objects.get_or_create(user=serializer.validated_data['user'])
+                time_now = datetime.datetime.now()
+                if created or token.created < time_now - datetime.timedelta(minutes=EXPIRE_MINUTES):
+                    # 更新创建时间,保持token有效
+                    token.delete()
+                    token = Token.objects.create(user=serializer.validated_data['user'])
+                    token.created = time_now
+                    token.save()
+                null_response['status'] = 'ok'
+                null_response['currentAuthority'] = 'admin'  # 当前为单用户模式,默认为admin
+                null_response['token'] = token.key
+                context = data_return(201, null_response, BASEAUTH_MSG_ZH.get(201), BASEAUTH_MSG_EN.get(201))
+                return Response(context)
+            else:
+                context = data_return(301, null_response, BASEAUTH_MSG_ZH.get(301), BASEAUTH_MSG_EN.get(301))
+                return Response(context)
+        except Exception as E:
+            logger.exception(E)
+            context = data_return(301, null_response, BASEAUTH_MSG_ZH.get(301), BASEAUTH_MSG_EN.get(301))
+            return Response(context)
+
+
+class CurrentUserView(BaseView):
+    def list(self, request, **kwargs):
+        """查询数据库中的host信息"""
+        user = request.user
+        user_info = CurrentUser.list(user)
+        context = data_return(301, user_info, BASEAUTH_MSG_ZH.get(301), BASEAUTH_MSG_EN.get(301))
+        return Response(context)

Docker/Forward/app.py

@@ -1,13 +1,36 @@
-import sys
 import os
+import sys
 
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-from flask import Flask, request, jsonify
 
+from flask import Flask, request, jsonify
 from CONFIG import FLASK_LISTEN_PORT, FLASK_LISTEN_HOST
-from Forwarder.log import logger
 from Lib.redis_stream_api import RedisStreamAPI
 
+import logging
+import sys
+
+
+def setup_logging(log_file='forwarder.log'):
+    """
+    Configures the root logger for the entire application.
+    - Logs to both a file and the console.
+    - Sets a standardized format for log messages.
+    """
+    logging.basicConfig(
+        level=logging.INFO,
+        format='%(levelname)s - %(asctime)s - [%(name)s] - %(message)s',
+        datefmt='%Y-%m-%d %H:%M:%S',
+        handlers=[
+            logging.FileHandler(log_file),
+            logging.StreamHandler(sys.stdout)
+        ]
+    )
+
+
+setup_logging()
+logger = logging.getLogger("forwarder")
+
 redis_stream_api = RedisStreamAPI()
 app = Flask(__name__)
 

External/nocolyapi.py

@@ -1,8 +1,26 @@
-from typing import TypedDict, Literal, List, Union, Any
+from typing import TypedDict, Literal, List, Union, Any, Dict, Optional
 
 import requests
 
 from CONFIG import NOCOLY_URL, AISOAR_APPKEY, AISOAR_SIGN
+from Lib.api import get_current_time_string, string_to_timestamp
+from Lib.ruledefinition import RuleDefinition
+
+
+class InputAlert(TypedDict):
+    source: str
+    rule_id: str
+    rule_name: str
+    name: str
+    alert_date: str
+    create_data: str
+    tags: List[str]
+    severity: str
+    reference: str
+    description: str
+    summary_ai: Optional[Union[str, Dict[str, Any]]]
+    artifacts: List[Dict]
+    raw_log: Dict
 
 
 class FieldType(TypedDict):
@@ -396,3 +414,92 @@ class Option(object):
         for option in options:
             value_list.append(option.get("value"))
         return value_list
+
+
+def common_handler(alert: InputAlert, rule_def: RuleDefinition) -> str:
+    # artifact
+    artifact_rowid_list = []
+    artifacts = alert.get("artifacts", [])
+    for artifact in artifacts:
+        deduplication_key = artifact["deduplication_key"]
+        artifact_fields = [
+            {"id": "type", "value": artifact["type"]},
+            {"id": "value", "value": artifact["value"]},
+            {"id": "enrichment", "value": artifact["enrichment"]},
+            {"id": "deduplication_key", "value": deduplication_key},
+        ]
+
+        row = Artifact.get_by_deduplication_key(deduplication_key)
+        if row is None:
+            row_id = Artifact.create(artifact_fields)
+        else:
+            row_id = row.get("rowId")
+            Artifact.update(row_id, artifact_fields)
+
+        artifact_rowid_list.append(row_id)
+
+    alert_fields = [
+        {"id": "tags", "value": alert.get("tags"), "type": 2},
+        {"id": "severity", "value": alert.get("severity")},
+        {"id": "source", "value": alert.get("source")},
+        {"id": "alert_date", "value": alert.get("alert_date")},
+        {"id": "reference", "value": alert.get("reference")},
+        {"id": "description", "value": alert.get("description")},
+        {"id": "raw_log", "value": alert.get("raw_log")},
+        {"id": "rule_id", "value": alert.get("rule_id")},
+        {"id": "rule_name", "value": alert.get("rule_name")},
+        {"id": "name", "value": alert.get("name")},
+        {"id": "summary_ai", "value": alert.get("summary_ai")},
+        {"id": "artifacts", "value": artifact_rowid_list},
+    ]
+
+    # alert
+    row_id_alert = Alert.create(alert_fields)
+
+    # case
+    timestamp = string_to_timestamp(alert["alert_date"], "%Y-%m-%dT%H:%M:%SZ")
+    deduplication_key = rule_def.generate_deduplication_key(artifacts=artifacts, timestamp=timestamp)
+
+    row = Case.get_by_deduplication_key(deduplication_key)
+    if row is None:
+        case_field = [
+            {"id": "title", "value": rule_def.generate_case_title(artifacts=artifacts)},
+            {"id": "deduplication_key", "value": deduplication_key},
+            {"id": "alert", "value": [row_id_alert]},
+            {"id": "case_status", "value": "New"},
+            {"id": "created_at", "value": get_current_time_string()},
+            {"id": "tags", "value": alert["tags"], "type": 2},
+            {"id": "severity", "value": alert["severity"]},
+            {"id": "type", "value": rule_def.source},
+            {"id": "description", "value": alert["description"]},
+        ]
+        row_id_create = Case.create(case_field)
+        return row_id_create
+    else:
+        row_id_case = row.get("rowId")
+        existing_alerts = row.get("alert", [])
+        if row_id_alert not in existing_alerts:
+            existing_alerts.append(row_id_alert)
+
+        option_new_score = OptionSet.get_option_by_name_and_value("alert_case_severity", alert["severity"]).get("score", 0)
+
+        severity_value_exist = row.get("severity")[0].get("value")
+        option_exist_score = OptionSet.get_option_by_name_and_value("alert_case_severity", severity_value_exist).get("score", 0)
+
+        if option_new_score > option_exist_score:
+            severity = alert["severity"]
+        else:
+            severity = severity_value_exist
+
+        tags_exist = Option.to_value_list(row.get("tags", []))
+        for tag in alert["tags"]:
+            if tag not in tags_exist:
+                tags_exist.append(tag)
+
+        case_field = [
+            {"id": "alert", "value": existing_alerts},
+            {"id": "severity", "value": severity},
+            {"id": "tags", "value": tags_exist, "type": 2}
+        ]
+        row_id_updated = Case.update(row_id_case, case_field)
+        return row_id_updated

Forwarder/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class ForwarderConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'Forwarder'

Forwarder/log.py

@@ -1,23 +0,0 @@
-import logging
-import sys
-
-
-def setup_logging(log_file='forwarder.log'):
-    """
-    Configures the root logger for the entire application.
-    - Logs to both a file and the console.
-    - Sets a standardized format for log messages.
-    """
-    logging.basicConfig(
-        level=logging.INFO,
-        format='%(levelname)s - %(asctime)s - [%(name)s] - %(message)s',
-        datefmt='%Y-%m-%d %H:%M:%S',
-        handlers=[
-            logging.FileHandler(log_file),
-            logging.StreamHandler(sys.stdout)
-        ]
-    )
-
-
-setup_logging()
-logger = logging.getLogger("forwarder")

Forwarder/models.py

@@ -0,0 +1 @@
+# Create your models here.

Forwarder/views.py

@@ -0,0 +1,53 @@
+from rest_framework.permissions import AllowAny
+from rest_framework.response import Response
+
+from Lib.api import data_return
+from Lib.baseview import BaseView
+from Lib.configs import CODE_MSG_ZH, CODE_MSG_EN
+from Lib.log import logger
+from Lib.redis_stream_api import RedisStreamAPI
+
+
+class WebhookSplunkView(BaseView):
+    permission_classes = [AllowAny]
+
+    def create(self, request, **kwargs):
+        try:
+            result = request.data.get('result')
+            search_name = request.data.get('search_name')
+            sid = request.data.get('sid')
+            app = request.data.get('app')
+            owner = request.data.get('owner')
+            results_link = request.data.get('results_link')
+            app = request.data.get('app')
+            logger.info(f"Splunk webhook: {request.data}")
+            redis_stream_api = RedisStreamAPI()
+            redis_stream_api.send_message(search_name, result)
+            logger.info("Message sent to Redis stream")
+            context = data_return(200, {}, CODE_MSG_ZH.get(200), CODE_MSG_EN.get(200))
+            return Response(context)
+        except Exception as E:
+            logger.exception(E)
+            context = data_return(500, {}, CODE_MSG_ZH.get(500), CODE_MSG_EN.get(500))
+        return Response(context)
+
+
+class WebhookKibanaView(BaseView):
+    permission_classes = [AllowAny]
+
+    def create(self, request, **kwargs):
+        try:
+            redis_stream_api = RedisStreamAPI()
+            rule_name = request.data.get('rule').get("name")
+            hits = request.data.get('context').get("hits")
+            for hit in hits:
+                _source = hit.pop('_source', {})
+                logger.info(f"elasticsearch webhook: {hit}")
+                redis_stream_api.send_message(rule_name, _source)
+                logger.info("Message sent to Redis stream")
+            context = data_return(200, {}, CODE_MSG_ZH.get(200), CODE_MSG_EN.get(200))
+            return Response(context)
+        except Exception as E:
+            logger.exception(E)
+            context = data_return(500, {}, CODE_MSG_ZH.get(500), CODE_MSG_EN.get(500))
+        return Response(context)

Lib/api.py

@@ -1,5 +1,18 @@
 import datetime
+import ipaddress
+import json
+import random
+import re
+import shlex
+import socket
+import string
+import subprocess
 import time
+import uuid
+from urllib.parse import urlparse
+
+import dns.resolver
+import tldextract
 
 
 def timestamp_to_string(timestamp, format_str: str = "%Y-%m-%d %H:%M:%S") -> str:
@@ -40,6 +53,17 @@ def string_to_timestamp(time_string: str, format_str: str = "%Y-%m-%dT%H:%M:%S")
     return int(dt_object.timestamp())
 
 
+def string_to_string_time(time_string: str, from_format: str, to_format: str) -> str:
+    """
+    time_string = "2023-01-01 00:00:00"
+
+    converted_time = string_to_string_time(time_string, "%Y-%m-%d %H:%M:%S", "%Y/%m/%d %I:%M %p")
+    print(f"转换后的时间字符串: {converted_time}")
+    """
+    dt_object = datetime.datetime.strptime(time_string, from_format)
+    return dt_object.strftime(to_format)
+
+
 def get_current_timestamp() -> int:
     """
     current_ts = get_current_timestamp()
@@ -60,3 +84,243 @@ def get_current_time_string(format_str: str = "%Y-%m-%dT%H:%M:%SZ") -> str:
     print(f"当前时间字符串（自定义格式）: {current_date_str}")
     """
     return datetime.datetime.now().strftime(format_str)
+
+
+def exec_system(cmd, **kwargs):
+    cmd = " ".join(cmd)
+    timeout = 4 * 60 * 60
+
+    if kwargs.get('timeout'):
+        timeout = kwargs['timeout']
+        kwargs.pop('timeout')
+
+    completed = subprocess.run(shlex.split(cmd), timeout=timeout, check=False, close_fds=True, **kwargs)
+
+    return completed
+
+
+def random_str(len):
+    value = ''.join(random.sample(string.ascii_letters + string.digits, len))
+    return value
+
+
+def random_str_no_num(len):
+    value = ''.join(random.sample(string.ascii_letters, len))
+    return value
+
+
+def random_int(num):
+    """生成随机字符串"""
+    return random.randint(1, num)
+
+
+def is_json(data):
+    try:
+        json.loads(data)
+        return True
+    except Exception as E:
+        return False
+
+
+def is_ipaddress(ip_str):
+    try:
+        ip = ipaddress.IPv4Address(ip_str)
+        return True
+    except Exception as E:
+        return False
+
+
+def is_domain(url):
+    regex = r"^([a-zA-Z]+:\/\/)?([\da-zA-Z\.-]+)\.([a-zA-Z]{2,6})([\/\w \.-]*)*\/?$"
+    return True if re.match(regex, url) else False
+
+
+def is_root_domain(domain):
+    ext = tldextract.extract(domain)
+    return ext.fqdn == domain and not ext.subdomain
+
+
+def get_one_uuid_str():
+    uuid_str = str(uuid.uuid1()).replace('-', "")[0:16]
+    return uuid_str
+
+
+def data_return(code=500, data=None,
+                msg_zh="服务器发生错误，请检查服务器",
+                msg_en="An error occurred on the server, please check the server."):
+    return {'code': code, 'data': data, 'msg_zh': msg_zh, "msg_en": msg_en}
+
+
+class UnicodeEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, bytes):
+            return obj.decode(encoding='utf-8', errors="ignore").encode(encoding='utf-8', errors="ignore")
+        elif isinstance(obj, str):
+            return obj.encode(encoding='utf-8', errors="ignore").decode(encoding='utf-8', errors="ignore")
+        return json.JSONEncoder.default(self, obj)
+
+
+class UnicodeDecoder(json.JSONDecoder):
+    def decode(self, s):
+        s = s.encode(encoding='utf-8', errors="ignore").decode(encoding='utf-8', errors="ignore")
+        return super().decode(s)
+
+
+def u_json_dumps(data):
+    return json.dumps(data, cls=UnicodeEncoder)
+
+
+def u_json_loads(data):
+    return json.loads(data, cls=UnicodeDecoder)
+
+
+def dqtoi(dq):
+    """将字符串ip地址转换为int数字."""
+    octets = dq.split(".")
+    if len(octets) != 4:
+        raise ValueError
+    for octet in octets:
+        if int(octet) > 255:
+            raise ValueError
+    return (int(octets[0]) << 24) + \
+        (int(octets[1]) << 16) + \
+        (int(octets[2]) << 8) + \
+        (int(octets[3]))
+
+
+def str_to_ips(ipstr):
+    """字符串转ip地址列表"""
+    iplist = []
+    lines = ipstr.split(",")
+    for raw in lines:
+        if '/' in raw:
+            addr, mask = raw.split('/')
+            mask = int(mask)
+
+            bin_addr = ''.join([(8 - len(bin(int(i))[2:])) * '0' + bin(int(i))[2:] for i in addr.split('.')])
+            start = bin_addr[:mask] + (32 - mask) * '0'
+            end = bin_addr[:mask] + (32 - mask) * '1'
+            bin_addrs = [(32 - len(bin(int(i))[2:])) * '0' + bin(i)[2:] for i in
+                         range(int(start, 2), int(end, 2) + 1)]
+
+            dec_addrs = ['.'.join([str(int(bin_addr[8 * i:8 * (i + 1)], 2)) for i in range(0, 4)]) for bin_addr in
+                         bin_addrs]
+
+            iplist.extend(dec_addrs)
+
+        elif '-' in raw:
+            addr, end = raw.split('-')
+            end = int(end)
+            start = int(addr.split('.')[3])
+            prefix = '.'.join(addr.split('.')[:-1])
+            addrs = [prefix + '.' + str(i) for i in range(start, end + 1)]
+            iplist.extend(addrs)
+            return addrs
+        else:
+            iplist.extend([raw])
+    return iplist
+
+
+# 定义协议及其默认端口号
+DEFAULT_PORTS = {
+    'http': 80,
+    'https': 443,
+    'ftp': 21,
+    'ssh': 22,
+    'telnet': 23,
+    'smtp': 25,
+    'redis': 6379,
+    # 你可以继续添加更多协议及其默认端口号
+}
+
+
+def parse_url_simple(url):
+    parsed_url = urlparse(url)
+    scheme = parsed_url.scheme
+    # host = parsed_url.netloc
+    host = parsed_url.hostname
+    port = parsed_url.port or DEFAULT_PORTS.get(scheme, None)
+
+    return scheme, host, port
+
+
+def clean_record(ipdomain_port_list):
+    new_list = []
+    for item in ipdomain_port_list:
+        ipdomain = item[0]
+        port = item[1]
+        new_list.append({"ipdomain": ipdomain, "port": port})
+
+    return new_list
+
+
+def get_list_common(list1, list2):
+    # list1 = [{'name': 'a', 'age': 20}, {'name': 'b', 'age': 30}, {'name': 'c', 'age': 25}]
+    # list2 = [{'name': 'b', 'age': 30}, {'name': 'c', 'age': 25}, {'name': 'd', 'age': 35}]
+
+    intersect = [i for i in set(list1) & set(list2)]
+    return intersect
+
+
+def get_list_diff(list1, list2):
+    # list1 = [{'name': 'a', 'age': 20}, {'name': 'b', 'age': 30}, {'name': 'c', 'age': 25}]
+    # list2 = [{'name': 'b', 'age': 30}, {'name': 'c', 'age': 25}, {'name': 'd', 'age': 35}]
+    list1 = list(list1)
+    list2 = list(list2)
+    for one in list2:
+        if one in list1:
+            list1.remove(one)
+    return list1
+
+
+def is_ipaddress_port_in_use(ip, port):
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        try:
+            s.bind((ip, port))
+        except socket.error as e:
+            if e.errno == 98:  # 地址已在使用
+                return True
+            else:
+                raise e
+        else:
+            return False
+
+
+def get_dns_cname(domain):
+    try:
+        # 创建一个DNS解析器
+        resolver = dns.resolver.Resolver()
+
+        # 查询CNAME记录
+        cname = resolver.resolve(domain, 'CNAME')
+
+        # 返回CNAME记录列表
+        return [cname_record.to_text() for cname_record in cname]
+    except dns.resolver.NXDOMAIN:
+        print(f"Domain {domain} does not exist.")
+    except dns.resolver.NoAnswer:
+        print(f"No CNAME record found for {domain}.")
+    except Exception as e:
+        print(f"An error occurred: {e}")
+
+    return []
+
+
+def get_dns_a(domain):
+    try:
+        # 创建一个DNS解析器
+        resolver = dns.resolver.Resolver()
+
+        # 查询CNAME记录
+        A = resolver.resolve(domain, "A")
+
+        # 返回CNAME记录列表
+        return [a.to_text() for a in A]
+    except dns.resolver.NXDOMAIN:
+        print(f"Domain {domain} does not exist.")
+    except dns.resolver.NoAnswer:
+        print(f"No CNAME record found for {domain}.")
+    except Exception as e:
+        print(f"An error occurred: {e}")
+
+    return []

Lib/baseview.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+# @File  : baseview.py
+# @Date  : 2021/2/25
+# @Desc  :
+from rest_framework.generics import UpdateAPIView, DestroyAPIView
+from rest_framework.serializers import Serializer
+from rest_framework.viewsets import ModelViewSet
+
+
+class FakeSerializer(Serializer):
+    pass
+
+
+class BaseView(ModelViewSet, UpdateAPIView, DestroyAPIView):
+    queryset = None  # 设置类的queryset
+    serializer_class = FakeSerializer  # 设置类的serializer_class

Lib/configs.py

@@ -4,3 +4,89 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 MODULE_DATA_DIR = os.path.join(BASE_DIR, 'MODULES_DATA')
 REDIS_CONSUMER_GROUP = 'AI_SOC_FRAMEWORK_GROUP'
 REDIS_CONSUMER_NAME = 'AI_SOC_FRAMEWORK_CONSUMER_0'
+
+CODE_MSG_ZH = {
+    200: '服务器成功返回请求的数据',
+    201: '新建或修改数据成功',
+    202: '一个请求已经进入后台排队（异步任务）',
+    204: '删除数据成功',
+    400: '发出的请求有错误，服务器没有进行新建或修改数据的操作',
+    401: '用户没有权限（令牌、用户名、密码错误）',
+    403: '用户得到授权，但是访问是被禁止的',
+    404: '发出的请求针对的是不存在的记录，服务器没有进行操作',
+    405: '发送的新建请求失败,返回空数据',
+    406: '请求的格式不可得',
+    409: '请求的资源存在异常',
+    410: '请求的资源被永久删除，且不会再得到的',
+    422: '当创建一个对象时，发生一个验证错误',
+    500: '服务器发生错误，请检查服务器',
+    502: '网关错误',
+    503: '服务不可用，服务器暂时过载或维护',
+    504: '网关超时',
+
+    # 自定义的错误码
+    505: "MSFRPC调用失败",
+}
+
+CODE_MSG_EN = {
+    200: "The server successfully returned the requested data. ",
+    201: "New or modified data succeeded. ",
+    202: "A request has entered the background queue (asynchronous task). ",
+    204: "Data deleted successfully. ",
+    400: "There was an error in the request. The server did not create or modify the data. ",
+    401: "The user does not have permission (wrong token, user name, password). ",
+    403: "The user is authorized, but access is forbidden. ",
+    404: "The request is for a non-existent record, and the server has not operated. ",
+    405: "The request method is not allowed. ",
+    406: "The format of the request is not available. ",
+    410: "The requested resource has been permanently deleted and will no longer be available. ",
+    422: "A validation error occurred while creating an object. ",
+    500: "An error occurred on the server, please check the server. ",
+    502: "Gateway error. ",
+    503: "The service is not available. The server is temporarily overloaded or maintained. ",
+    504: "Gateway timed out. ",
+
+    # 自定义的错误码
+    505: "MSFRPC call failed",
+}
+
+BASEAUTH_MSG_ZH = {
+    201: '登录成功',
+
+    301: '登录失败,密码错误',
+    302: '配置错误,VIPER不允许使用diypassword作为密码!',
+    303: 'Viper被暴力破解,请修改密码后登录',
+}
+BASEAUTH_MSG_EN = {
+    201: 'Login successful',
+
+    301: 'Login failed,password error',
+    302: 'Configuration error, VIPER does not allow diypassword as a password!',
+    303: 'Viper has been brute force attack,please change password',
+}
+
+Empty_MSG = {
+    201: "",
+    202: "",
+    203: "",
+    204: "",
+    205: "",
+    206: "",
+
+    301: "",
+    302: "",
+    303: "",
+    304: "",
+    305: "",
+    306: "",
+}
+
+# token超时时间
+EXPIRE_MINUTES = 24 * 60
+
+# 静态文件目录
+STATIC_STORE_PATH = "STATICFILES/STATIC/"
+
+# lang
+CN = "zh-CN"
+EN = "en-US"

Lib/customexception.py

@@ -0,0 +1,34 @@
+from Lib.api import data_return
+from Lib.log import logger
+
+
+# https://www.cnblogs.com/KongHuZi/p/13696504.html#_lab2_3_2
+def views_except_handler(func):
+    def wrapper(*args, **kwargs):
+        try:
+            result = func(*args, **kwargs)
+            return result
+        except CustomException as E:
+            context = data_return(E.code, {}, E.msg_zh, E.msg_en)
+            return context
+        except Exception as E:
+            logger.exception(E)
+            context = data_return(500, {}, str(E), str(E))
+            return context
+
+    return wrapper
+
+
+class CustomException(Exception):
+    def __init__(self, msg_zh="", msg_en="", code=300, ):
+        self.code = code
+        self.msg_zh = msg_zh
+        self.msg_en = msg_en
+        super().__init__(f"{self.code}-{self.msg_zh}-{self.msg_en}")
+
+
+class LLMModuleException(Exception):
+    def __init__(self, msg_zh="", msg_en=""):
+        self.msg_zh = msg_zh
+        self.msg_en = msg_en
+        super().__init__(f"{self.msg_zh}-{self.msg_en}")

Lib/engine.py

@@ -3,7 +3,6 @@ import os
 import threading
 import time
 
-from Core.module_loader import start_watching
 from Lib.log import logger
 
 
@@ -11,22 +10,16 @@ class Engine:
     def __init__(self):
         self.modules = {}
         self.modules_dir = "MODULES"
-        self.observer = None
 
     def start(self):
         if not os.path.isdir(self.modules_dir):
             os.makedirs(self.modules_dir)
 
         self._load_initial_modules()
-        self.observer = start_watching(self, self.modules_dir)
 
         logger.info("Engine started successfully, beginning module monitoring")
 
     def stop(self):
-        if self.observer:
-            self.observer.stop()
-            self.observer.join()
-
         for module_name in list(self.modules.keys()):
             self.unload_module(module_name)
         logger.info("All modules have been stopped")

Lib/log.py

@@ -1,23 +1,3 @@
 import logging
-import sys
 
-
-def setup_logging(log_file='ai-soc-framework.log'):
-    """
-    Configures the root logger for the entire application.
-    - Logs to both a file and the console.
-    - Sets a standardized format for log messages.
-    """
-    logging.basicConfig(
-        level=logging.INFO,
-        format='%(levelname)s - %(asctime)s - [%(name)s] - %(message)s',
-        datefmt='%Y-%m-%d %H:%M:%S',
-        handlers=[
-            logging.FileHandler(log_file),
-            logging.StreamHandler(sys.stdout)
-        ]
-    )
-
-
-setup_logging()
 logger = logging.getLogger("ai-soc-framework")

Lib/montior.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+# @File  : montior.py
+# @Date  : 2021/2/25
+# @Desc  :
+
+
+from apscheduler.schedulers.background import BackgroundScheduler
+
+from Lib.engine import Engine
+from Lib.log import logger
+
+
+class MainMonitor(object):
+    BotScheduler: BackgroundScheduler
+    MainScheduler: BackgroundScheduler
+    HeartBeatScheduler: BackgroundScheduler
+    WebModuleScheduler: BackgroundScheduler
+    _background_threads = {}
+
+    def __init__(self):
+        pass
+
+    def start(self):
+        logger.info("后台服务启动")
+        engine = Engine()
+        engine.start()
+        logger.info("后台服务启动成功")

Lib/redis_client.py

@@ -1,9 +1,9 @@
 import redis
-from Lib.log import logger
 
 from CONFIG import (
     REDIS_HOST, REDIS_PORT, REDIS_DB, REDIS_PASSWORD
 )
+from Lib.log import logger
 
 
 class RedisClient(object):

Lib/xcache.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+# @File  : xcache.py
+# @Date  : 2021/2/25
+# @Desc  :
+
+from django.core.cache import cache
+
+from Lib.configs import EXPIRE_MINUTES
+
+
+class Xcache(object):
+    XCACHE_TOKEN = "XCACHE_TOKEN"
+
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def alive_token(token):
+        key = f"{Xcache.XCACHE_TOKEN}-{token}"
+        cache_user = cache.get(key)
+        return cache_user
+
+    @staticmethod
+    def set_token_user(token, user):
+        key = f"{Xcache.XCACHE_TOKEN}-{token}"
+        cache.set(key, user, EXPIRE_MINUTES)
+
+    @staticmethod
+    def clean_all_token():
+        re_key = f"{Xcache.XCACHE_TOKEN}-*"
+        keys = cache.keys(re_key)
+        for key in keys:
+            cache.delete(key)

MODULES/ES-Rule-21-Phishing_user_report_mail.py

@@ -0,0 +1,233 @@
+import json
+import textwrap
+from datetime import datetime
+from typing import Optional, Union, Dict, Any
+
+from langchain_core.messages import AIMessage, HumanMessage
+from langgraph.graph import StateGraph
+from langgraph.graph.state import CompiledStateGraph
+from pydantic import BaseModel, Field
+
+from External.nocolyapi import InputAlert, common_handler
+from External.opanaiapi import OpenAIAPI
+from External.thehiveclient import TheHiveClient
+from Lib.api import string_to_string_time, get_current_time_string
+from Lib.basemodule import LanggraphModule
+from Lib.llmapi import AgentState
+from Lib.ruledefinition import RuleDefinition
+
+
+class AnalyzeResult(BaseModel):
+    """用于从文本中提取用户信息的结构"""
+    is_phishing: bool = Field(description="是否为钓鱼邮件，True或False")
+    confidence: float = Field(description="信心指数,范围0到1之间")
+    reasoning: Optional[Union[str, Dict[str, Any]]] = Field(description="推理过程", default=None)
+
+
+class Module(LanggraphModule):
+    thread_num = 2
+
+    def __init__(self):
+        super().__init__()
+        self.thehive_client = TheHiveClient()
+        self.init()
+
+    def init(self):
+        def alert_preprocess_node(state: AgentState):
+            """预处理告警数据"""
+            # 获取stream中的原始告警
+            alert = self.read_message()
+            if alert is None:
+                return
+
+            alert = json.loads(alert["_raw"])
+            # 解析数据,此处是获取Kibana获取的alert样例
+            headers = alert["headers"]
+            headers = {"From": headers["From"], "To": headers["To"], "Subject": headers["Subject"], "Date": headers["Date"],
+                       "Return-Path": headers["Return-Path"],
+                       "Authentication-Results": headers["Authentication-Results"]}
+            alert["headers"] = headers
+
+            state.alert_raw = alert
+            return state
+
+        # 定义node
+        def alert_analyze_node(state: AgentState):
+            """AI分析告警数据"""
+
+            # 加载system prompt
+            system_prompt_template = self.load_system_prompt_template(f"senior_phishing_expert")
+
+            # 演示如何生成动态提示词
+            current_date = datetime.now().strftime("%Y-%m-%d")
+            system_message = system_prompt_template.format(current_date=current_date)
+
+            # 构建few-shot示例
+            few_shot_examples = [
+                HumanMessage(
+                    content=json.dumps({
+                        "headers": {
+                            "From": "\"Wang Lei, Project Manager\" <lei.wang@example-corp.com>",
+                            "To": "\"Li Na, Marketing Department\" <na.li@example-corp.com>",
+                            "Subject": "Project Alpha Weekly Status Report",
+                            "Date": "Tue, 2 Sep 2025 10:15:00 +0800",
+                            "Return-Path": "lei.wang@example-corp.com",
+                            "Authentication-Results": "mx.example-corp.com; spf=pass smtp.mail=lei.wang@example-corp.com;"
+                        },
+                        "body": {
+                            "plain_text": "Hi Li Na,\n\nPlease find attached the weekly status report for Project Alpha.\n\nThis week, we have completed the initial design phase and are on track to begin development next Monday as planned. Please review the attached document and let me know if you have any feedback before our sync-up meeting on Wednesday.\n\nThanks,\n\nBest Regards\nWang Lei / 王雷\nProject Manager / 项目经理\nTechnology Department / 技术部\nExample Corporation / 示例公司\nMobile: +86 13800138000\nEmail / 邮箱: lei.wang@example-corp.com\n",
+                            "html": ""
+                        },
+                        "attachments": [
+                            {
+                                "filename": "Project_Alpha_Weekly_Report_W35.pdf",
+                                "filepath": "attachments/Project_Alpha_Weekly_Report_W35.pdf",
+                                "content_type": "application/pdf"
+                            }
+                        ]
+                    })
+                ),
+                AIMessage(
+                    content=str(AnalyzeResult(is_phishing=False, confidence=0.95,
+                                              reasoning="The email is from a known colleague within the same organization, discussing a legitimate project.").model_dump())
+                ),
+                HumanMessage(
+                    content=json.dumps({
+                        "headers": {
+                            "From": "\"Microsoft Support\" <support-noreply@microsft.com>",
+                            "To": "\"Valued Customer\" <user@example.com>",
+                            "Subject": "紧急：您的账户已被暂停，需要立即验证 Urgent: Your Account is Suspended, Immediate Verification Required",
+                            "Date": "Tue, 2 Sep 2025 14:30:10 +0800",
+                            "Return-Path": "<bounce-scam@phish-delivery.net>",
+                            "Authentication-Results": "mx.example.com; spf=fail smtp.mail=support-noreply@microsft.com; dkim=fail header.d=microsft.com; dmarc=fail (p=REJECT sp=REJECT) header.from=microsft.com",
+                            "X-Coremail-Antispam": "1Uf129KBjvdXoW7GF18tw4xZF4xWF4rtw4kCrg_yoWfZFg_GF4DC348Wrnxtr15J398ZwnFy3ZFgrZ8CF9a9r4DZrZ8X3WkXa4kJr98K3y8C3WfJw1fXFW3ArnrZa93tF15tjkaLaAFLSUrUUUUUb8apTn2vfkv8UJUUUU8Yxn0WfASr-VFAUDa7-sFnT9fnUUvcSsGvfC2KfnxnUUI43ZEXa7IU04v35UUUUU=="
+                        },
+                        "body": {
+                            "plain_text": "尊敬的用户,\n\n我们的系统检测到您的帐户存在异常登录活动。为了保护您的安全，我们已临时暂停您的帐户。\n\n请立即点击以下链接以验证您的身份并恢复您的帐户访问权限：\n\nhttps://login.microsoftonline.com/common/oauth2/v2.0/authorize?client_id=... (请注意，这只是显示文本，实际链接是恶意的)\n\n如果您不在24小时内完成验证，您的帐户将被永久锁定。\n\n感谢您的合作。\n\n微软安全团队\n\n---\n\nDear User,\n\nOur system has detected unusual sign-in activity on your account. For your security, we have temporarily suspended your account.\n\nPlease click the link below immediately to verify your identity and restore access:\n\nhttp://secure-login-update-required.com/reset-password?user=user@example.com\n\nIf you do not verify within 24 hours, your account will be permanently locked.\n\nThank you for your cooperation.\n\nThe Microsoft Security Team",
+                            "html": "<html><head></head><body><p>尊敬的用户,</p><p>我们的系统检测到您的帐户存在异常登录活动。为了保护您的安全，我们已临时暂停您的帐户。</p><p>请立即点击以下链接以验证您的身份并恢复您的帐户访问权限：</p><p><a href='http://secure-login-update-required.com/reset-password?user=user@example.com'>https://login.microsoftonline.com/common/oauth2/v2.0/authorize?client_id=...</a></p><p>如果您不在24小时内完成验证，您的帐户将被永久锁定。</p><p>感谢您的合作。</p><p><b>微软安全团队</b></p></body></html>"
+                        },
+                        "attachments": [
+                            {
+                                "filename": "Account_Verification_Form.html",
+                                "filepath": "attachments/Account_Verification_Form.html",
+                                "content_type": "text/html"
+                            }
+                        ]
+                    })
+                ),
+                AIMessage(
+                    content=str(AnalyzeResult(is_phishing=True, confidence=0.92,
+                                              reasoning="The email contains several red flags: the sender's domain is misspelled, the Return-Path is from a suspicious domain, SPF and DKIM checks fail, and the email urges immediate action with threatening language. Additionally, the provided links do not match official Microsoft URLs.").model_dump())
+                ),
+            ]
+
+            # 构建消息列表
+            messages = [
+                system_message,
+                *few_shot_examples,
+                HumanMessage(content=json.dumps(state.alert_raw)),
+            ]
+
+            # 运行
+            openai_api = OpenAIAPI()
+
+            model_kwargs = {
+                "extra_body": {
+                    "enable_thinking": False
+                }
+            }
+
+            llm = openai_api.get_model(model_kwargs)
+            llm = llm.with_structured_output(AnalyzeResult)
+            response: AnalyzeResult = llm.invoke(messages)
+
+            state.analyze_result = response.model_dump()
+            return state
+
+        def alert_output_node(state: AgentState):
+            """处理分析结果"""
+            analyze_result: AnalyzeResult = AnalyzeResult(**state.analyze_result)
+            alert_raw = state.alert_raw
+
+            mail_to = alert_raw["headers"]["To"]
+            mail_subject = alert_raw["headers"]["Subject"]
+            mail_from = alert_raw["headers"]["From"]
+            if analyze_result.is_phishing and analyze_result.confidence > 0.8:
+                severity = "High"
+            else:
+                severity = "Info"
+
+            alert_date = string_to_string_time(alert_raw.get("@timestamp"), "%Y-%m-%dT%H:%M:%S.%fZ", "%Y-%m-%dT%H:%M:%SZ")
+            description = f"""
+                ## Analyze Result (AI)
+
+                * **confidence**: {analyze_result.confidence}
+                * **is_phishing**: <font color="green">{analyze_result.is_phishing}</font>
+                """
+            description = textwrap.dedent(description).strip()
+
+            rule_name = "用户上报的钓鱼邮件"
+            input_alert: InputAlert = {
+                "source": "Email",
+                "rule_id": self.module_name,
+                "rule_name": rule_name,
+                "name": f"用户上报的钓鱼邮件: {mail_subject}",
+                "alert_date": alert_date,
+                "create_data": get_current_time_string(),
+                "tags": ["phishing", "user-report"],
+                "severity": severity,
+                "description": description,
+                "reference": "https://your-siem-or-device-url.com/data?source=123456",
+                "summary_ai": analyze_result.reasoning,
+                "artifacts": [
+                    {
+                        "type": "mail_to",
+                        "value": mail_to,
+                        "deduplication_key": f"mail_to-{mail_to}",
+                        "enrichment": {"update_time": get_current_time_string()}  # just for test, no meaning, data should come from TI or other cmdb
+                    },
+                    {
+                        "type": "mail_subject",
+                        "value": mail_subject,
+                        "deduplication_key": f"mail_subject-{mail_subject}",
+                        "enrichment": {"update_time": get_current_time_string()}
+                    },
+                    {
+                        "type": "mail_from",
+                        "value": mail_from,
+                        "deduplication_key": f"mail_from-{mail_from}",
+                        "enrichment": {"update_time": get_current_time_string()}
+                    },
+                ],
+                "raw_log": alert_raw
+            }
+            rule = RuleDefinition(
+                rule_id=self.module_name,
+                rule_name=rule_name,
+                deduplication_fields=["mail_from"],
+                source="Email"
+            )
+            case_row_id = common_handler(input_alert, rule)
+            return state
+
+        # 编译graph
+        workflow = StateGraph(AgentState)
+
+        workflow.add_node("alert_preprocess_node", alert_preprocess_node)
+        workflow.add_node("alert_analyze_node", alert_analyze_node)
+        workflow.add_node("alert_output_node", alert_output_node)
+
+        workflow.set_entry_point("alert_preprocess_node")
+        workflow.add_edge("alert_preprocess_node", "alert_analyze_node")
+        workflow.add_edge("alert_analyze_node", "alert_output_node")
+        workflow.set_finish_point("alert_output_node")
+
+        self.graph: CompiledStateGraph = workflow.compile(checkpointer=self.get_checkpointer())
+        return True
+
+
+if __name__ == "__main__":
+    module = Module()
+    module.debug_alert_name = "ES-Rule-21-Phishing_user_report_mail"  # needed when debug module, framework will read redis stream by this name
+    module.debug_message_id = "0-0"
+    module.run()

MODULES_DATA/ES-Rule-21-Phishing_user_report_mail/senior_phishing_expert.md

@@ -0,0 +1,42 @@
+# 高级钓鱼邮件分析专家
+
+## 角色定位
+
+你是一位专业的钓鱼邮件分析专家，由先进的AI技术驱动。你擅长分析一封邮件是否为钓鱼邮件。
+
+## 目标
+
+通过分析用户提供的邮件信息来分析邮件是否为钓鱼邮件。
+
+## 所需输入
+
+JSON格式的邮件信息，包含以下字段：
+
+- 发件人邮箱地址
+- 收件人邮箱地址
+- 邮件主题
+- 邮件正文内容
+- 附件信息（如果有）
+- 链接信息（如果有）
+
+## 任务描述
+
+- 多维度分析用户提供的邮件信息
+- 确认邮件是否为钓鱼邮件
+- 提供详细的分析报告，说明判断依据
+
+## 思考步骤
+
+1. 分析发件人是否为可信来源
+2. 分析From和Reply-To地址是否匹配
+3. 检查邮件主题是否包含可疑关键词
+4. 分析邮件正文内容，寻找可疑链接或附件
+5. 检查邮件中的链接是否指向可疑网站
+6. 检查邮件中的附件是否包含恶意软件
+7. 综合以上分析，判断邮件是否为钓鱼邮件
+8. 提供详细的分析报告，说明判断依据
+
+## 注意事项
+- 今天是：{current_date}
+- 回复必须使用JSON格式
+

MODULE_DEBUG/Phishing_User_Report_Kibana_Langgraph_Thehive.py

@@ -11,7 +11,7 @@ from thehive4py.types.alert import InputAlert, OutputAlert
 
 from External.opanaiapi import OpenAIAPI
 from External.thehiveclient import TheHiveClient
-from Lib.base import LanggraphModule
+from Lib.basemodule import LanggraphModule
 from Lib.llmapi import AgentState
 
 

MODULE_DEBUG/Phishing_User_Report_Splunk_Dify_Nocodb.py

@@ -5,7 +5,7 @@ from pydantic import BaseModel, Field
 
 from External.difyclient import DifyClient
 from External.nocodbclient import NocodbClient
-from Lib.base import BaseModule
+from Lib.basemodule import BaseModule
 
 
 class AnalyzeResult(BaseModel):

README.md

@@ -7,7 +7,7 @@
 * **模块化引擎**: 动态加载和执行告警分析模块
 * **LLM集成**:
     * **LLM接口**: 包含Dify和Langgraph的集成接口及样例模块
-    * **工单接口**: 包含Thehive及Nocodb的集成接口及样例模块
+    * **工单接口**: 包含Thehive/Nocodb/nocoly的集成接口及样例模块
 * **事件驱动**: 使用 Redis Stream 作为消息总线，实现模块化告警流式处理
 
 ## 架构图

main.py

@@ -1,24 +0,0 @@
-import time
-
-from Core.engine import Engine
-from Lib.log import logger
-
-
-def main():
-    try:
-        engine = Engine()
-        engine.start()
-        while True:
-            time.sleep(1)
-    except KeyboardInterrupt:
-        logger.info("Program interrupted by user")
-    except Exception as e:
-        logger.error(f"Program error occurred: {e}")
-    finally:
-        if 'engine' in locals() and engine:
-            engine.stop()
-        logger.info("Program has been closed")
-
-
-if __name__ == '__main__':
-    main()

manage.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+import os
+import sys
+
+if __name__ == '__main__':
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ASF.settings')
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError as exc:
+        raise ImportError(
+            "Couldn't import Django. Are you sure it's installed and "
+            "available on your PYTHONPATH environment variable? Did you "
+            "forget to activate a virtual environment?"
+        ) from exc
+    execute_from_command_line(sys.argv)

requirements.txt

@@ -1,5 +1,4 @@
 pydantic
-watchdog
 redis
 langgraph
 langgraph-checkpoint-redis
@@ -8,4 +7,19 @@ langchain-community
 langchain-openai
 thehive4py
 requests
-flask
+Django
+djangorestframework
+channels
+requests
+APScheduler
+django-redis
+dnspython
+tldextract
+pycryptodome
+cryptography
+Jinja2
+dnslib
+pluginbase
+rsa
+pluginbase
+lxml