diff --git a/AGENTS/agent_knowledge.py b/AGENTS/agent_knowledge.py index 50020e7..47c1a11 100644 --- a/AGENTS/agent_knowledge.py +++ b/AGENTS/agent_knowledge.py @@ -5,10 +5,6 @@ from langchain_core.documents import Document from Lib.log import logger from PLUGINS.Embeddings.embeddings_qdrant import embedding_api_singleton_qdrant, SIRP_KNOWLEDGE_COLLECTION -from PLUGINS.Mem0.CONFIG import USE as MEM_ZERO_USE - -if MEM_ZERO_USE: - from PLUGINS.Mem0.mem_zero import mem_zero_singleton class AgentKnowledge(object): @@ -32,19 +28,6 @@ class AgentKnowledge(object): if doc.metadata["rerank_score"] >= threshold: result_all.append(doc.page_content) - if MEM_ZERO_USE: - result = mem_zero_singleton.search_mem(user_id=SIRP_KNOWLEDGE_COLLECTION, query=query, limit=3) - results = result.get("results", []) - relations = result.get("relations", []) - logger.debug(results) - logger.debug(relations) - for one_record in results: - id = one_record.get("id") - rerank_score = one_record.get("rerank_score", 0) - memory = one_record.get("memory", "") - if rerank_score >= threshold: - result_all.append(memory) - results = json.dumps(result_all, ensure_ascii=False) logger.debug(f"Knowledge search results : {results}") return results diff --git a/Docker/Neo4j/docker-compose.yml b/Docker/Neo4j/docker-compose.yml deleted file mode 100644 index 556a954..0000000 --- a/Docker/Neo4j/docker-compose.yml +++ /dev/null @@ -1,37 +0,0 @@ -services: - neo4j: - image: docker.1ms.run/neo4j:latest - #image: neo4j:latest - container_name: neo4j-server - restart: always - ports: - # HTTP (浏览器访问 7474) - - "7474:7474" - # Bolt (程序连接 7687) - - "7687:7687" - environment: - # 配置账号密码 (用户名: neo4j, 密码: neo4j-password-for-agentic-soc-platform) - - NEO4J_AUTH=neo4j/neo4j-password-for-agentic-soc-platform - - # 绑定地址到 0.0.0.0 以允许外部访问 - - NEO4J_server_default__listen__address=0.0.0.0 - - # 预装核心插件：APOC (实用函数库) 和 GDS (图算法库) - - NEO4J_PLUGINS=["apoc", "graph-data-science"] - - # APOC 权限配置：允许访问系统命令及导入导出 - - NEO4J_apoc_export_file_enabled=true - - NEO4J_apoc_import_file_enabled=true - - NEO4J_dbms_security_procedures_unrestricted=apoc.*,gds.* - - # 内存限制建议 (根据您的机器内存调整) - - NEO4J_server_memory_heap_initial__size=512M - - NEO4J_server_memory_heap_max__size=1G - - NEO4J_server_memory_pagecache_size=512M - - volumes: - # 数据持久化 - - ./neo4j/data:/data - - ./neo4j/logs:/logs - - ./neo4j/import:/var/lib/neo4j/import - - ./neo4j/plugins:/plugins \ No newline at end of file diff --git a/Lib/montior.py b/Lib/montior.py index bd01355..45689dd 100644 --- a/Lib/montior.py +++ b/Lib/montior.py @@ -6,25 +6,18 @@ import time from typing import Callable from apscheduler.schedulers.background import BackgroundScheduler -from django.contrib.auth.models import User from Lib.baseplaybook import BasePlaybook -from Lib.moduleengine import ModuleEngine from Lib.log import logger +from Lib.moduleengine import ModuleEngine from Lib.playbookloader import PlaybookLoader from Lib.threadmodulemanager import thread_module_manager from Lib.xcache import Xcache from PLUGINS.Embeddings.embeddings_qdrant import embedding_api_singleton_qdrant, SIRP_KNOWLEDGE_COLLECTION -from PLUGINS.Mem0.CONFIG import USE as MEM_ZERO_USE from PLUGINS.Redis.redis_stream_api import RedisStreamAPI from PLUGINS.SIRP.sirpapi import Playbook, Knowledge from PLUGINS.SIRP.sirpmodel import PlaybookJobStatus, KnowledgeAction, PlaybookModel -if MEM_ZERO_USE: - from PLUGINS.Mem0.mem_zero import mem_zero_singleton - -ASP_REST_API_TOKEN = "nocoly_token_for_playbook" - class MainMonitor(object): MainScheduler: BackgroundScheduler @@ -82,15 +75,6 @@ class MainMonitor(object): def start(self): logger.info("Starting background services...") - - # add api user - logger.info("Write ASP_TOKEN to cache") - api_usr = User() - api_usr.username = "api_token" - api_usr.is_active = True - - Xcache.set_token_user(ASP_REST_API_TOKEN, api_usr, None) - logger.info("Load PlaybookLoader module config") PlaybookLoader.load_all_playbook_config() @@ -159,13 +143,6 @@ class MainMonitor(object): except Exception as E: logger.exception(E) - try: - if MEM_ZERO_USE: - result = mem_zero_singleton.add_mem(user_id=SIRP_KNOWLEDGE_COLLECTION, run_id=model.rowid, content=payload_content, - metadata={"rowid": model.rowid}) - except Exception as E: - logger.exception(E) - model.action = KnowledgeAction.DONE model.using = True logger.info(f"Knowledge stored,rowid: {model.rowid}") @@ -176,12 +153,6 @@ class MainMonitor(object): except Exception as E: logger.exception(E) - try: - if MEM_ZERO_USE: - result = mem_zero_singleton.delete_mem(user_id=SIRP_KNOWLEDGE_COLLECTION, run_id=model.rowid) - except Exception as E: - logger.exception(E) - model.action = KnowledgeAction.DONE model.using = False logger.info(f"Knowledge removed,rowid: {model.rowid}") diff --git a/PLUGINS/Mem0/CONFIG.example.py b/PLUGINS/Mem0/CONFIG.example.py deleted file mode 100644 index 79c3758..0000000 --- a/PLUGINS/Mem0/CONFIG.example.py +++ /dev/null @@ -1,3 +0,0 @@ -# 是否启用记忆插件, mem0会占用较多资源,请根据实际情况选择启用与否 -# Whether to enable the memory plugin, Mem0 will consume more resources, please choose to enable or not according to the actual situation -USE = False diff --git a/PLUGINS/Mem0/__init__.py b/PLUGINS/Mem0/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/PLUGINS/Mem0/mem_zero.py b/PLUGINS/Mem0/mem_zero.py deleted file mode 100644 index 035617d..0000000 --- a/PLUGINS/Mem0/mem_zero.py +++ /dev/null @@ -1,126 +0,0 @@ -import os - -from mem0 import Memory - -from Lib.configs import BASE_DIR -from Lib.log import logger -from PLUGINS.Embeddings.CONFIG import EMBEDDINGS_SIZE -from PLUGINS.Embeddings.embeddings_qdrant import EmbeddingsAPI -from PLUGINS.LLM.llmapi import LLMAPI -from PLUGINS.Neo4j.CONFIG import NEO4J_URL, NEO4J_PASSWORD, NEO4J_USER -from PLUGINS.Qdrant.qdrant import Qdrant - - -class MemZero(object): - - def __init__(self): - # embeddings - self.embeddings_model = EmbeddingsAPI.get_dense_model() - - # llm - llm_api = LLMAPI() - self.llm_model = llm_api.get_model(tag=["fast"]) - - self.vector_store = Qdrant.get_client() - # your need to download the model from huggingface.co for the first run. - config = { - "reranker": { - "provider": "huggingface", - "config": { - # you need to use Docker/huggingface/download_model.py to download the bge-reranker-v2-m3 model first - "model": os.path.join(BASE_DIR, 'Docker', 'Huggingface', 'bge-reranker-v2-m3'), - "device": "cpu", - "local_files_only": True, - - # you can use the online model if you have GPU and internet access - # "model": "BAAI/bge-reranker-v2-m3", - # "device": "cuda", - } - }, - - "graph_store": { - "provider": "neo4j", - "config": { - "url": NEO4J_URL, - "username": NEO4J_USER, - "password": NEO4J_PASSWORD, - } - }, - "vector_store": { - "provider": "qdrant", - "config": { - "collection_name": "knowledge_mem0", - "client": self.vector_store, - "embedding_model_dims": EMBEDDINGS_SIZE, - "on_disk": True, - } - }, - "llm": { - "provider": "langchain", - "config": { - "model": self.llm_model, - } - }, - "embedder": { - "provider": "langchain", - "config": { - "model": self.embeddings_model, - } - }, - - } - - self.memory = Memory.from_config(config) - logger.info("MemZero initialized successfully.") - - def add_mem(self, user_id: str, run_id: str, content: str, metadata: dict): - """ - result = { - "results": vector_store_result, - "relations": graph_result, - } - """ - result = self.memory.add(content, user_id=user_id, run_id=run_id, metadata=metadata) - return result - - def search_mem(self, user_id: str, query: str, limit: int = 5, rerank: bool = True): - """ - result = {"results": [{"id": "...", "memory": "...", "score": 0.8, ...}],"relations":[...]} - """ - result = self.memory.siem_search_by_natural_language( - query, - user_id=user_id, - limit=limit, - rerank=rerank, - ) - return result - - def delete_mem(self, user_id: str, run_id: str): - result = self.memory.delete_all(user_id=user_id, run_id=run_id) - return result - - # def demo(self): - # conversation = [ - # {"role": "user", "content": "10.198.125.16是安全部门的扫描器,可能会产生NDR告警,直接忽略"}, - # {"role": "user", "content": "test@gmail.com是钓鱼模拟邮箱,如果用户上报的钓鱼邮件是这个邮箱,直接降低等级"}, - # ] - # - # result_add = self.memory.add(conversation, user_id="demo-user") - # print(result_add) - # print(time.time()) - # results = self.memory.search( - # "test@gmail.com需要安全部门封禁吗?", - # user_id="demo-user", - # limit=3, - # rerank=True, - # ) - # for hit in results["results"]: - # print(hit) - # print(time.time()) - # - # def delete(self): - # result_delete = self.memory.delete_all(user_id="demo-user") - # print(result_delete) - - -mem_zero_singleton = MemZero()