Feat/deployment update (#56)

* feat: update analyzer thresholds and add crawl configuration options * feat: update Helm chart version and add README for installation instructions * feat: update installation instructions in README and add Docker support * feat: update deployment manifests and configuration for improved service handling and analyzer settings * feat: add API endpoint for paginated IP retrieval and enhance dashboard visualization with category filters * feat: update configuration for Krawl service to use external config file * feat: refactor code for improved readability and consistency across multiple files * feat: remove Flake8, Pylint, and test steps from PR checks workflow
2026-01-26 12:36:22 +01:00
parent 130e81ad64
commit 8c76f6c847
20 changed files with 1025 additions and 269 deletions
--- a/src/config.py
+++ b/src/config.py
@@ -76,10 +76,10 @@ class Config:
            # Try multiple external IP detection services (fallback chain)
            ip_detection_services = [
                "https://api.ipify.org",  # Plain text response
-                "http://ident.me",         # Plain text response
-                "https://ifconfig.me",     # Plain text response
+                "http://ident.me",  # Plain text response
+                "https://ifconfig.me",  # Plain text response
            ]
-            
+
            ip = None
            for service_url in ip_detection_services:
                try:
@@ -90,7 +90,7 @@ class Config:
                            break
                except Exception:
                    continue
-            
+
            if not ip:
                get_app_logger().warning(
                    "Could not determine server IP from external services. "
--- a/src/database.py
+++ b/src/database.py
@@ -587,7 +587,9 @@ class DatabaseManager:
                    "analyzed_metrics": s.analyzed_metrics,
                    "category": s.category,
                    "manual_category": s.manual_category,
-                    "last_analysis": s.last_analysis.isoformat() if s.last_analysis else None,
+                    "last_analysis": (
+                        s.last_analysis.isoformat() if s.last_analysis else None
+                    ),
                }
                for s in stats
            ]
@@ -638,7 +640,13 @@ class DatabaseManager:
        finally:
            self.close_session()

-    def get_attackers_paginated(self, page: int = 1, page_size: int = 25, sort_by: str = "total_requests", sort_order: str = "desc") -> Dict[str, Any]:
+    def get_attackers_paginated(
+        self,
+        page: int = 1,
+        page_size: int = 25,
+        sort_by: str = "total_requests",
+        sort_order: str = "desc",
+    ) -> Dict[str, Any]:
        """
        Retrieve paginated list of attacker IPs ordered by specified field.

@@ -658,29 +666,35 @@ class DatabaseManager:
            # Validate sort parameters
            valid_sort_fields = {"total_requests", "first_seen", "last_seen"}
            sort_by = sort_by if sort_by in valid_sort_fields else "total_requests"
-            sort_order = sort_order.lower() if sort_order.lower() in {"asc", "desc"} else "desc"
+            sort_order = (
+                sort_order.lower() if sort_order.lower() in {"asc", "desc"} else "desc"
+            )

            # Get total count of attackers
            total_attackers = (
-                session.query(IpStats)
-                .filter(IpStats.category == "attacker")
-                .count()
+                session.query(IpStats).filter(IpStats.category == "attacker").count()
            )

            # Build query with sorting
            query = session.query(IpStats).filter(IpStats.category == "attacker")
-            
+
            if sort_by == "total_requests":
                query = query.order_by(
-                    IpStats.total_requests.desc() if sort_order == "desc" else IpStats.total_requests.asc()
+                    IpStats.total_requests.desc()
+                    if sort_order == "desc"
+                    else IpStats.total_requests.asc()
                )
            elif sort_by == "first_seen":
                query = query.order_by(
-                    IpStats.first_seen.desc() if sort_order == "desc" else IpStats.first_seen.asc()
+                    IpStats.first_seen.desc()
+                    if sort_order == "desc"
+                    else IpStats.first_seen.asc()
                )
            elif sort_by == "last_seen":
                query = query.order_by(
-                    IpStats.last_seen.desc() if sort_order == "desc" else IpStats.last_seen.asc()
+                    IpStats.last_seen.desc()
+                    if sort_order == "desc"
+                    else IpStats.last_seen.asc()
                )

            # Get paginated attackers
@@ -693,7 +707,9 @@ class DatabaseManager:
                    {
                        "ip": a.ip,
                        "total_requests": a.total_requests,
-                        "first_seen": a.first_seen.isoformat() if a.first_seen else None,
+                        "first_seen": (
+                            a.first_seen.isoformat() if a.first_seen else None
+                        ),
                        "last_seen": a.last_seen.isoformat() if a.last_seen else None,
                        "country_code": a.country_code,
                        "city": a.city,
@@ -716,6 +732,101 @@ class DatabaseManager:
        finally:
            self.close_session()

+    def get_all_ips_paginated(
+        self,
+        page: int = 1,
+        page_size: int = 25,
+        sort_by: str = "total_requests",
+        sort_order: str = "desc",
+        categories: Optional[List[str]] = None,
+    ) -> Dict[str, Any]:
+        """
+        Retrieve paginated list of all IPs (or filtered by categories) ordered by specified field.
+
+        Args:
+            page: Page number (1-indexed)
+            page_size: Number of results per page
+            sort_by: Field to sort by (total_requests, first_seen, last_seen)
+            sort_order: Sort order (asc or desc)
+            categories: Optional list of categories to filter by
+
+        Returns:
+            Dictionary with IPs list and pagination info
+        """
+        session = self.session
+        try:
+            offset = (page - 1) * page_size
+
+            # Validate sort parameters
+            valid_sort_fields = {"total_requests", "first_seen", "last_seen"}
+            sort_by = sort_by if sort_by in valid_sort_fields else "total_requests"
+            sort_order = (
+                sort_order.lower() if sort_order.lower() in {"asc", "desc"} else "desc"
+            )
+
+            # Build query with optional category filter
+            query = session.query(IpStats)
+            if categories:
+                query = query.filter(IpStats.category.in_(categories))
+
+            # Get total count
+            total_ips = query.count()
+
+            # Apply sorting
+            if sort_by == "total_requests":
+                query = query.order_by(
+                    IpStats.total_requests.desc()
+                    if sort_order == "desc"
+                    else IpStats.total_requests.asc()
+                )
+            elif sort_by == "first_seen":
+                query = query.order_by(
+                    IpStats.first_seen.desc()
+                    if sort_order == "desc"
+                    else IpStats.first_seen.asc()
+                )
+            elif sort_by == "last_seen":
+                query = query.order_by(
+                    IpStats.last_seen.desc()
+                    if sort_order == "desc"
+                    else IpStats.last_seen.asc()
+                )
+
+            # Get paginated IPs
+            ips = query.offset(offset).limit(page_size).all()
+
+            total_pages = (total_ips + page_size - 1) // page_size
+
+            return {
+                "ips": [
+                    {
+                        "ip": ip.ip,
+                        "total_requests": ip.total_requests,
+                        "first_seen": (
+                            ip.first_seen.isoformat() if ip.first_seen else None
+                        ),
+                        "last_seen": ip.last_seen.isoformat() if ip.last_seen else None,
+                        "country_code": ip.country_code,
+                        "city": ip.city,
+                        "asn": ip.asn,
+                        "asn_org": ip.asn_org,
+                        "reputation_score": ip.reputation_score,
+                        "reputation_source": ip.reputation_source,
+                        "category": ip.category,
+                        "category_scores": ip.category_scores or {},
+                    }
+                    for ip in ips
+                ],
+                "pagination": {
+                    "page": page,
+                    "page_size": page_size,
+                    "total": total_ips,
+                    "total_pages": total_pages,
+                },
+            }
+        finally:
+            self.close_session()
+
    def get_dashboard_counts(self) -> Dict[str, int]:
        """
        Get aggregate statistics for the dashboard (excludes local/private IPs and server IP).
@@ -728,28 +839,34 @@ class DatabaseManager:
        try:
            # Get server IP to filter it out
            from config import get_config
+
            config = get_config()
            server_ip = config.get_server_ip()
-            
+
            # Get all accesses first, then filter out local IPs and server IP
            all_accesses = session.query(AccessLog).all()
-            
+
            # Filter out local/private IPs and server IP
            public_accesses = [
-                log for log in all_accesses
-                if is_valid_public_ip(log.ip, server_ip)
+                log for log in all_accesses if is_valid_public_ip(log.ip, server_ip)
            ]
-            
+
            # Calculate counts from filtered data
            total_accesses = len(public_accesses)
            unique_ips = len(set(log.ip for log in public_accesses))
            unique_paths = len(set(log.path for log in public_accesses))
            suspicious_accesses = sum(1 for log in public_accesses if log.is_suspicious)
-            honeypot_triggered = sum(1 for log in public_accesses if log.is_honeypot_trigger)
-            honeypot_ips = len(set(log.ip for log in public_accesses if log.is_honeypot_trigger))
-            
+            honeypot_triggered = sum(
+                1 for log in public_accesses if log.is_honeypot_trigger
+            )
+            honeypot_ips = len(
+                set(log.ip for log in public_accesses if log.is_honeypot_trigger)
+            )
+
            # Count unique attackers from IpStats (matching the "Attackers by Total Requests" table)
-            unique_attackers = session.query(IpStats).filter(IpStats.category == "attacker").count()
+            unique_attackers = (
+                session.query(IpStats).filter(IpStats.category == "attacker").count()
+            )

            return {
                "total_accesses": total_accesses,
@@ -777,9 +894,10 @@ class DatabaseManager:
        try:
            # Get server IP to filter it out
            from config import get_config
+
            config = get_config()
            server_ip = config.get_server_ip()
-            
+
            results = (
                session.query(AccessLog.ip, func.count(AccessLog.id).label("count"))
                .group_by(AccessLog.ip)
@@ -862,9 +980,10 @@ class DatabaseManager:
        try:
            # Get server IP to filter it out
            from config import get_config
+
            config = get_config()
            server_ip = config.get_server_ip()
-            
+
            logs = (
                session.query(AccessLog)
                .filter(AccessLog.is_suspicious == True)
@@ -874,8 +993,7 @@ class DatabaseManager:

            # Filter out local/private IPs and server IP
            filtered_logs = [
-                log for log in logs
-                if is_valid_public_ip(log.ip, server_ip)
+                log for log in logs if is_valid_public_ip(log.ip, server_ip)
            ]

            return [
@@ -902,9 +1020,10 @@ class DatabaseManager:
        try:
            # Get server IP to filter it out
            from config import get_config
+
            config = get_config()
            server_ip = config.get_server_ip()
-            
+
            # Get all honeypot triggers grouped by IP
            results = (
                session.query(AccessLog.ip, AccessLog.path)
@@ -961,7 +1080,13 @@ class DatabaseManager:
        finally:
            self.close_session()

-    def get_honeypot_paginated(self, page: int = 1, page_size: int = 5, sort_by: str = "count", sort_order: str = "desc") -> Dict[str, Any]:
+    def get_honeypot_paginated(
+        self,
+        page: int = 1,
+        page_size: int = 5,
+        sort_by: str = "count",
+        sort_order: str = "desc",
+    ) -> Dict[str, Any]:
        """
        Retrieve paginated list of honeypot-triggered IPs with their paths.

@@ -977,6 +1102,7 @@ class DatabaseManager:
        session = self.session
        try:
            from config import get_config
+
            config = get_config()
            server_ip = config.get_server_ip()

@@ -1007,17 +1133,15 @@ class DatabaseManager:

            if sort_by == "count":
                honeypot_list.sort(
-                    key=lambda x: x["count"],
-                    reverse=(sort_order == "desc")
+                    key=lambda x: x["count"], reverse=(sort_order == "desc")
                )
            else:  # sort by ip
                honeypot_list.sort(
-                    key=lambda x: x["ip"],
-                    reverse=(sort_order == "desc")
+                    key=lambda x: x["ip"], reverse=(sort_order == "desc")
                )

            total_honeypots = len(honeypot_list)
-            paginated = honeypot_list[offset:offset + page_size]
+            paginated = honeypot_list[offset : offset + page_size]
            total_pages = (total_honeypots + page_size - 1) // page_size

            return {
@@ -1032,7 +1156,13 @@ class DatabaseManager:
        finally:
            self.close_session()

-    def get_credentials_paginated(self, page: int = 1, page_size: int = 5, sort_by: str = "timestamp", sort_order: str = "desc") -> Dict[str, Any]:
+    def get_credentials_paginated(
+        self,
+        page: int = 1,
+        page_size: int = 5,
+        sort_by: str = "timestamp",
+        sort_order: str = "desc",
+    ) -> Dict[str, Any]:
        """
        Retrieve paginated list of credential attempts.

@@ -1052,7 +1182,9 @@ class DatabaseManager:
            # Validate sort parameters
            valid_sort_fields = {"timestamp", "ip", "username"}
            sort_by = sort_by if sort_by in valid_sort_fields else "timestamp"
-            sort_order = sort_order.lower() if sort_order.lower() in {"asc", "desc"} else "desc"
+            sort_order = (
+                sort_order.lower() if sort_order.lower() in {"asc", "desc"} else "desc"
+            )

            total_credentials = session.query(CredentialAttempt).count()

@@ -1061,15 +1193,21 @@ class DatabaseManager:

            if sort_by == "timestamp":
                query = query.order_by(
-                    CredentialAttempt.timestamp.desc() if sort_order == "desc" else CredentialAttempt.timestamp.asc()
+                    CredentialAttempt.timestamp.desc()
+                    if sort_order == "desc"
+                    else CredentialAttempt.timestamp.asc()
                )
            elif sort_by == "ip":
                query = query.order_by(
-                    CredentialAttempt.ip.desc() if sort_order == "desc" else CredentialAttempt.ip.asc()
+                    CredentialAttempt.ip.desc()
+                    if sort_order == "desc"
+                    else CredentialAttempt.ip.asc()
                )
            elif sort_by == "username":
                query = query.order_by(
-                    CredentialAttempt.username.desc() if sort_order == "desc" else CredentialAttempt.username.asc()
+                    CredentialAttempt.username.desc()
+                    if sort_order == "desc"
+                    else CredentialAttempt.username.asc()
                )

            credentials = query.offset(offset).limit(page_size).all()
@@ -1096,7 +1234,13 @@ class DatabaseManager:
        finally:
            self.close_session()

-    def get_top_ips_paginated(self, page: int = 1, page_size: int = 5, sort_by: str = "count", sort_order: str = "desc") -> Dict[str, Any]:
+    def get_top_ips_paginated(
+        self,
+        page: int = 1,
+        page_size: int = 5,
+        sort_by: str = "count",
+        sort_order: str = "desc",
+    ) -> Dict[str, Any]:
        """
        Retrieve paginated list of top IP addresses by access count.

@@ -1112,6 +1256,7 @@ class DatabaseManager:
        session = self.session
        try:
            from config import get_config
+
            config = get_config()
            server_ip = config.get_server_ip()

@@ -1136,7 +1281,7 @@ class DatabaseManager:
                filtered.sort(key=lambda x: x["ip"], reverse=(sort_order == "desc"))

            total_ips = len(filtered)
-            paginated = filtered[offset:offset + page_size]
+            paginated = filtered[offset : offset + page_size]
            total_pages = (total_ips + page_size - 1) // page_size

            return {
@@ -1151,7 +1296,13 @@ class DatabaseManager:
        finally:
            self.close_session()

-    def get_top_paths_paginated(self, page: int = 1, page_size: int = 5, sort_by: str = "count", sort_order: str = "desc") -> Dict[str, Any]:
+    def get_top_paths_paginated(
+        self,
+        page: int = 1,
+        page_size: int = 5,
+        sort_by: str = "count",
+        sort_order: str = "desc",
+    ) -> Dict[str, Any]:
        """
        Retrieve paginated list of top paths by access count.

@@ -1175,18 +1326,17 @@ class DatabaseManager:
            )

            # Create list and sort
-            paths_list = [
-                {"path": row.path, "count": row.count}
-                for row in results
-            ]
+            paths_list = [{"path": row.path, "count": row.count} for row in results]

            if sort_by == "count":
-                paths_list.sort(key=lambda x: x["count"], reverse=(sort_order == "desc"))
+                paths_list.sort(
+                    key=lambda x: x["count"], reverse=(sort_order == "desc")
+                )
            else:  # sort by path
                paths_list.sort(key=lambda x: x["path"], reverse=(sort_order == "desc"))

            total_paths = len(paths_list)
-            paginated = paths_list[offset:offset + page_size]
+            paginated = paths_list[offset : offset + page_size]
            total_pages = (total_paths + page_size - 1) // page_size

            return {
@@ -1201,7 +1351,13 @@ class DatabaseManager:
        finally:
            self.close_session()

-    def get_top_user_agents_paginated(self, page: int = 1, page_size: int = 5, sort_by: str = "count", sort_order: str = "desc") -> Dict[str, Any]:
+    def get_top_user_agents_paginated(
+        self,
+        page: int = 1,
+        page_size: int = 5,
+        sort_by: str = "count",
+        sort_order: str = "desc",
+    ) -> Dict[str, Any]:
        """
        Retrieve paginated list of top user agents by access count.

@@ -1219,7 +1375,9 @@ class DatabaseManager:
            offset = (page - 1) * page_size

            results = (
-                session.query(AccessLog.user_agent, func.count(AccessLog.id).label("count"))
+                session.query(
+                    AccessLog.user_agent, func.count(AccessLog.id).label("count")
+                )
                .filter(AccessLog.user_agent.isnot(None), AccessLog.user_agent != "")
                .group_by(AccessLog.user_agent)
                .all()
@@ -1227,17 +1385,18 @@ class DatabaseManager:

            # Create list and sort
            ua_list = [
-                {"user_agent": row.user_agent, "count": row.count}
-                for row in results
+                {"user_agent": row.user_agent, "count": row.count} for row in results
            ]

            if sort_by == "count":
                ua_list.sort(key=lambda x: x["count"], reverse=(sort_order == "desc"))
            else:  # sort by user_agent
-                ua_list.sort(key=lambda x: x["user_agent"], reverse=(sort_order == "desc"))
+                ua_list.sort(
+                    key=lambda x: x["user_agent"], reverse=(sort_order == "desc")
+                )

            total_uas = len(ua_list)
-            paginated = ua_list[offset:offset + page_size]
+            paginated = ua_list[offset : offset + page_size]
            total_pages = (total_uas + page_size - 1) // page_size

            return {
@@ -1252,7 +1411,13 @@ class DatabaseManager:
        finally:
            self.close_session()

-    def get_attack_types_paginated(self, page: int = 1, page_size: int = 5, sort_by: str = "timestamp", sort_order: str = "desc") -> Dict[str, Any]:
+    def get_attack_types_paginated(
+        self,
+        page: int = 1,
+        page_size: int = 5,
+        sort_by: str = "timestamp",
+        sort_order: str = "desc",
+    ) -> Dict[str, Any]:
        """
        Retrieve paginated list of detected attack types with access logs.

@@ -1272,17 +1437,18 @@ class DatabaseManager:
            # Validate sort parameters
            valid_sort_fields = {"timestamp", "ip", "attack_type"}
            sort_by = sort_by if sort_by in valid_sort_fields else "timestamp"
-            sort_order = sort_order.lower() if sort_order.lower() in {"asc", "desc"} else "desc"
+            sort_order = (
+                sort_order.lower() if sort_order.lower() in {"asc", "desc"} else "desc"
+            )

            # Get all access logs with attack detections
-            query = (
-                session.query(AccessLog)
-                .join(AttackDetection)
-            )
+            query = session.query(AccessLog).join(AttackDetection)

            if sort_by == "timestamp":
                query = query.order_by(
-                    AccessLog.timestamp.desc() if sort_order == "desc" else AccessLog.timestamp.asc()
+                    AccessLog.timestamp.desc()
+                    if sort_order == "desc"
+                    else AccessLog.timestamp.asc()
                )
            elif sort_by == "ip":
                query = query.order_by(
@@ -1307,11 +1473,11 @@ class DatabaseManager:
            if sort_by == "attack_type":
                attack_list.sort(
                    key=lambda x: x["attack_types"][0] if x["attack_types"] else "",
-                    reverse=(sort_order == "desc")
+                    reverse=(sort_order == "desc"),
                )

            total_attacks = len(attack_list)
-            paginated = attack_list[offset:offset + page_size]
+            paginated = attack_list[offset : offset + page_size]
            total_pages = (total_attacks + page_size - 1) // page_size

            return {
--- a/src/handler.py
+++ b/src/handler.py
@@ -511,7 +511,10 @@ class Handler(BaseHTTPRequestHandler):
            return

        # API endpoint for fetching all IP statistics
-        if self.config.dashboard_secret_path and self.path == f"{self.config.dashboard_secret_path}/api/all-ip-stats":
+        if (
+            self.config.dashboard_secret_path
+            and self.path == f"{self.config.dashboard_secret_path}/api/all-ip-stats"
+        ):
            self.send_response(200)
            self.send_header("Content-type", "application/json")
            self.send_header("Access-Control-Allow-Origin", "*")
@@ -554,7 +557,7 @@ class Handler(BaseHTTPRequestHandler):
                from urllib.parse import urlparse, parse_qs

                db = get_database()
-                
+
                # Parse query parameters
                parsed_url = urlparse(self.path)
                query_params = parse_qs(parsed_url.query)
@@ -567,7 +570,12 @@ class Handler(BaseHTTPRequestHandler):
                page = max(1, page)
                page_size = min(max(1, page_size), 100)  # Max 100 per page

-                result = db.get_attackers_paginated(page=page, page_size=page_size, sort_by=sort_by, sort_order=sort_order)
+                result = db.get_attackers_paginated(
+                    page=page,
+                    page_size=page_size,
+                    sort_by=sort_by,
+                    sort_order=sort_order,
+                )
                self.wfile.write(json.dumps(result).encode())
            except BrokenPipeError:
                pass
@@ -576,6 +584,52 @@ class Handler(BaseHTTPRequestHandler):
                self.wfile.write(json.dumps({"error": str(e)}).encode())
            return

+        # API endpoint for fetching all IPs (all categories)
+        if self.config.dashboard_secret_path and self.path.startswith(
+            f"{self.config.dashboard_secret_path}/api/all-ips"
+        ):
+            self.send_response(200)
+            self.send_header("Content-type", "application/json")
+            self.send_header("Access-Control-Allow-Origin", "*")
+            self.send_header(
+                "Cache-Control", "no-store, no-cache, must-revalidate, max-age=0"
+            )
+            self.send_header("Pragma", "no-cache")
+            self.send_header("Expires", "0")
+            self.end_headers()
+            try:
+                from database import get_database
+                import json
+                from urllib.parse import urlparse, parse_qs
+
+                db = get_database()
+
+                # Parse query parameters
+                parsed_url = urlparse(self.path)
+                query_params = parse_qs(parsed_url.query)
+                page = int(query_params.get("page", ["1"])[0])
+                page_size = int(query_params.get("page_size", ["25"])[0])
+                sort_by = query_params.get("sort_by", ["total_requests"])[0]
+                sort_order = query_params.get("sort_order", ["desc"])[0]
+
+                # Ensure valid parameters
+                page = max(1, page)
+                page_size = min(max(1, page_size), 100)  # Max 100 per page
+
+                result = db.get_all_ips_paginated(
+                    page=page,
+                    page_size=page_size,
+                    sort_by=sort_by,
+                    sort_order=sort_order,
+                )
+                self.wfile.write(json.dumps(result).encode())
+            except BrokenPipeError:
+                pass
+            except Exception as e:
+                self.app_logger.error(f"Error fetching all IPs: {e}")
+                self.wfile.write(json.dumps({"error": str(e)}).encode())
+            return
+
        # API endpoint for fetching IP stats
        if self.config.dashboard_secret_path and self.path.startswith(
            f"{self.config.dashboard_secret_path}/api/ip-stats/"
@@ -639,7 +693,12 @@ class Handler(BaseHTTPRequestHandler):
                page = max(1, page)
                page_size = min(max(1, page_size), 100)

-                result = db.get_honeypot_paginated(page=page, page_size=page_size, sort_by=sort_by, sort_order=sort_order)
+                result = db.get_honeypot_paginated(
+                    page=page,
+                    page_size=page_size,
+                    sort_by=sort_by,
+                    sort_order=sort_order,
+                )
                self.wfile.write(json.dumps(result).encode())
            except BrokenPipeError:
                pass
@@ -677,7 +736,12 @@ class Handler(BaseHTTPRequestHandler):
                page = max(1, page)
                page_size = min(max(1, page_size), 100)

-                result = db.get_credentials_paginated(page=page, page_size=page_size, sort_by=sort_by, sort_order=sort_order)
+                result = db.get_credentials_paginated(
+                    page=page,
+                    page_size=page_size,
+                    sort_by=sort_by,
+                    sort_order=sort_order,
+                )
                self.wfile.write(json.dumps(result).encode())
            except BrokenPipeError:
                pass
@@ -715,7 +779,12 @@ class Handler(BaseHTTPRequestHandler):
                page = max(1, page)
                page_size = min(max(1, page_size), 100)

-                result = db.get_top_ips_paginated(page=page, page_size=page_size, sort_by=sort_by, sort_order=sort_order)
+                result = db.get_top_ips_paginated(
+                    page=page,
+                    page_size=page_size,
+                    sort_by=sort_by,
+                    sort_order=sort_order,
+                )
                self.wfile.write(json.dumps(result).encode())
            except BrokenPipeError:
                pass
@@ -753,7 +822,12 @@ class Handler(BaseHTTPRequestHandler):
                page = max(1, page)
                page_size = min(max(1, page_size), 100)

-                result = db.get_top_paths_paginated(page=page, page_size=page_size, sort_by=sort_by, sort_order=sort_order)
+                result = db.get_top_paths_paginated(
+                    page=page,
+                    page_size=page_size,
+                    sort_by=sort_by,
+                    sort_order=sort_order,
+                )
                self.wfile.write(json.dumps(result).encode())
            except BrokenPipeError:
                pass
@@ -791,7 +865,12 @@ class Handler(BaseHTTPRequestHandler):
                page = max(1, page)
                page_size = min(max(1, page_size), 100)

-                result = db.get_top_user_agents_paginated(page=page, page_size=page_size, sort_by=sort_by, sort_order=sort_order)
+                result = db.get_top_user_agents_paginated(
+                    page=page,
+                    page_size=page_size,
+                    sort_by=sort_by,
+                    sort_order=sort_order,
+                )
                self.wfile.write(json.dumps(result).encode())
            except BrokenPipeError:
                pass
@@ -829,7 +908,12 @@ class Handler(BaseHTTPRequestHandler):
                page = max(1, page)
                page_size = min(max(1, page_size), 100)

-                result = db.get_attack_types_paginated(page=page, page_size=page_size, sort_by=sort_by, sort_order=sort_order)
+                result = db.get_attack_types_paginated(
+                    page=page,
+                    page_size=page_size,
+                    sort_by=sort_by,
+                    sort_order=sort_order,
+                )
                self.wfile.write(json.dumps(result).encode())
            except BrokenPipeError:
                pass
--- a/src/ip_utils.py
+++ b/src/ip_utils.py
@@ -12,7 +12,7 @@ from typing import Optional
 def is_local_or_private_ip(ip_str: str) -> bool:
    """
    Check if an IP address is local, private, or reserved.
-    
+
    Filters out:
    - 127.0.0.1 (localhost)
    - 127.0.0.0/8 (loopback)
@@ -22,10 +22,10 @@ def is_local_or_private_ip(ip_str: str) -> bool:
    - 0.0.0.0/8 (this network)
    - ::1 (IPv6 localhost)
    - ::ffff:127.0.0.0/104 (IPv6-mapped IPv4 loopback)
-    
+
    Args:
        ip_str: IP address string
-        
+
    Returns:
        True if IP is local/private/reserved, False if it's public
    """
@@ -46,15 +46,15 @@ def is_local_or_private_ip(ip_str: str) -> bool:
 def is_valid_public_ip(ip: str, server_ip: Optional[str] = None) -> bool:
    """
    Check if an IP is public and not the server's own IP.
-    
+
    Returns True only if:
    - IP is not in local/private ranges AND
    - IP is not the server's own public IP (if server_ip provided)
-    
+
    Args:
        ip: IP address string to check
        server_ip: Server's public IP (optional). If provided, filters out this IP too.
-        
+
    Returns:
        True if IP is a valid public IP to track, False otherwise
    """
--- a/src/tasks/memory_cleanup.py
+++ b/src/tasks/memory_cleanup.py
@@ -45,8 +45,13 @@ def main():
        stats_after = Handler.tracker.get_memory_stats()

        # Log changes
-        access_log_reduced = stats_before["access_log_size"] - stats_after["access_log_size"]
-        cred_reduced = stats_before["credential_attempts_size"] - stats_after["credential_attempts_size"]
+        access_log_reduced = (
+            stats_before["access_log_size"] - stats_after["access_log_size"]
+        )
+        cred_reduced = (
+            stats_before["credential_attempts_size"]
+            - stats_after["credential_attempts_size"]
+        )

        if access_log_reduced > 0 or cred_reduced > 0:
            app_logger.info(
--- a/src/tasks/top_attacking_ips.py
+++ b/src/tasks/top_attacking_ips.py
@@ -71,11 +71,8 @@ def main():
        # Filter out local/private IPs and the server's own IP
        config = get_config()
        server_ip = config.get_server_ip()
-        
-        public_ips = [
-            ip for (ip,) in results
-            if is_valid_public_ip(ip, server_ip)
-        ]
+
+        public_ips = [ip for (ip,) in results if is_valid_public_ip(ip, server_ip)]

        # Ensure exports directory exists
        os.makedirs(EXPORTS_DIR, exist_ok=True)
--- a/src/templates/dashboard_template.py
+++ b/src/templates/dashboard_template.py
@@ -548,10 +548,7 @@ def generate_dashboard(stats: dict, dashboard_path: str = "") -> str:
            background: #161b22;
            border-top: 6px solid #30363d;
        }}
-        .attacker-marker {{
-            width: 20px;
-            height: 20px;
-            background: #f85149;
+        .ip-marker {{
            border: 2px solid #fff;
            border-radius: 50%;
            display: flex;
@@ -560,20 +557,27 @@ def generate_dashboard(stats: dict, dashboard_path: str = "") -> str:
            font-size: 10px;
            font-weight: bold;
            color: white;
-            box-shadow: 0 0 8px rgba(248, 81, 73, 0.8), inset 0 0 4px rgba(248, 81, 73, 0.5);
            cursor: pointer;
        }}
-        .attacker-marker-cluster {{
-            background: #f85149 !important;
-            border: 2px solid #fff !important;
-            background-clip: padding-box !important;
+        .marker-attacker {{
+            background: #f85149;
+            box-shadow: 0 0 8px rgba(248, 81, 73, 0.8), inset 0 0 4px rgba(248, 81, 73, 0.5);
        }}
-        .attacker-marker-cluster div {{
-            background: #f85149 !important;
+        .marker-bad_crawler {{
+            background: #f0883e;
+            box-shadow: 0 0 8px rgba(240, 136, 62, 0.8), inset 0 0 4px rgba(240, 136, 62, 0.5);
        }}
-        .attacker-marker-cluster span {{
-            color: white !important;
-            font-weight: bold !important;
+        .marker-good_crawler {{
+            background: #3fb950;
+            box-shadow: 0 0 8px rgba(63, 185, 80, 0.8), inset 0 0 4px rgba(63, 185, 80, 0.5);
+        }}
+        .marker-regular_user {{
+            background: #58a6ff;
+            box-shadow: 0 0 8px rgba(88, 166, 255, 0.8), inset 0 0 4px rgba(88, 166, 255, 0.5);
+        }}
+        .marker-unknown {{
+            background: #8b949e;
+            box-shadow: 0 0 8px rgba(139, 148, 158, 0.8), inset 0 0 4px rgba(139, 148, 158, 0.5);
        }}
        .leaflet-bottom.leaflet-right {{
            display: none !important;
@@ -734,7 +738,31 @@ def generate_dashboard(stats: dict, dashboard_path: str = "") -> str:

        <div id="ip-stats" class="tab-content">
            <div class="table-container" style="margin-bottom: 30px;">
-                <h2>Attacker Origins Map</h2>
+                <div style="display: flex; align-items: center; justify-content: space-between; margin-bottom: 16px;">
+                    <h2 style="margin: 0;">IP Origins Map</h2>
+                    <div style="display: flex; gap: 16px; align-items: center; flex-wrap: wrap;">
+                        <label style="display: flex; align-items: center; gap: 6px; cursor: pointer; color: #c9d1d9; font-size: 13px;">
+                            <input type="checkbox" id="filter-attacker" checked onchange="updateMapFilters()" style="cursor: pointer;">
+                            <span style="color: #f85149;">● Attackers</span>
+                        </label>
+                        <label style="display: flex; align-items: center; gap: 6px; cursor: pointer; color: #c9d1d9; font-size: 13px;">
+                            <input type="checkbox" id="filter-bad-crawler" checked onchange="updateMapFilters()" style="cursor: pointer;">
+                            <span style="color: #f0883e;">● Bad Crawlers</span>
+                        </label>
+                        <label style="display: flex; align-items: center; gap: 6px; cursor: pointer; color: #c9d1d9; font-size: 13px;">
+                            <input type="checkbox" id="filter-good-crawler" checked onchange="updateMapFilters()" style="cursor: pointer;">
+                            <span style="color: #3fb950;">● Good Crawlers</span>
+                        </label>
+                        <label style="display: flex; align-items: center; gap: 6px; cursor: pointer; color: #c9d1d9; font-size: 13px;">
+                            <input type="checkbox" id="filter-regular-user" checked onchange="updateMapFilters()" style="cursor: pointer;">
+                            <span style="color: #58a6ff;">● Regular Users</span>
+                        </label>
+                        <label style="display: flex; align-items: center; gap: 6px; cursor: pointer; color: #c9d1d9; font-size: 13px;">
+                            <input type="checkbox" id="filter-unknown" checked onchange="updateMapFilters()" style="cursor: pointer;">
+                            <span style="color: #8b949e;">● Unknown</span>
+                        </label>
+                    </div>
+                </div>
                <div id="attacker-map" style="height: 500px; border-radius: 6px; overflow: hidden; border: 1px solid #30363d; background: #161b22;">
                    <div style="display: flex; align-items: center; justify-content: center; height: 100%; color: #8b949e;">Loading map...</div>
                </div>
@@ -1862,9 +1890,20 @@ def generate_dashboard(stats: dict, dashboard_path: str = "") -> str:
        `;
        document.head.appendChild(style);

-        // Attacker Map Visualization
+        // IP Map Visualization
        let attackerMap = null;
+        let allIps = [];
        let mapMarkers = [];
+        let markerLayers = {{}};
+        let circleLayers = {{}};
+
+        const categoryColors = {{
+            attacker: '#f85149',
+            bad_crawler: '#f0883e',
+            good_crawler: '#3fb950',
+            regular_user: '#58a6ff',
+            unknown: '#8b949e'
+        }};

        async function initializeAttackerMap() {{
            const mapContainer = document.getElementById('attacker-map');
@@ -1884,8 +1923,8 @@ def generate_dashboard(stats: dict, dashboard_path: str = "") -> str:
                    ]
                }});

-                // Fetch all attackers
-                const response = await fetch(DASHBOARD_PATH + '/api/attackers?page=1&page_size=100&sort_by=total_requests&sort_order=desc', {{
+                // Fetch all IPs (not just attackers)
+                const response = await fetch(DASHBOARD_PATH + '/api/all-ips?page=1&page_size=100&sort_by=total_requests&sort_order=desc', {{
                    cache: 'no-store',
                    headers: {{
                        'Cache-Control': 'no-cache',
@@ -1893,18 +1932,18 @@ def generate_dashboard(stats: dict, dashboard_path: str = "") -> str:
                    }}
                }});

-                if (!response.ok) throw new Error('Failed to fetch attackers');
-                
-                const data = await response.json();
-                const attackers = data.attackers || [];
+                if (!response.ok) throw new Error('Failed to fetch IPs');

-                if (attackers.length === 0) {{
-                    mapContainer.innerHTML = '<div style="display: flex; align-items: center; justify-content: center; height: 100%; color: #8b949e;\">No attacker location data available</div>';
+                const data = await response.json();
+                allIps = data.ips || [];
+
+                if (allIps.length === 0) {{
+                    mapContainer.innerHTML = '<div style="display: flex; align-items: center; justify-content: center; height: 100%; color: #8b949e;\">No IP location data available</div>';
                    return;
                }}

                // Get max request count for scaling
-                const maxRequests = Math.max(...attackers.map(a => a.total_requests || 0));
+                const maxRequests = Math.max(...allIps.map(ip => ip.total_requests || 0));

                // Create a map of country locations (approximate country centers)
                const countryCoordinates = {{
@@ -1922,22 +1961,40 @@ def generate_dashboard(stats: dict, dashboard_path: str = "") -> str:
                    'FI': [61.9, 25.8], 'NO': [60.5, 8.5], 'GR': [39.1, 21.8], 'PT': [39.4, -8.2]
                }};

-                // Add markers for each attacker
-                const markerGroup = L.featureGroup();
+                // Create layer groups for each category
+                markerLayers = {{
+                    attacker: L.featureGroup(),
+                    bad_crawler: L.featureGroup(),
+                    good_crawler: L.featureGroup(),
+                    regular_user: L.featureGroup(),
+                    unknown: L.featureGroup()
+                }};

-                attackers.slice(0, 50).forEach(attacker => {{
-                    if (!attacker.country_code) return;
+                circleLayers = {{
+                    attacker: L.featureGroup(),
+                    bad_crawler: L.featureGroup(),
+                    good_crawler: L.featureGroup(),
+                    regular_user: L.featureGroup(),
+                    unknown: L.featureGroup()
+                }};

-                    const coords = countryCoordinates[attacker.country_code];
+                // Add markers for each IP
+                allIps.slice(0, 100).forEach(ip => {{
+                    if (!ip.country_code || !ip.category) return;
+
+                    const coords = countryCoordinates[ip.country_code];
                    if (!coords) return;

+                    const category = ip.category.toLowerCase();
+                    if (!markerLayers[category]) return;
+
                    // Calculate marker size based on request count
-                    const sizeRatio = (attacker.total_requests / maxRequests) * 0.7 + 0.3;
+                    const sizeRatio = (ip.total_requests / maxRequests) * 0.7 + 0.3;
                    const markerSize = Math.max(15, Math.min(40, 20 * sizeRatio));

-                    // Create custom marker element
+                    // Create custom marker element with category-specific class
                    const markerElement = document.createElement('div');
-                    markerElement.className = 'attacker-marker';
+                    markerElement.className = `ip-marker marker-${{category}}`;
                    markerElement.style.width = markerSize + 'px';
                    markerElement.style.height = markerSize + 'px';
                    markerElement.style.fontSize = (markerSize * 0.5) + 'px';
@@ -1947,62 +2004,89 @@ def generate_dashboard(stats: dict, dashboard_path: str = "") -> str:
                        icon: L.divIcon({{
                            html: markerElement,
                            iconSize: [markerSize, markerSize],
-                            className: 'attacker-custom-marker'
+                            className: `ip-custom-marker category-${{category}}`
                        }})
                    }});

-                    // Create popup content
+                    // Create popup content with category badge
+                    const categoryColor = categoryColors[category] || '#8b949e';
+                    const categoryLabels = {{
+                        attacker: 'Attacker',
+                        bad_crawler: 'Bad Crawler',
+                        good_crawler: 'Good Crawler',
+                        regular_user: 'Regular User',
+                        unknown: 'Unknown'
+                    }};
+
                    const popupContent = `
-                        <div style="padding: 8px; min-width: 200px;">
-                            <strong style="color: #58a6ff;">${{attacker.ip}}</strong><br/>
+                        <div style="padding: 8px; min-width: 220px;">
+                            <div style="display: flex; align-items: center; justify-content: space-between; margin-bottom: 8px;">
+                                <strong style="color: #58a6ff;">${{ip.ip}}</strong>
+                                <span style="background: ${{categoryColor}}1a; color: ${{categoryColor}}; padding: 2px 8px; border-radius: 12px; font-size: 11px; font-weight: 600;">
+                                    ${{categoryLabels[category]}}
+                                </span>
+                            </div>
                            <span style="color: #8b949e; font-size: 12px;">
-                                ${{attacker.city || ''}}${{attacker.city && attacker.country_code ? ', ' : ''}}${{attacker.country_code || 'Unknown'}}
+                                ${{ip.city || ''}}${{ip.city && ip.country_code ? ', ' : ''}}${{ip.country_code || 'Unknown'}}
                            </span><br/>
                            <div style="margin-top: 8px; border-top: 1px solid #30363d; padding-top: 8px;">
-                                <div><span style="color: #8b949e;">Requests:</span> <span style="color: #f85149; font-weight: bold;">${{attacker.total_requests}}</span></div>
-                                <div><span style="color: #8b949e;">First Seen:</span> <span style="color: #58a6ff;">${{formatTimestamp(attacker.first_seen)}}</span></div>
-                                <div><span style="color: #8b949e;">Last Seen:</span> <span style="color: #58a6ff;">${{formatTimestamp(attacker.last_seen)}}</span></div>
+                                <div><span style="color: #8b949e;">Requests:</span> <span style="color: ${{categoryColor}}; font-weight: bold;">${{ip.total_requests}}</span></div>
+                                <div><span style="color: #8b949e;">First Seen:</span> <span style="color: #58a6ff;">${{formatTimestamp(ip.first_seen)}}</span></div>
+                                <div><span style="color: #8b949e;">Last Seen:</span> <span style="color: #58a6ff;">${{formatTimestamp(ip.last_seen)}}</span></div>
                            </div>
                        </div>
                    `;

                    marker.bindPopup(popupContent);
-                    markerGroup.addLayer(marker);
-                    mapMarkers.push(marker);
+                    markerLayers[category].addLayer(marker);
                }});

-                // Add cluster circle effect
-                const circleGroup = L.featureGroup();
-                const countryAttackerCount = {{}};
-                
-                attackers.forEach(attacker => {{
-                    if (attacker.country_code) {{
-                        countryAttackerCount[attacker.country_code] = (countryAttackerCount[attacker.country_code] || 0) + 1;
+                // Add cluster circles for each category
+                const categoryCountryCounts = {{}};
+
+                allIps.forEach(ip => {{
+                    if (ip.country_code && ip.category) {{
+                        const category = ip.category.toLowerCase();
+                        if (!categoryCountryCounts[category]) {{
+                            categoryCountryCounts[category] = {{}};
+                        }}
+                        categoryCountryCounts[category][ip.country_code] =
+                            (categoryCountryCounts[category][ip.country_code] || 0) + 1;
                    }}
                }});

-                Object.entries(countryAttackerCount).forEach(([country, count]) => {{
-                    const coords = countryCoordinates[country];
-                    if (coords) {{
-                        const circle = L.circle(coords, {{
-                            radius: 100000 + (count * 150000),
-                            color: '#f85149',
-                            fillColor: '#f85149',
-                            fillOpacity: 0.15,
-                            weight: 1,
-                            opacity: 0.4,
-                            dashArray: '3'
-                        }});
-                        circleGroup.addLayer(circle);
-                    }}
+                Object.entries(categoryCountryCounts).forEach(([category, countryCounts]) => {{
+                    Object.entries(countryCounts).forEach(([country, count]) => {{
+                        const coords = countryCoordinates[country];
+                        if (coords && circleLayers[category]) {{
+                            const color = categoryColors[category] || '#8b949e';
+                            const circle = L.circle(coords, {{
+                                radius: 100000 + (count * 150000),
+                                color: color,
+                                fillColor: color,
+                                fillOpacity: 0.15,
+                                weight: 1,
+                                opacity: 0.4,
+                                dashArray: '3'
+                            }});
+                            circleLayers[category].addLayer(circle);
+                        }}
+                    }});
                }});

-                attackerMap.addLayer(circleGroup);
-                markerGroup.addTo(attackerMap);
-                
-                // Fit map to markers
-                if (markerGroup.getLayers().length > 0) {{
-                    attackerMap.fitBounds(markerGroup.getBounds(), {{ padding: [50, 50] }});
+                // Add all layers to map initially
+                Object.values(circleLayers).forEach(layer => attackerMap.addLayer(layer));
+                Object.values(markerLayers).forEach(layer => attackerMap.addLayer(layer));
+
+                // Fit map to all markers
+                const allMarkers = Object.values(markerLayers).reduce((acc, layer) => {{
+                    acc.push(...layer.getLayers());
+                    return acc;
+                }}, []);
+
+                if (allMarkers.length > 0) {{
+                    const bounds = L.featureGroup(allMarkers).getBounds();
+                    attackerMap.fitBounds(bounds, {{ padding: [50, 50] }});
                }}

            }} catch (err) {{
@@ -2011,6 +2095,46 @@ def generate_dashboard(stats: dict, dashboard_path: str = "") -> str:
            }}
        }}

+        // Update map filters based on checkbox selection
+        function updateMapFilters() {{
+            if (!attackerMap) return;
+
+            const filters = {{
+                attacker: document.getElementById('filter-attacker').checked,
+                bad_crawler: document.getElementById('filter-bad-crawler').checked,
+                good_crawler: document.getElementById('filter-good-crawler').checked,
+                regular_user: document.getElementById('filter-regular-user').checked,
+                unknown: document.getElementById('filter-unknown').checked
+            }};
+
+            // Update marker and circle layers visibility
+            Object.entries(filters).forEach(([category, show]) => {{
+                if (markerLayers[category]) {{
+                    if (show) {{
+                        if (!attackerMap.hasLayer(markerLayers[category])) {{
+                            attackerMap.addLayer(markerLayers[category]);
+                        }}
+                    }} else {{
+                        if (attackerMap.hasLayer(markerLayers[category])) {{
+                            attackerMap.removeLayer(markerLayers[category]);
+                        }}
+                    }}
+                }}
+
+                if (circleLayers[category]) {{
+                    if (show) {{
+                        if (!attackerMap.hasLayer(circleLayers[category])) {{
+                            attackerMap.addLayer(circleLayers[category]);
+                        }}
+                    }} else {{
+                        if (attackerMap.hasLayer(circleLayers[category])) {{
+                            attackerMap.removeLayer(circleLayers[category]);
+                        }}
+                    }}
+                }}
+            }});
+        }}
+
        // Initialize map when Attacks tab is opened
        const originalSwitchTab = window.switchTab;
        let attackTypesChartLoaded = false;
--- a/src/tracker.py
+++ b/src/tracker.py
@@ -173,6 +173,7 @@ class AccessTracker:
        """
        # Skip if this is the server's own IP
        from config import get_config
+
        config = get_config()
        server_ip = config.get_server_ip()
        if server_ip and ip == server_ip:
@@ -228,6 +229,7 @@ class AccessTracker:
        """
        # Skip if this is the server's own IP
        from config import get_config
+
        config = get_config()
        server_ip = config.get_server_ip()
        if server_ip and ip == server_ip:
@@ -397,6 +399,7 @@ class AccessTracker:
        """
        # Skip if this is the server's own IP
        from config import get_config
+
        config = get_config()
        server_ip = config.get_server_ip()
        if server_ip and client_ip == server_ip:
@@ -429,7 +432,9 @@ class AccessTracker:
                self.ip_page_visits[client_ip]["ban_multiplier"] = 2 ** (violations - 1)

                # Set ban timestamp
-                self.ip_page_visits[client_ip]["ban_timestamp"] = datetime.now().isoformat()
+                self.ip_page_visits[client_ip][
+                    "ban_timestamp"
+                ] = datetime.now().isoformat()

            return self.ip_page_visits[client_ip]["count"]

@@ -572,7 +577,8 @@ class AccessTracker:
        suspicious = [
            log
            for log in self.access_log
-            if log.get("suspicious", False) and not is_local_or_private_ip(log.get("ip", ""))
+            if log.get("suspicious", False)
+            and not is_local_or_private_ip(log.get("ip", ""))
        ]
        return suspicious[-limit:]

@@ -624,7 +630,7 @@ class AccessTracker:
        """
        # Trim access_log to max size (keep most recent)
        if len(self.access_log) > self.max_access_log_size:
-            self.access_log = self.access_log[-self.max_access_log_size:]
+            self.access_log = self.access_log[-self.max_access_log_size :]

        # Trim credential_attempts to max size (keep most recent)
        if len(self.credential_attempts) > self.max_credential_log_size: