added categorization visualization and itmeline

src/database.py

@@ -13,7 +13,7 @@ from typing import Optional, List, Dict, Any
 from sqlalchemy import create_engine, func, distinct, case
 from sqlalchemy.orm import sessionmaker, scoped_session, Session
 
-from models import Base, AccessLog, CredentialAttempt, AttackDetection, IpStats
+from models import Base, AccessLog, CredentialAttempt, AttackDetection, IpStats, CategoryHistory
 from sanitizer import (
     sanitize_ip,
     sanitize_path,
@@ -226,6 +226,7 @@ class DatabaseManager:
     def  update_ip_stats_analysis(self, ip: str, analyzed_metrics: Dict[str, object], category: str, category_scores: Dict[str, int], last_analysis: datetime) -> None:
         """
         Update IP statistics (ip is already persisted).
+        Records category change in history if category has changed.
 
         Args:
             ip: IP address to update
@@ -241,6 +242,11 @@ class DatabaseManager:
         sanitized_ip = sanitize_ip(ip)
         ip_stats = session.query(IpStats).filter(IpStats.ip == sanitized_ip).first()
 
+        # Check if category has changed and record it
+        old_category = ip_stats.category
+        if old_category != category:
+            self._record_category_change(sanitized_ip, old_category, category, last_analysis)
+
         ip_stats.analyzed_metrics = analyzed_metrics
         ip_stats.category = category
         ip_stats.category_scores = category_scores
@@ -259,9 +265,66 @@ class DatabaseManager:
         sanitized_ip = sanitize_ip(ip)
         ip_stats = session.query(IpStats).filter(IpStats.ip == sanitized_ip).first()
 
+        # Record the manual category change
+        old_category = ip_stats.category
+        if old_category != category:
+            self._record_category_change(sanitized_ip, old_category, category, datetime.utcnow())
+
         ip_stats.category = category
         ip_stats.manual_category = True
 
+    def _record_category_change(self, ip: str, old_category: Optional[str], new_category: str, timestamp: datetime) -> None:
+        """
+        Internal method to record category changes in history.
+
+        Args:
+            ip: IP address
+            old_category: Previous category (None if first categorization)
+            new_category: New category
+            timestamp: When the change occurred
+        """
+        session = self.session
+        try:
+            history_entry = CategoryHistory(
+                ip=ip,
+                old_category=old_category,
+                new_category=new_category,
+                timestamp=timestamp
+            )
+            session.add(history_entry)
+            session.commit()
+        except Exception as e:
+            session.rollback()
+            print(f"Error recording category change: {e}")
+
+    def get_category_history(self, ip: str) -> List[Dict[str, Any]]:
+        """
+        Retrieve category change history for a specific IP.
+
+        Args:
+            ip: IP address to get history for
+
+        Returns:
+            List of category change records ordered by timestamp
+        """
+        session = self.session
+        try:
+            sanitized_ip = sanitize_ip(ip)
+            history = session.query(CategoryHistory).filter(
+                CategoryHistory.ip == sanitized_ip
+            ).order_by(CategoryHistory.timestamp.asc()).all()
+
+            return [
+                {
+                    'old_category': h.old_category,
+                    'new_category': h.new_category,
+                    'timestamp': h.timestamp.isoformat()
+                }
+                for h in history
+            ]
+        finally:
+            self.close_session()
+
     def get_access_logs(
         self,
         limit: int = 100,
@@ -456,6 +519,9 @@ class DatabaseManager:
             if not stat:
                 return None
             
+            # Get category history for this IP
+            category_history = self.get_category_history(ip)
+            
             return {
                 'ip': stat.ip,
                 'total_requests': stat.total_requests,
@@ -471,7 +537,8 @@ class DatabaseManager:
                 'category': stat.category,
                 'category_scores': stat.category_scores or {},
                 'manual_category': stat.manual_category,
-                'last_analysis': stat.last_analysis.isoformat() if stat.last_analysis else None
+                'last_analysis': stat.last_analysis.isoformat() if stat.last_analysis else None,
+                'category_history': category_history
             }
         finally:
             self.close_session()

src/migrations/add_category_history.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+"""
+Migration script to add CategoryHistory table to existing databases.
+Run this once to upgrade your database schema.
+"""
+
+import sys
+from pathlib import Path
+
+# Add parent directory to path to import modules
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from database import get_database, DatabaseManager
+from models import Base, CategoryHistory
+
+
+def migrate():
+    """Create CategoryHistory table if it doesn't exist."""
+    print("Starting migration: Adding CategoryHistory table...")
+    
+    try:
+        db = get_database()
+        
+        # Initialize database if not already done
+        if not db._initialized:
+            db.initialize()
+        
+        # Create only the CategoryHistory table
+        CategoryHistory.__table__.create(db._engine, checkfirst=True)
+        
+        print("✓ Migration completed successfully!")
+        print("  - CategoryHistory table created")
+        
+    except Exception as e:
+        print(f"✗ Migration failed: {e}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    migrate()

src/models.py

@@ -151,6 +151,31 @@ class IpStats(Base):
     def __repr__(self) -> str:
         return f"<IpStats(ip='{self.ip}', total_requests={self.total_requests})>"
 
+
+class CategoryHistory(Base):
+    """
+    Records category changes for IP addresses over time.
+
+    Tracks when an IP's category changes, storing both the previous
+    and new category along with timestamp for timeline visualization.
+    """
+    __tablename__ = 'category_history'
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    ip: Mapped[str] = mapped_column(String(MAX_IP_LENGTH), nullable=False, index=True)
+    old_category: Mapped[Optional[str]] = mapped_column(String(50), nullable=True)
+    new_category: Mapped[str] = mapped_column(String(50), nullable=False)
+    timestamp: Mapped[datetime] = mapped_column(DateTime, nullable=False, default=datetime.utcnow, index=True)
+
+    # Composite index for efficient IP-based timeline queries
+    __table_args__ = (
+        Index('ix_category_history_ip_timestamp', 'ip', 'timestamp'),
+    )
+
+    def __repr__(self) -> str:
+        return f"<CategoryHistory(ip='{self.ip}', {self.old_category} -> {self.new_category})>"
+
+
 # class IpLog(Base):
 #     """
 #     Records all IPs that have accessed the honeypot, along with aggregated stats and inferred user category.

src/templates/dashboard_template.py

@@ -284,8 +284,8 @@ def generate_dashboard(stats: dict) -> str:
         }}
         .radar-chart {{
             position: relative;
-            width: 180px;
+            width: 220px;
-            height: 180px;
+            height: 220px;
             overflow: visible;
         }}
         .radar-legend {{
@@ -352,6 +352,72 @@ def generate_dashboard(stats: dict) -> str:
             color: #58a6ff;
             border: 1px solid #58a6ff;
         }}
+        .timeline-container {{
+            margin-top: 15px;
+            padding-top: 15px;
+            border-top: 1px solid #30363d;
+        }}
+        .timeline-title {{
+            color: #58a6ff;
+            font-size: 13px;
+            font-weight: 600;
+            margin-bottom: 10px;
+        }}
+        .timeline {{
+            position: relative;
+            padding-left: 30px;
+        }}
+        .timeline::before {{
+            content: '';
+            position: absolute;
+            left: 12px;
+            top: 5px;
+            bottom: 5px;
+            width: 3px;
+            background: #30363d;
+        }}
+        .timeline-item {{
+            position: relative;
+            padding-bottom: 15px;
+        }}
+        .timeline-item:last-child {{
+            padding-bottom: 0;
+        }}
+        .timeline-marker {{
+            position: absolute;
+            left: -26px;
+            width: 16px;
+            height: 16px;
+            border-radius: 50%;
+            border: 2px solid #0d1117;
+        }}
+        .timeline-marker.attacker {{
+            background: #f85149;
+        }}
+        .timeline-marker.good-crawler {{
+            background: #3fb950;
+        }}
+        .timeline-marker.bad-crawler {{
+            background: #f0883e;
+        }}
+        .timeline-marker.regular-user {{
+            background: #58a6ff;
+        }}
+        .timeline-content {{
+            font-size: 12px;
+        }}
+        .timeline-category {{
+            font-weight: 600;
+        }}
+        .timeline-timestamp {{
+            color: #8b949e;
+            font-size: 11px;
+            margin-top: 2px;
+        }}
+        .timeline-arrow {{
+            color: #8b949e;
+            margin: 0 7px;
+        }}
 
     </style>
 </head>
@@ -658,11 +724,45 @@ def generate_dashboard(stats: dict) -> str:
                 html += '</div>';
             }}
             
+            // Category History Timeline
+            if (stats.category_history && stats.category_history.length > 0) {{
+                html += '<div class="timeline-container">';
+                html += '<div class="timeline-title">Behavior Timeline</div>';
+                html += '<div class="timeline">';
+                
+                stats.category_history.forEach((change, index) => {{
+                    const categoryClass = change.new_category.toLowerCase().replace('_', '-');
+                    const timestamp = new Date(change.timestamp).toLocaleString();
+                    
+                    html += '<div class="timeline-item">';
+                    html += `<div class="timeline-marker ${{categoryClass}}"></div>`;
+                    html += '<div class="timeline-content">';
+                    
+                    if (change.old_category) {{
+                        const oldCategoryBadge = 'category-' + change.old_category.toLowerCase().replace('_', '-');
+                        html += `<span class="category-badge ${{oldCategoryBadge}}">${{change.old_category}}</span>`;
+                        html += '<span class="timeline-arrow">→</span>';
+                    }} else {{
+                        html += '<span style="color: #8b949e;">Initial:</span> ';
+                    }}
+                    
+                    const newCategoryBadge = 'category-' + change.new_category.toLowerCase().replace('_', '-');
+                    html += `<span class="category-badge ${{newCategoryBadge}}">${{change.new_category}}</span>`;
+                    html += `<div class="timeline-timestamp">${{timestamp}}</div>`;
+                    html += '</div>';
+                    html += '</div>';
+                }});
+                
+                html += '</div>';
+                html += '</div>';
+            }}
+            
             html += '</div>';
             
             // Radar chart on the right
             if (stats.category_scores && Object.keys(stats.category_scores).length > 0) {{
                 html += '<div class="stats-right">';
+                html += '<div style="font-size: 13px; font-weight: 600; color: #58a6ff; margin-bottom: 10px;">Category Score</div>';
                 html += '<svg class="radar-chart" viewBox="-30 -30 260 260" preserveAspectRatio="xMidYMid meet">';
                 
                 const scores = {{
@@ -705,7 +805,7 @@ def generate_dashboard(stats: dict) -> str:
                 
                 // Draw axes
                 const angles = [0, 90, 180, 270];
-                const keys = ['attacker', 'good_crawler', 'bad_crawler', 'regular_user'];
+                const keys = ['good_crawler', 'regular_user', 'bad_crawler', 'attacker'];
                 
                 angles.forEach((angle, i) => {{
                     const rad = (angle - 90) * Math.PI / 180;
@@ -713,8 +813,8 @@ def generate_dashboard(stats: dict) -> str:
                     const y2 = cy + maxRadius * Math.sin(rad);
                     html += `<line x1="${{cx}}" y1="${{cy}}" x2="${{x2}}" y2="${{y2}}" stroke="#30363d" stroke-width="0.5"/>`;
                     
-                    // Add labels
+                    // Add labels at consistent distance
-                    const labelDist = maxRadius + 30;
+                    const labelDist = maxRadius + 35;
                     const lx = cx + labelDist * Math.cos(rad);
                     const ly = cy + labelDist * Math.sin(rad);
                     html += `<text x="${{lx}}" y="${{ly}}" fill="#8b949e" font-size="12" text-anchor="middle" dominant-baseline="middle">${{labels[keys[i]]}}</text>`;
@@ -755,7 +855,7 @@ def generate_dashboard(stats: dict) -> str:
                 keys.forEach(key => {{
                     html += '<div class="radar-legend-item">';
                     html += `<div class="radar-legend-color" style="background: ${{colors[key]}};"></div>`;
-                    html += `<span style="color: #8b949e;">${{labels[key]}}: ${{scores[key]}}%</span>`;
+                    html += `<span style="color: #8b949e;">${{labels[key]}}: ${{scores[key]}} pt</span>`;
                     html += '</div>';
                 }});
                 html += '</div>';