Malin/krawl.es
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added categorization visualization and itmeline

Browse Source
This commit is contained in:
Patrick Di Fazio
2026-01-07 18:24:43 +01:00
parent 02aed9e65a
commit 7690841029
4 changed files with 240 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
src/database.py
View File

@@ -13,7 +13,7 @@ from typing import Optional, List, Dict, Any
from sqlalchemy import create_engine, func, distinct, case
from sqlalchemy.orm import sessionmaker, scoped_session, Session
from models import Base, AccessLog, CredentialAttempt, AttackDetection, IpStats
from models import Base, AccessLog, CredentialAttempt, AttackDetection, IpStats, CategoryHistory
from sanitizer import (
sanitize_ip,
sanitize_path,
@@ -226,6 +226,7 @@ class DatabaseManager:
def update_ip_stats_analysis(self, ip: str, analyzed_metrics: Dict[str, object], category: str, category_scores: Dict[str, int], last_analysis: datetime) -> None:
"""
Update IP statistics (ip is already persisted).
Records category change in history if category has changed.
Args:
ip: IP address to update
@@ -241,6 +242,11 @@ class DatabaseManager:
sanitized_ip = sanitize_ip(ip)
ip_stats = session.query(IpStats).filter(IpStats.ip == sanitized_ip).first()
# Check if category has changed and record it
old_category = ip_stats.category
if old_category != category:
self._record_category_change(sanitized_ip, old_category, category, last_analysis)
ip_stats.analyzed_metrics = analyzed_metrics
ip_stats.category = category
ip_stats.category_scores = category_scores
@@ -259,9 +265,66 @@ class DatabaseManager:
sanitized_ip = sanitize_ip(ip)
ip_stats = session.query(IpStats).filter(IpStats.ip == sanitized_ip).first()
# Record the manual category change
old_category = ip_stats.category
if old_category != category:
self._record_category_change(sanitized_ip, old_category, category, datetime.utcnow())
ip_stats.category = category
ip_stats.manual_category = True
def _record_category_change(self, ip: str, old_category: Optional[str], new_category: str, timestamp: datetime) -> None:
"""
Internal method to record category changes in history.
Args:
ip: IP address
old_category: Previous category (None if first categorization)
new_category: New category
timestamp: When the change occurred
"""
session = self.session
try:
history_entry = CategoryHistory(
ip=ip,
old_category=old_category,
new_category=new_category,
timestamp=timestamp
)
session.add(history_entry)
session.commit()
except Exception as e:
session.rollback()
print(f"Error recording category change: {e}")
def get_category_history(self, ip: str) -> List[Dict[str, Any]]:
"""
Retrieve category change history for a specific IP.
Args:
ip: IP address to get history for
Returns:
List of category change records ordered by timestamp
"""
session = self.session
try:
sanitized_ip = sanitize_ip(ip)
history = session.query(CategoryHistory).filter(
CategoryHistory.ip == sanitized_ip
).order_by(CategoryHistory.timestamp.asc()).all()
return [
{
'old_category': h.old_category,
'new_category': h.new_category,
'timestamp': h.timestamp.isoformat()
}
for h in history
]
finally:
self.close_session()
def get_access_logs(
self,
limit: int = 100,
@@ -456,6 +519,9 @@ class DatabaseManager:
if not stat:
return None
# Get category history for this IP
category_history = self.get_category_history(ip)
return {
'ip': stat.ip,
'total_requests': stat.total_requests,
@@ -471,7 +537,8 @@ class DatabaseManager:
'category': stat.category,
'category_scores': stat.category_scores or {},
'manual_category': stat.manual_category,
'last_analysis': stat.last_analysis.isoformat() if stat.last_analysis else None
'last_analysis': stat.last_analysis.isoformat() if stat.last_analysis else None,
'category_history': category_history
}
finally:
self.close_session()

40
src/migrations/add_category_history.py Normal file
View File

@@ -0,0 +1,40 @@
#!/usr/bin/env python3
"""
Migration script to add CategoryHistory table to existing databases.
Run this once to upgrade your database schema.
"""
import sys
from pathlib import Path
# Add parent directory to path to import modules
sys.path.insert(0, str(Path(__file__).parent.parent))
from database import get_database, DatabaseManager
from models import Base, CategoryHistory
def migrate():
"""Create CategoryHistory table if it doesn't exist."""
print("Starting migration: Adding CategoryHistory table...")
try:
db = get_database()
# Initialize database if not already done
if not db._initialized:
db.initialize()
# Create only the CategoryHistory table
CategoryHistory.__table__.create(db._engine, checkfirst=True)
print("✓ Migration completed successfully!")
print(" - CategoryHistory table created")
except Exception as e:
print(f"✗ Migration failed: {e}")
sys.exit(1)
if __name__ == "__main__":
migrate()

25
src/models.py
View File

@@ -151,6 +151,31 @@ class IpStats(Base):
def __repr__(self) -> str:
return f"<IpStats(ip='{self.ip}', total_requests={self.total_requests})>"
class CategoryHistory(Base):
"""
Records category changes for IP addresses over time.
Tracks when an IP's category changes, storing both the previous
and new category along with timestamp for timeline visualization.
"""
__tablename__ = 'category_history'
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
ip: Mapped[str] = mapped_column(String(MAX_IP_LENGTH), nullable=False, index=True)
old_category: Mapped[Optional[str]] = mapped_column(String(50), nullable=True)
new_category: Mapped[str] = mapped_column(String(50), nullable=False)
timestamp: Mapped[datetime] = mapped_column(DateTime, nullable=False, default=datetime.utcnow, index=True)
# Composite index for efficient IP-based timeline queries
__table_args__ = (
Index('ix_category_history_ip_timestamp', 'ip', 'timestamp'),
)
def __repr__(self) -> str:
return f"<CategoryHistory(ip='{self.ip}', {self.old_category} -> {self.new_category})>"
# class IpLog(Base):
# """
# Records all IPs that have accessed the honeypot, along with aggregated stats and inferred user category.

112
src/templates/dashboard_template.py
View File

@@ -284,8 +284,8 @@ def generate_dashboard(stats: dict) -> str:
}}
.radar-chart {{
position: relative;
width: 180px;
height: 180px;
width: 220px;
height: 220px;
overflow: visible;
}}
.radar-legend {{
@@ -352,6 +352,72 @@ def generate_dashboard(stats: dict) -> str:
color: #58a6ff;
border: 1px solid #58a6ff;
}}
.timeline-container {{
margin-top: 15px;
padding-top: 15px;
border-top: 1px solid #30363d;
}}
.timeline-title {{
color: #58a6ff;
font-size: 13px;
font-weight: 600;
margin-bottom: 10px;
}}
.timeline {{
position: relative;
padding-left: 30px;
}}
.timeline::before {{
content: '';
position: absolute;
left: 12px;
top: 5px;
bottom: 5px;
width: 3px;
background: #30363d;
}}
.timeline-item {{
position: relative;
padding-bottom: 15px;
}}
.timeline-item:last-child {{
padding-bottom: 0;
}}
.timeline-marker {{
position: absolute;
left: -26px;
width: 16px;
height: 16px;
border-radius: 50%;
border: 2px solid #0d1117;
}}
.timeline-marker.attacker {{
background: #f85149;
}}
.timeline-marker.good-crawler {{
background: #3fb950;
}}
.timeline-marker.bad-crawler {{
background: #f0883e;
}}
.timeline-marker.regular-user {{
background: #58a6ff;
}}
.timeline-content {{
font-size: 12px;
}}
.timeline-category {{
font-weight: 600;
}}
.timeline-timestamp {{
color: #8b949e;
font-size: 11px;
margin-top: 2px;
}}
.timeline-arrow {{
color: #8b949e;
margin: 0 7px;
}}
</style>
</head>
@@ -658,11 +724,45 @@ def generate_dashboard(stats: dict) -> str:
html += '</div>';
}}
// Category History Timeline
if (stats.category_history && stats.category_history.length > 0) {{
html += '<div class="timeline-container">';
html += '<div class="timeline-title">Behavior Timeline</div>';
html += '<div class="timeline">';
stats.category_history.forEach((change, index) => {{
const categoryClass = change.new_category.toLowerCase().replace('_', '-');
const timestamp = new Date(change.timestamp).toLocaleString();
html += '<div class="timeline-item">';
html += `<div class="timeline-marker ${{categoryClass}}"></div>`;
html += '<div class="timeline-content">';
if (change.old_category) {{
const oldCategoryBadge = 'category-' + change.old_category.toLowerCase().replace('_', '-');
html += `<span class="category-badge ${{oldCategoryBadge}}">${{change.old_category}}</span>`;
html += '<span class="timeline-arrow">→</span>';
}} else {{
html += '<span style="color: #8b949e;">Initial:</span> ';
}}
const newCategoryBadge = 'category-' + change.new_category.toLowerCase().replace('_', '-');
html += `<span class="category-badge ${{newCategoryBadge}}">${{change.new_category}}</span>`;
html += `<div class="timeline-timestamp">${{timestamp}}</div>`;
html += '</div>';
html += '</div>';
}});
html += '</div>';
html += '</div>';
}}
html += '</div>';
// Radar chart on the right
if (stats.category_scores && Object.keys(stats.category_scores).length > 0) {{
html += '<div class="stats-right">';
html += '<div style="font-size: 13px; font-weight: 600; color: #58a6ff; margin-bottom: 10px;">Category Score</div>';
html += '<svg class="radar-chart" viewBox="-30 -30 260 260" preserveAspectRatio="xMidYMid meet">';
const scores = {{
@@ -705,7 +805,7 @@ def generate_dashboard(stats: dict) -> str:
// Draw axes
const angles = [0, 90, 180, 270];
const keys = ['attacker', 'good_crawler', 'bad_crawler', 'regular_user'];
const keys = ['good_crawler', 'regular_user', 'bad_crawler', 'attacker'];
angles.forEach((angle, i) => {{
const rad = (angle - 90) * Math.PI / 180;
@@ -713,8 +813,8 @@ def generate_dashboard(stats: dict) -> str:
const y2 = cy + maxRadius * Math.sin(rad);
html += `<line x1="${{cx}}" y1="${{cy}}" x2="${{x2}}" y2="${{y2}}" stroke="#30363d" stroke-width="0.5"/>`;
// Add labels
const labelDist = maxRadius + 30;
// Add labels at consistent distance
const labelDist = maxRadius + 35;
const lx = cx + labelDist * Math.cos(rad);
const ly = cy + labelDist * Math.sin(rad);
html += `<text x="${{lx}}" y="${{ly}}" fill="#8b949e" font-size="12" text-anchor="middle" dominant-baseline="middle">${{labels[keys[i]]}}</text>`;
@@ -755,7 +855,7 @@ def generate_dashboard(stats: dict) -> str:
keys.forEach(key => {{
html += '<div class="radar-legend-item">';
html += `<div class="radar-legend-color" style="background: ${{colors[key]}};"></div>`;
html += `<span style="color: #8b949e;">${{labels[key]}}: ${{scores[key]}}%</span>`;
html += `<span style="color: #8b949e;">${{labels[key]}}: ${{scores[key]}} pt</span>`;
html += '</div>';
}});
html += '</div>';