Merge pull request #99 from BlessedRebuS/feat/improved-map

Feat/improved map

helm/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: krawl-chart
 description: A Helm chart for Krawl honeypot server
 type: application
-version: 1.0.8
+version: 1.0.9
-appVersion: 1.0.8
+appVersion: 1.0.9
 keywords:
   - honeypot
   - security

src/routes/api.py

@@ -73,7 +73,7 @@ async def all_ips(
 ):
     db = get_db()
     page = max(1, page)
-    page_size = min(max(1, page_size), 100)
+    page_size = min(max(1, page_size), 10000)
 
     try:
         result = db.get_all_ips_paginated(

src/templates/jinja2/base.html

@@ -6,8 +6,11 @@
     <title>Krawl Dashboard</title>
     <link rel="icon" type="image/svg+xml" href="{{ dashboard_path }}/static/krawl-svg.svg" />
     <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/leaflet/1.9.4/leaflet.min.css" crossorigin="anonymous" />
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/leaflet.markercluster/1.5.3/MarkerCluster.css" crossorigin="anonymous" />
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/leaflet.markercluster/1.5.3/MarkerCluster.Default.css" crossorigin="anonymous" />
     <link rel="stylesheet" href="{{ dashboard_path }}/static/css/dashboard.css" />
     <script src="https://cdnjs.cloudflare.com/ajax/libs/leaflet/1.9.4/leaflet.min.js" crossorigin="anonymous" defer></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/leaflet.markercluster/1.5.3/leaflet.markercluster.js" crossorigin="anonymous" defer></script>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/3.9.1/chart.min.js" defer></script>
     <script src="https://unpkg.com/htmx.org@2.0.4" defer></script>
     <script defer src="https://cdn.jsdelivr.net/npm/alpinejs@3.14.8/dist/cdn.min.js"></script>

src/templates/jinja2/dashboard/partials/map_section.html

@@ -2,6 +2,17 @@
 <div class="table-container">
     <h2>IP Origins Map</h2>
     <div style="margin-bottom: 10px; display: flex; gap: 15px; flex-wrap: wrap; align-items: center;">
+        <label style="display: flex; align-items: center; gap: 6px; cursor: pointer; color: #8b949e; font-size: 13px;">
+            Show top
+            <select id="map-ip-limit" onchange="if(typeof reloadMapWithLimit==='function') reloadMapWithLimit(this.value)" style="background: #161b22; color: #c9d1d9; border: 1px solid #30363d; border-radius: 4px; padding: 2px 6px; font-size: 13px; cursor: pointer;">
+                <option value="10">10</option>
+                <option value="100" selected>100</option>
+                <option value="1000">1,000</option>
+                <option value="all">All</option>
+            </select>
+            IPs
+        </label>
+        <span style="color: #30363d;">|</span>
         <label style="display: flex; align-items: center; gap: 4px; cursor: pointer;">
             <input type="checkbox" checked onchange="if(typeof updateMapFilters==='function') updateMapFilters()" class="map-filter" data-category="attacker">
             <span style="color: #f85149;">Attackers</span>

src/templates/static/css/dashboard.css

@@ -626,6 +626,11 @@ tbody {
 .marker-unknown:hover {
     box-shadow: 0 0 15px rgba(139, 148, 158, 1), inset 0 0 6px rgba(139, 148, 158, 0.7);
 }
+/* Custom pie-chart cluster icons */
+.ip-cluster-icon {
+    background: none !important;
+    border: none !important;
+}
 .leaflet-bottom.leaflet-right {
     display: none !important;
 }

src/templates/static/js/map.js

@@ -3,8 +3,9 @@
 
 let attackerMap = null;
 let allIps = [];
-let mapMarkers = [];
+let mapMarkers = [];       // all marker objects, each tagged with .options.category
-let markerLayers = {};
+let clusterGroup = null;   // single shared MarkerClusterGroup
+let hiddenCategories = new Set();
 
 const categoryColors = {
     attacker: '#f85149',
@@ -14,47 +15,41 @@ const categoryColors = {
     unknown: '#8b949e'
 };
 
-async function initializeAttackerMap() {
+// Build a conic-gradient pie icon showing the category mix inside a cluster
-    const DASHBOARD_PATH = window.__DASHBOARD_PATH__ || '';
+function createClusterIcon(cluster) {
-    const mapContainer = document.getElementById('attacker-map');
+    const children = cluster.getAllChildMarkers();
-    if (!mapContainer || attackerMap) return;
+    const counts = {};
-
+    children.forEach(m => {
-    try {
+        const cat = m.options.category || 'unknown';
-        // Initialize map
+        counts[cat] = (counts[cat] || 0) + 1;
-        attackerMap = L.map('attacker-map', {
-            center: [20, 0],
-            zoom: 2,
-            layers: [
-                L.tileLayer('https://{s}.basemaps.cartocdn.com/dark_all/{z}/{x}/{y}{r}.png', {
-                    attribution: '© CartoDB | © OpenStreetMap contributors',
-                    maxZoom: 19,
-                    subdomains: 'abcd'
-                })
-            ]
     });
 
-        // Fetch all IPs (not just attackers)
+    const total = children.length;
-        const response = await fetch(DASHBOARD_PATH + '/api/all-ips?page=1&page_size=100&sort_by=total_requests&sort_order=desc', {
+    const sorted = Object.entries(counts).sort((a, b) => b[1] - a[1]);
-            cache: 'no-store',
+    let gradientStops = [];
-            headers: {
+    let cumulative = 0;
-                'Cache-Control': 'no-cache',
+    sorted.forEach(([cat, count]) => {
-                'Pragma': 'no-cache'
+        const start = (cumulative / total) * 360;
-            }
+        cumulative += count;
+        const end = (cumulative / total) * 360;
+        const color = categoryColors[cat] || '#8b949e';
+        gradientStops.push(`${color} ${start.toFixed(1)}deg ${end.toFixed(1)}deg`);
     });
 
-        if (!response.ok) throw new Error('Failed to fetch IPs');
+    const size = Math.max(32, Math.min(50, 32 + Math.log2(total) * 5));
+    const inner = size - 10;
+    const offset = 5; // (size - inner) / 2
 
-        const data = await response.json();
+    return L.divIcon({
-        allIps = data.ips || [];
+        html: `<div style="position:relative;width:${size}px;height:${size}px;">` +
-
+            `<div style="position:absolute;top:0;left:0;width:${size}px;height:${size}px;border-radius:50%;background:conic-gradient(${gradientStops.join(', ')});box-shadow:0 0 6px rgba(0,0,0,0.5);"></div>` +
-        if (allIps.length === 0) {
+            `<div style="position:absolute;top:${offset}px;left:${offset}px;width:${inner}px;height:${inner}px;border-radius:50%;background:rgba(13,17,23,0.85);color:#e6edf3;font-size:11px;font-weight:700;line-height:${inner}px;text-align:center;">${total}</div>` +
-            mapContainer.innerHTML = '<div style="display: flex; align-items: center; justify-content: center; height: 100%; color: #8b949e;">No IP location data available</div>';
+            `</div>`,
-            return;
+        className: 'ip-cluster-icon',
+        iconSize: L.point(size, size)
+    });
 }
 
-        // Get max request count for scaling
-        const maxRequests = Math.max(...allIps.map(ip => ip.total_requests || 0));
-
 // City coordinates database (major cities worldwide)
 const cityCoordinates = {
     // United States
@@ -92,8 +87,8 @@ async function initializeAttackerMap() {
     'Karachi': [24.8607, 67.0011], 'Islamabad': [33.6844, 73.0479],
     'Dhaka': [23.8103, 90.4125], 'Colombo': [6.9271, 79.8612],
     // South America
-            'S\u00e3o Paulo': [-23.5505, -46.6333], 'Rio de Janeiro': [-22.9068, -43.1729],
+    'São Paulo': [-23.5505, -46.6333], 'Rio de Janeiro': [-22.9068, -43.1729],
-            'Buenos Aires': [-34.6037, -58.3816], 'Bogot\u00e1': [4.7110, -74.0721],
+    'Buenos Aires': [-34.6037, -58.3816], 'Bogotá': [4.7110, -74.0721],
     'Lima': [-12.0464, -77.0428], 'Santiago': [-33.4489, -70.6693],
     // Middle East & Africa
     'Cairo': [30.0444, 31.2357], 'Dubai': [25.2048, 55.2708],
@@ -128,21 +123,71 @@ async function initializeAttackerMap() {
 
 // Helper function to get coordinates for an IP
 function getIPCoordinates(ip) {
-            // Use actual latitude and longitude if available
     if (ip.latitude != null && ip.longitude != null) {
         return [ip.latitude, ip.longitude];
     }
-            // Fall back to city lookup
     if (ip.city && cityCoordinates[ip.city]) {
         return cityCoordinates[ip.city];
     }
-            // Fall back to country
     if (ip.country_code && countryCoordinates[ip.country_code]) {
         return countryCoordinates[ip.country_code];
     }
     return null;
 }
 
+// Fetch IPs from the API, handling pagination for "all"
+async function fetchIpsForMap(limit) {
+    const DASHBOARD_PATH = window.__DASHBOARD_PATH__ || '';
+    const headers = { 'Cache-Control': 'no-cache', 'Pragma': 'no-cache' };
+
+    if (limit === 'all') {
+        // Fetch in pages of 1000 until we have everything
+        let collected = [];
+        let page = 1;
+        const pageSize = 1000;
+        while (true) {
+            const response = await fetch(
+                `${DASHBOARD_PATH}/api/all-ips?page=${page}&page_size=${pageSize}&sort_by=total_requests&sort_order=desc`,
+                { cache: 'no-store', headers }
+            );
+            if (!response.ok) throw new Error('Failed to fetch IPs');
+            const data = await response.json();
+            collected = collected.concat(data.ips || []);
+            if (page >= data.pagination.total_pages) break;
+            page++;
+        }
+        return collected;
+    }
+
+    const pageSize = parseInt(limit, 10);
+    const response = await fetch(
+        `${DASHBOARD_PATH}/api/all-ips?page=1&page_size=${pageSize}&sort_by=total_requests&sort_order=desc`,
+        { cache: 'no-store', headers }
+    );
+    if (!response.ok) throw new Error('Failed to fetch IPs');
+    const data = await response.json();
+    return data.ips || [];
+}
+
+// Build markers from an IP list and add them to the map
+function buildMapMarkers(ips) {
+    // Clear existing cluster group
+    if (clusterGroup) {
+        attackerMap.removeLayer(clusterGroup);
+        clusterGroup.clearLayers();
+    }
+    mapMarkers = [];
+
+    // Single cluster group with custom pie-chart icons
+    clusterGroup = L.markerClusterGroup({
+        maxClusterRadius: 20,
+        spiderfyOnMaxZoom: true,
+        showCoverageOnHover: false,
+        zoomToBoundsOnClick: true,
+        disableClusteringAtZoom: 10,
+        iconCreateFunction: createClusterIcon
+    });
+
     // Track used coordinates to add small offsets for overlapping markers
     const usedCoordinates = {};
     function getUniqueCoordinates(baseCoords) {
@@ -152,15 +197,12 @@ async function initializeAttackerMap() {
         }
         usedCoordinates[key]++;
 
-            // If this is the first marker at this location, use exact coordinates
         if (usedCoordinates[key] === 1) {
             return baseCoords;
         }
 
-            // Add small random offset for subsequent markers
+        const angle = (usedCoordinates[key] * 137.5) % 360;
-            // Offset increases with each marker to create a spread pattern
+        const distance = 0.05 * Math.sqrt(usedCoordinates[key]);
-            const angle = (usedCoordinates[key] * 137.5) % 360; // Golden angle for even distribution
-            const distance = 0.05 * Math.sqrt(usedCoordinates[key]); // Increase distance with more markers
         const latOffset = distance * Math.cos(angle * Math.PI / 180);
         const lngOffset = distance * Math.sin(angle * Math.PI / 180);
 
@@ -170,36 +212,22 @@ async function initializeAttackerMap() {
         ];
     }
 
-        // Create layer groups for each category
+    const DASHBOARD_PATH = window.__DASHBOARD_PATH__ || '';
-        markerLayers = {
-            attacker: L.featureGroup(),
-            bad_crawler: L.featureGroup(),
-            good_crawler: L.featureGroup(),
-            regular_user: L.featureGroup(),
-            unknown: L.featureGroup()
-        };
 
-        // Add markers for each IP
+    ips.forEach(ip => {
-        allIps.slice(0, 100).forEach(ip => {
         if (!ip.country_code || !ip.category) return;
 
-            // Get coordinates (city first, then country)
         const baseCoords = getIPCoordinates(ip);
         if (!baseCoords) return;
 
-            // Get unique coordinates with offset to prevent overlap
         const coords = getUniqueCoordinates(baseCoords);
-
         const category = ip.category.toLowerCase();
-            if (!markerLayers[category]) return;
+        if (!categoryColors[category]) return;
 
-            // Calculate marker size based on request count with more dramatic scaling
-            // Scale up to 10,000 requests, then cap it
         const requestsForScale = Math.min(ip.total_requests, 10000);
-            const sizeRatio = Math.pow(requestsForScale / 10000, 0.5); // Square root for better visual scaling
+        const sizeRatio = Math.pow(requestsForScale / 10000, 0.5);
         const markerSize = Math.max(10, Math.min(30, 10 + (sizeRatio * 20)));
 
-            // Create custom marker element with category-specific class
         const markerElement = document.createElement('div');
         markerElement.className = `ip-marker marker-${category}`;
         markerElement.style.width = markerSize + 'px';
@@ -212,10 +240,10 @@ async function initializeAttackerMap() {
                 html: markerElement.outerHTML,
                 iconSize: [markerSize, markerSize],
                 className: `ip-custom-marker category-${category}`
-                })
+            }),
+            category: category
         });
 
-            // Create popup with category badge and chart
         const categoryColor = categoryColors[category] || '#8b949e';
         const categoryLabels = {
             attacker: 'Attacker',
@@ -225,15 +253,12 @@ async function initializeAttackerMap() {
             unknown: 'Unknown'
         };
 
-            // Bind popup once when marker is created
         marker.bindPopup('', {
             maxWidth: 550,
             className: 'ip-detail-popup'
         });
 
-            // Add click handler to fetch data and show popup
         marker.on('click', async function(e) {
-                // Show loading popup first
             const loadingPopup = `
                 <div style="padding: 12px; min-width: 280px; max-width: 320px;">
                     <div style="display: flex; align-items: center; justify-content: space-between; margin-bottom: 8px;">
@@ -252,14 +277,11 @@ async function initializeAttackerMap() {
             marker.openPopup();
 
             try {
-                    console.log('Fetching IP stats for:', ip.ip);
                 const response = await fetch(`${DASHBOARD_PATH}/api/ip-stats/${ip.ip}`);
                 if (!response.ok) throw new Error('Failed to fetch IP stats');
 
                 const stats = await response.json();
-                    console.log('Received stats:', stats);
 
-                    // Build complete popup content with chart
                 let popupContent = `
                     <div style="padding: 12px; min-width: 200px;">
                         <div style="display: flex; align-items: center; justify-content: space-between; margin-bottom: 8px;">
@@ -278,11 +300,8 @@ async function initializeAttackerMap() {
                         </div>
                 `;
 
-                    // Add chart if category scores exist
                 if (stats.category_scores && Object.keys(stats.category_scores).length > 0) {
-                        console.log('Category scores found:', stats.category_scores);
                     const chartHtml = generateMapPanelRadarChart(stats.category_scores);
-                        console.log('Generated chart HTML length:', chartHtml.length);
                     popupContent += `
                         <div style="margin-top: 12px; border-top: 1px solid #30363d; padding-top: 12px;">
                             ${chartHtml}
@@ -291,9 +310,6 @@ async function initializeAttackerMap() {
                 }
 
                 popupContent += '</div>';
-
-                    // Update popup content
-                    console.log('Updating popup content');
                 marker.setPopupContent(popupContent);
             } catch (err) {
                 console.error('Error fetching IP stats:', err);
@@ -322,25 +338,54 @@ async function initializeAttackerMap() {
             }
         });
 
-            markerLayers[category].addLayer(marker);
+        mapMarkers.push(marker);
+        // Only add to cluster if category is not hidden
+        if (!hiddenCategories.has(category)) {
+            clusterGroup.addLayer(marker);
+        }
     });
 
-        // Add all marker layers to map initially
+    attackerMap.addLayer(clusterGroup);
-        Object.values(markerLayers).forEach(layer => attackerMap.addLayer(layer));
 
-        // Fit map to all markers
+    // Fit map to visible markers
-        const allMarkers = Object.values(markerLayers).reduce((acc, layer) => {
+    const visibleMarkers = mapMarkers.filter(m => !hiddenCategories.has(m.options.category));
-            acc.push(...layer.getLayers());
+    if (visibleMarkers.length > 0) {
-            return acc;
+        const bounds = L.featureGroup(visibleMarkers).getBounds();
-        }, []);
-
-        if (allMarkers.length > 0) {
-            const bounds = L.featureGroup(allMarkers).getBounds();
         attackerMap.fitBounds(bounds, { padding: [50, 50] });
     }
+}
+
+async function initializeAttackerMap() {
+    const mapContainer = document.getElementById('attacker-map');
+    if (!mapContainer || attackerMap) return;
+
+    try {
+        attackerMap = L.map('attacker-map', {
+            center: [20, 0],
+            zoom: 2,
+            layers: [
+                L.tileLayer('https://{s}.basemaps.cartocdn.com/dark_all/{z}/{x}/{y}{r}.png', {
+                    attribution: '&copy; CartoDB | &copy; OpenStreetMap contributors',
+                    maxZoom: 19,
+                    subdomains: 'abcd'
+                })
+            ]
+        });
+
+        // Get the selected limit from the dropdown (default 100)
+        const limitSelect = document.getElementById('map-ip-limit');
+        const limit = limitSelect ? limitSelect.value : '100';
+
+        allIps = await fetchIpsForMap(limit);
+
+        if (allIps.length === 0) {
+            mapContainer.innerHTML = '<div style="display: flex; align-items: center; justify-content: center; height: 100%; color: #8b949e;">No IP location data available</div>';
+            return;
+        }
+
+        buildMapMarkers(allIps);
 
         // Force Leaflet to recalculate container size after the tab becomes visible.
-        // Without this, tiles may not render correctly when the container was hidden.
         setTimeout(() => {
             if (attackerMap) attackerMap.invalidateSize();
         }, 300);
@@ -351,30 +396,44 @@ async function initializeAttackerMap() {
     }
 }
 
-// Update map filters based on checkbox selection
+// Reload map markers when the user changes the IP limit selector
-function updateMapFilters() {
+async function reloadMapWithLimit(limit) {
     if (!attackerMap) return;
 
-    const filters = {};
+    // Show loading state
+    const mapContainer = document.getElementById('attacker-map');
+    const overlay = document.createElement('div');
+    overlay.id = 'map-loading-overlay';
+    overlay.style.cssText = 'position:absolute;top:0;left:0;right:0;bottom:0;background:rgba(13,17,23,0.7);display:flex;align-items:center;justify-content:center;z-index:1000;color:#8b949e;font-size:14px;';
+    overlay.textContent = 'Loading IPs...';
+    mapContainer.style.position = 'relative';
+    mapContainer.appendChild(overlay);
+
+    try {
+        allIps = await fetchIpsForMap(limit);
+        buildMapMarkers(allIps);
+    } catch (err) {
+        console.error('Error reloading map:', err);
+    } finally {
+        const existing = document.getElementById('map-loading-overlay');
+        if (existing) existing.remove();
+    }
+}
+
+// Update map filters based on checkbox selection
+function updateMapFilters() {
+    if (!attackerMap || !clusterGroup) return;
+
+    hiddenCategories.clear();
     document.querySelectorAll('.map-filter').forEach(cb => {
         const category = cb.getAttribute('data-category');
-        if (category) filters[category] = cb.checked;
+        if (category && !cb.checked) hiddenCategories.add(category);
     });
 
-    // Update marker and circle layers visibility
+    // Rebuild cluster group with only visible markers
-    Object.entries(filters).forEach(([category, show]) => {
+    clusterGroup.clearLayers();
-        if (markerLayers[category]) {
+    const visible = mapMarkers.filter(m => !hiddenCategories.has(m.options.category));
-            if (show) {
+    clusterGroup.addLayers(visible);
-                if (!attackerMap.hasLayer(markerLayers[category])) {
-                    attackerMap.addLayer(markerLayers[category]);
-                }
-            } else {
-                if (attackerMap.hasLayer(markerLayers[category])) {
-                    attackerMap.removeLayer(markerLayers[category]);
-                }
-            }
-        }
-    });
 }
 
 // Generate radar chart SVG for map panel popups