feat: enhance IP map visualization with custom pie-chart cluster icons and improved marker handling

- Added custom CSS for pie-chart cluster icons to improve visual representation.
- Implemented a function to create cluster icons using conic gradients based on IP categories.
- Refactored marker creation logic to utilize unique coordinates for overlapping markers.
- Introduced a loading overlay during IP data fetching to enhance user experience.
- Updated map filters to dynamically show/hide markers based on user selection.
- Improved error handling for IP stats fetching and added informative popups.

src/templates/static/css/dashboard.css

@@ -626,6 +626,11 @@ tbody {
 .marker-unknown:hover {
     box-shadow: 0 0 15px rgba(139, 148, 158, 1), inset 0 0 6px rgba(139, 148, 158, 0.7);
 }
+/* Custom pie-chart cluster icons */
+.ip-cluster-icon {
+    background: none !important;
+    border: none !important;
+}
 .leaflet-bottom.leaflet-right {
     display: none !important;
 }

src/templates/static/js/map.js

@@ -3,8 +3,9 @@
 
 let attackerMap = null;
 let allIps = [];
-let mapMarkers = [];
-let markerLayers = {};
+let mapMarkers = [];       // all marker objects, each tagged with .options.category
+let clusterGroup = null;   // single shared MarkerClusterGroup
+let hiddenCategories = new Set();
 
 const categoryColors = {
     attacker: '#f85149',
@@ -14,49 +15,43 @@ const categoryColors = {
     unknown: '#8b949e'
 };
 
-async function initializeAttackerMap() {
-    const DASHBOARD_PATH = window.__DASHBOARD_PATH__ || '';
-    const mapContainer = document.getElementById('attacker-map');
-    if (!mapContainer || attackerMap) return;
-
-    try {
-        // Initialize map
-        attackerMap = L.map('attacker-map', {
-            center: [20, 0],
-            zoom: 2,
-            layers: [
-                L.tileLayer('https://{s}.basemaps.cartocdn.com/dark_all/{z}/{x}/{y}{r}.png', {
-                    attribution: '© CartoDB | © OpenStreetMap contributors',
-                    maxZoom: 19,
-                    subdomains: 'abcd'
-                })
-            ]
+// Build a conic-gradient pie icon showing the category mix inside a cluster
+function createClusterIcon(cluster) {
+    const children = cluster.getAllChildMarkers();
+    const counts = {};
+    children.forEach(m => {
+        const cat = m.options.category || 'unknown';
+        counts[cat] = (counts[cat] || 0) + 1;
     });
 
-        // Fetch all IPs (not just attackers)
-        const response = await fetch(DASHBOARD_PATH + '/api/all-ips?page=1&page_size=100&sort_by=total_requests&sort_order=desc', {
-            cache: 'no-store',
-            headers: {
-                'Cache-Control': 'no-cache',
-                'Pragma': 'no-cache'
-            }
+    const total = children.length;
+    const sorted = Object.entries(counts).sort((a, b) => b[1] - a[1]);
+    let gradientStops = [];
+    let cumulative = 0;
+    sorted.forEach(([cat, count]) => {
+        const start = (cumulative / total) * 360;
+        cumulative += count;
+        const end = (cumulative / total) * 360;
+        const color = categoryColors[cat] || '#8b949e';
+        gradientStops.push(`${color} ${start.toFixed(1)}deg ${end.toFixed(1)}deg`);
     });
 
-        if (!response.ok) throw new Error('Failed to fetch IPs');
+    const size = Math.max(32, Math.min(50, 32 + Math.log2(total) * 5));
+    const inner = size - 10;
+    const offset = 5; // (size - inner) / 2
 
-        const data = await response.json();
-        allIps = data.ips || [];
+    return L.divIcon({
+        html: `<div style="position:relative;width:${size}px;height:${size}px;">` +
+            `<div style="position:absolute;top:0;left:0;width:${size}px;height:${size}px;border-radius:50%;background:conic-gradient(${gradientStops.join(', ')});box-shadow:0 0 6px rgba(0,0,0,0.5);"></div>` +
+            `<div style="position:absolute;top:${offset}px;left:${offset}px;width:${inner}px;height:${inner}px;border-radius:50%;background:rgba(13,17,23,0.85);color:#e6edf3;font-size:11px;font-weight:700;line-height:${inner}px;text-align:center;">${total}</div>` +
+            `</div>`,
+        className: 'ip-cluster-icon',
+        iconSize: L.point(size, size)
+    });
+}
 
-        if (allIps.length === 0) {
-            mapContainer.innerHTML = '<div style="display: flex; align-items: center; justify-content: center; height: 100%; color: #8b949e;">No IP location data available</div>';
-            return;
-        }
-
-        // Get max request count for scaling
-        const maxRequests = Math.max(...allIps.map(ip => ip.total_requests || 0));
-
-        // City coordinates database (major cities worldwide)
-        const cityCoordinates = {
+// City coordinates database (major cities worldwide)
+const cityCoordinates = {
     // United States
     'New York': [40.7128, -74.0060], 'Los Angeles': [34.0522, -118.2437],
     'San Francisco': [37.7749, -122.4194], 'Chicago': [41.8781, -87.6298],
@@ -92,8 +87,8 @@ async function initializeAttackerMap() {
     'Karachi': [24.8607, 67.0011], 'Islamabad': [33.6844, 73.0479],
     'Dhaka': [23.8103, 90.4125], 'Colombo': [6.9271, 79.8612],
     // South America
-            'S\u00e3o Paulo': [-23.5505, -46.6333], 'Rio de Janeiro': [-22.9068, -43.1729],
-            'Buenos Aires': [-34.6037, -58.3816], 'Bogot\u00e1': [4.7110, -74.0721],
+    'São Paulo': [-23.5505, -46.6333], 'Rio de Janeiro': [-22.9068, -43.1729],
+    'Buenos Aires': [-34.6037, -58.3816], 'Bogotá': [4.7110, -74.0721],
     'Lima': [-12.0464, -77.0428], 'Santiago': [-33.4489, -70.6693],
     // Middle East & Africa
     'Cairo': [30.0444, 31.2357], 'Dubai': [25.2048, 55.2708],
@@ -106,10 +101,10 @@ async function initializeAttackerMap() {
     'Auckland': [-36.8485, 174.7633],
     // Additional cities
     'Unknown': null
-        };
+};
 
-        // Country center coordinates (fallback when city not found)
-        const countryCoordinates = {
+// Country center coordinates (fallback when city not found)
+const countryCoordinates = {
     'US': [37.1, -95.7], 'GB': [55.4, -3.4], 'CN': [35.9, 104.1], 'RU': [61.5, 105.3],
     'JP': [36.2, 138.3], 'DE': [51.2, 10.5], 'FR': [46.6, 2.2], 'IN': [20.6, 78.96],
     'BR': [-14.2, -51.9], 'CA': [56.1, -106.3], 'AU': [-25.3, 133.8], 'MX': [23.6, -102.6],
@@ -124,24 +119,74 @@ async function initializeAttackerMap() {
     'FI': [61.9, 25.8], 'NO': [60.5, 8.5], 'GR': [39.1, 21.8], 'PT': [39.4, -8.2],
     'AR': [-38.4161, -63.6167], 'CO': [4.5709, -74.2973], 'CL': [-35.6751, -71.5430],
     'PE': [-9.1900, -75.0152], 'VE': [6.4238, -66.5897], 'LS': [40.0, -100.0]
-        };
+};
 
-        // Helper function to get coordinates for an IP
-        function getIPCoordinates(ip) {
-            // Use actual latitude and longitude if available
+// Helper function to get coordinates for an IP
+function getIPCoordinates(ip) {
     if (ip.latitude != null && ip.longitude != null) {
         return [ip.latitude, ip.longitude];
     }
-            // Fall back to city lookup
     if (ip.city && cityCoordinates[ip.city]) {
         return cityCoordinates[ip.city];
     }
-            // Fall back to country
     if (ip.country_code && countryCoordinates[ip.country_code]) {
         return countryCoordinates[ip.country_code];
     }
     return null;
+}
+
+// Fetch IPs from the API, handling pagination for "all"
+async function fetchIpsForMap(limit) {
+    const DASHBOARD_PATH = window.__DASHBOARD_PATH__ || '';
+    const headers = { 'Cache-Control': 'no-cache', 'Pragma': 'no-cache' };
+
+    if (limit === 'all') {
+        // Fetch in pages of 1000 until we have everything
+        let collected = [];
+        let page = 1;
+        const pageSize = 1000;
+        while (true) {
+            const response = await fetch(
+                `${DASHBOARD_PATH}/api/all-ips?page=${page}&page_size=${pageSize}&sort_by=total_requests&sort_order=desc`,
+                { cache: 'no-store', headers }
+            );
+            if (!response.ok) throw new Error('Failed to fetch IPs');
+            const data = await response.json();
+            collected = collected.concat(data.ips || []);
+            if (page >= data.pagination.total_pages) break;
+            page++;
         }
+        return collected;
+    }
+
+    const pageSize = parseInt(limit, 10);
+    const response = await fetch(
+        `${DASHBOARD_PATH}/api/all-ips?page=1&page_size=${pageSize}&sort_by=total_requests&sort_order=desc`,
+        { cache: 'no-store', headers }
+    );
+    if (!response.ok) throw new Error('Failed to fetch IPs');
+    const data = await response.json();
+    return data.ips || [];
+}
+
+// Build markers from an IP list and add them to the map
+function buildMapMarkers(ips) {
+    // Clear existing cluster group
+    if (clusterGroup) {
+        attackerMap.removeLayer(clusterGroup);
+        clusterGroup.clearLayers();
+    }
+    mapMarkers = [];
+
+    // Single cluster group with custom pie-chart icons
+    clusterGroup = L.markerClusterGroup({
+        maxClusterRadius: 20,
+        spiderfyOnMaxZoom: true,
+        showCoverageOnHover: false,
+        zoomToBoundsOnClick: true,
+        disableClusteringAtZoom: 10,
+        iconCreateFunction: createClusterIcon
+    });
 
     // Track used coordinates to add small offsets for overlapping markers
     const usedCoordinates = {};
@@ -152,15 +197,12 @@ async function initializeAttackerMap() {
         }
         usedCoordinates[key]++;
 
-            // If this is the first marker at this location, use exact coordinates
         if (usedCoordinates[key] === 1) {
             return baseCoords;
         }
 
-            // Add small random offset for subsequent markers
-            // Offset increases with each marker to create a spread pattern
-            const angle = (usedCoordinates[key] * 137.5) % 360; // Golden angle for even distribution
-            const distance = 0.05 * Math.sqrt(usedCoordinates[key]); // Increase distance with more markers
+        const angle = (usedCoordinates[key] * 137.5) % 360;
+        const distance = 0.05 * Math.sqrt(usedCoordinates[key]);
         const latOffset = distance * Math.cos(angle * Math.PI / 180);
         const lngOffset = distance * Math.sin(angle * Math.PI / 180);
 
@@ -170,36 +212,22 @@ async function initializeAttackerMap() {
         ];
     }
 
-        // Create layer groups for each category
-        markerLayers = {
-            attacker: L.featureGroup(),
-            bad_crawler: L.featureGroup(),
-            good_crawler: L.featureGroup(),
-            regular_user: L.featureGroup(),
-            unknown: L.featureGroup()
-        };
+    const DASHBOARD_PATH = window.__DASHBOARD_PATH__ || '';
 
-        // Add markers for each IP
-        allIps.slice(0, 100).forEach(ip => {
+    ips.forEach(ip => {
         if (!ip.country_code || !ip.category) return;
 
-            // Get coordinates (city first, then country)
         const baseCoords = getIPCoordinates(ip);
         if (!baseCoords) return;
 
-            // Get unique coordinates with offset to prevent overlap
         const coords = getUniqueCoordinates(baseCoords);
-
         const category = ip.category.toLowerCase();
-            if (!markerLayers[category]) return;
+        if (!categoryColors[category]) return;
 
-            // Calculate marker size based on request count with more dramatic scaling
-            // Scale up to 10,000 requests, then cap it
         const requestsForScale = Math.min(ip.total_requests, 10000);
-            const sizeRatio = Math.pow(requestsForScale / 10000, 0.5); // Square root for better visual scaling
+        const sizeRatio = Math.pow(requestsForScale / 10000, 0.5);
         const markerSize = Math.max(10, Math.min(30, 10 + (sizeRatio * 20)));
 
-            // Create custom marker element with category-specific class
         const markerElement = document.createElement('div');
         markerElement.className = `ip-marker marker-${category}`;
         markerElement.style.width = markerSize + 'px';
@@ -212,10 +240,10 @@ async function initializeAttackerMap() {
                 html: markerElement.outerHTML,
                 iconSize: [markerSize, markerSize],
                 className: `ip-custom-marker category-${category}`
-                })
+            }),
+            category: category
         });
 
-            // Create popup with category badge and chart
         const categoryColor = categoryColors[category] || '#8b949e';
         const categoryLabels = {
             attacker: 'Attacker',
@@ -225,15 +253,12 @@ async function initializeAttackerMap() {
             unknown: 'Unknown'
         };
 
-            // Bind popup once when marker is created
         marker.bindPopup('', {
             maxWidth: 550,
             className: 'ip-detail-popup'
         });
 
-            // Add click handler to fetch data and show popup
         marker.on('click', async function(e) {
-                // Show loading popup first
             const loadingPopup = `
                 <div style="padding: 12px; min-width: 280px; max-width: 320px;">
                     <div style="display: flex; align-items: center; justify-content: space-between; margin-bottom: 8px;">
@@ -252,14 +277,11 @@ async function initializeAttackerMap() {
             marker.openPopup();
 
             try {
-                    console.log('Fetching IP stats for:', ip.ip);
                 const response = await fetch(`${DASHBOARD_PATH}/api/ip-stats/${ip.ip}`);
                 if (!response.ok) throw new Error('Failed to fetch IP stats');
 
                 const stats = await response.json();
-                    console.log('Received stats:', stats);
 
-                    // Build complete popup content with chart
                 let popupContent = `
                     <div style="padding: 12px; min-width: 200px;">
                         <div style="display: flex; align-items: center; justify-content: space-between; margin-bottom: 8px;">
@@ -278,11 +300,8 @@ async function initializeAttackerMap() {
                         </div>
                 `;
 
-                    // Add chart if category scores exist
                 if (stats.category_scores && Object.keys(stats.category_scores).length > 0) {
-                        console.log('Category scores found:', stats.category_scores);
                     const chartHtml = generateMapPanelRadarChart(stats.category_scores);
-                        console.log('Generated chart HTML length:', chartHtml.length);
                     popupContent += `
                         <div style="margin-top: 12px; border-top: 1px solid #30363d; padding-top: 12px;">
                             ${chartHtml}
@@ -291,9 +310,6 @@ async function initializeAttackerMap() {
                 }
 
                 popupContent += '</div>';
-
-                    // Update popup content
-                    console.log('Updating popup content');
                 marker.setPopupContent(popupContent);
             } catch (err) {
                 console.error('Error fetching IP stats:', err);
@@ -322,25 +338,54 @@ async function initializeAttackerMap() {
             }
         });
 
-            markerLayers[category].addLayer(marker);
+        mapMarkers.push(marker);
+        // Only add to cluster if category is not hidden
+        if (!hiddenCategories.has(category)) {
+            clusterGroup.addLayer(marker);
+        }
     });
 
-        // Add all marker layers to map initially
-        Object.values(markerLayers).forEach(layer => attackerMap.addLayer(layer));
+    attackerMap.addLayer(clusterGroup);
 
-        // Fit map to all markers
-        const allMarkers = Object.values(markerLayers).reduce((acc, layer) => {
-            acc.push(...layer.getLayers());
-            return acc;
-        }, []);
-
-        if (allMarkers.length > 0) {
-            const bounds = L.featureGroup(allMarkers).getBounds();
+    // Fit map to visible markers
+    const visibleMarkers = mapMarkers.filter(m => !hiddenCategories.has(m.options.category));
+    if (visibleMarkers.length > 0) {
+        const bounds = L.featureGroup(visibleMarkers).getBounds();
         attackerMap.fitBounds(bounds, { padding: [50, 50] });
     }
+}
+
+async function initializeAttackerMap() {
+    const mapContainer = document.getElementById('attacker-map');
+    if (!mapContainer || attackerMap) return;
+
+    try {
+        attackerMap = L.map('attacker-map', {
+            center: [20, 0],
+            zoom: 2,
+            layers: [
+                L.tileLayer('https://{s}.basemaps.cartocdn.com/dark_all/{z}/{x}/{y}{r}.png', {
+                    attribution: '&copy; CartoDB | &copy; OpenStreetMap contributors',
+                    maxZoom: 19,
+                    subdomains: 'abcd'
+                })
+            ]
+        });
+
+        // Get the selected limit from the dropdown (default 100)
+        const limitSelect = document.getElementById('map-ip-limit');
+        const limit = limitSelect ? limitSelect.value : '100';
+
+        allIps = await fetchIpsForMap(limit);
+
+        if (allIps.length === 0) {
+            mapContainer.innerHTML = '<div style="display: flex; align-items: center; justify-content: center; height: 100%; color: #8b949e;">No IP location data available</div>';
+            return;
+        }
+
+        buildMapMarkers(allIps);
 
         // Force Leaflet to recalculate container size after the tab becomes visible.
-        // Without this, tiles may not render correctly when the container was hidden.
         setTimeout(() => {
             if (attackerMap) attackerMap.invalidateSize();
         }, 300);
@@ -351,30 +396,44 @@ async function initializeAttackerMap() {
     }
 }
 
-// Update map filters based on checkbox selection
-function updateMapFilters() {
+// Reload map markers when the user changes the IP limit selector
+async function reloadMapWithLimit(limit) {
     if (!attackerMap) return;
 
-    const filters = {};
+    // Show loading state
+    const mapContainer = document.getElementById('attacker-map');
+    const overlay = document.createElement('div');
+    overlay.id = 'map-loading-overlay';
+    overlay.style.cssText = 'position:absolute;top:0;left:0;right:0;bottom:0;background:rgba(13,17,23,0.7);display:flex;align-items:center;justify-content:center;z-index:1000;color:#8b949e;font-size:14px;';
+    overlay.textContent = 'Loading IPs...';
+    mapContainer.style.position = 'relative';
+    mapContainer.appendChild(overlay);
+
+    try {
+        allIps = await fetchIpsForMap(limit);
+        buildMapMarkers(allIps);
+    } catch (err) {
+        console.error('Error reloading map:', err);
+    } finally {
+        const existing = document.getElementById('map-loading-overlay');
+        if (existing) existing.remove();
+    }
+}
+
+// Update map filters based on checkbox selection
+function updateMapFilters() {
+    if (!attackerMap || !clusterGroup) return;
+
+    hiddenCategories.clear();
     document.querySelectorAll('.map-filter').forEach(cb => {
         const category = cb.getAttribute('data-category');
-        if (category) filters[category] = cb.checked;
+        if (category && !cb.checked) hiddenCategories.add(category);
     });
 
-    // Update marker and circle layers visibility
-    Object.entries(filters).forEach(([category, show]) => {
-        if (markerLayers[category]) {
-            if (show) {
-                if (!attackerMap.hasLayer(markerLayers[category])) {
-                    attackerMap.addLayer(markerLayers[category]);
-                }
-            } else {
-                if (attackerMap.hasLayer(markerLayers[category])) {
-                    attackerMap.removeLayer(markerLayers[category]);
-                }
-            }
-        }
-    });
+    // Rebuild cluster group with only visible markers
+    clusterGroup.clearLayers();
+    const visible = mapMarkers.filter(m => !hiddenCategories.has(m.options.category));
+    clusterGroup.addLayers(visible);
 }
 
 // Generate radar chart SVG for map panel popups