krawl.es/src/tasks/fetch_ip_rep.py

from database import get_database
from logger import get_app_logger
import requests
from sanitizer import sanitize_for_storage, sanitize_dict

# ----------------------
# TASK CONFIG
# ----------------------

TASK_CONFIG = {
    "name": "fetch-ip-rep",
    "cron": "*/5 * * * *",
    "enabled": True,
    "run_when_loaded": True,
}


def main():
    db_manager = get_database()
    app_logger = get_app_logger()

    # Only get IPs that haven't been enriched yet
    unenriched_ips = db_manager.get_unenriched_ips(limit=50)
    app_logger.info(
        f"{len(unenriched_ips)} IP's need to be have reputation enrichment."
    )
    for ip in unenriched_ips:
        try:
            api_url = "https://iprep.lcrawl.com/api/iprep/"
            params = {"cidr": ip}
            headers = {"Content-Type": "application/json"}
            response = requests.get(api_url, headers=headers, params=params, timeout=10)
            payload = response.json()

            if payload.get("results"):
                data = payload["results"][0]
                geoip_data = data["geoip_data"]
                country_iso_code = geoip_data.get("country_iso_code")
                asn = geoip_data.get("asn_autonomous_system_number")
                asn_org = geoip_data.get("asn_autonomous_system_organization")
                city = geoip_data.get("city_name")  # Extract city name from API
                list_on = data["list_on"]

                sanitized_country_iso_code = sanitize_for_storage(country_iso_code, 3)
                sanitized_asn = sanitize_for_storage(asn, 100)
                sanitized_asn_org = sanitize_for_storage(asn_org, 100)
                sanitized_city = sanitize_for_storage(city, 100) if city else None
                sanitized_list_on = sanitize_dict(list_on, 100000)

                db_manager.update_ip_rep_infos(
                    ip,
                    sanitized_country_iso_code,
                    sanitized_asn,
                    sanitized_asn_org,
                    sanitized_list_on,
                    sanitized_city,  # Pass city to database
                )
        except requests.RequestException as e:
            app_logger.warning(f"Failed to fetch IP rep for {ip}: {e}")
        except Exception as e:
            app_logger.error(f"Error processing IP {ip}: {e}")