krawl.es/src/config.py

#!/usr/bin/env python3

import os
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Optional, Tuple
from zoneinfo import ZoneInfo
import time

import yaml


@dataclass
class Config:
    """Configuration class for the deception server"""

    port: int = 5000
    delay: int = 100  # milliseconds
    server_header: str = ""
    links_length_range: Tuple[int, int] = (5, 15)
    links_per_page_range: Tuple[int, int] = (10, 15)
    char_space: str = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
    max_counter: int = 10
    canary_token_url: Optional[str] = None
    canary_token_tries: int = 10
    dashboard_secret_path: str = None
    api_server_url: Optional[str] = None
    api_server_port: int = 8080
    api_server_path: str = "/api/v2/users"
    probability_error_codes: int = 0  # Percentage (0-100)

    # Crawl limiting settings - for legitimate vs malicious crawlers
    max_pages_limit: int = (
        100  # Max pages limit for good crawlers and regular users (and bad crawlers/attackers if infinite_pages_for_malicious is False)
    )
    infinite_pages_for_malicious: bool = True  # Infinite pages for malicious crawlers
    ban_duration_seconds: int = 600  # Ban duration in seconds for IPs exceeding limits

    # Database settings
    database_path: str = "data/krawl.db"
    database_retention_days: int = 30

    # Analyzer settings
    http_risky_methods_threshold: float = None
    violated_robots_threshold: float = None
    uneven_request_timing_threshold: float = None
    uneven_request_timing_time_window_seconds: float = None
    user_agents_used_threshold: float = None
    attack_urls_threshold: float = None

    @classmethod
    def from_yaml(cls) -> "Config":
        """Create configuration from YAML file"""
        config_location = os.getenv("CONFIG_LOCATION", "config.yaml")
        config_path = Path(__file__).parent.parent / config_location

        try:
            with open(config_path, "r") as f:
                data = yaml.safe_load(f)
        except FileNotFoundError:
            print(
                f"Error: Configuration file '{config_path}' not found.", file=sys.stderr
            )
            print(
                f"Please create a config.yaml file or set CONFIG_LOCATION environment variable.",
                file=sys.stderr,
            )
            sys.exit(1)
        except yaml.YAMLError as e:
            print(
                f"Error: Invalid YAML in configuration file '{config_path}': {e}",
                file=sys.stderr,
            )
            sys.exit(1)

        if data is None:
            data = {}

        # Extract nested values with defaults
        server = data.get("server", {})
        links = data.get("links", {})
        canary = data.get("canary", {})
        dashboard = data.get("dashboard", {})
        api = data.get("api", {})
        database = data.get("database", {})
        behavior = data.get("behavior", {})
        analyzer = data.get("analyzer") or {}
        crawl = data.get("crawl", {})

        # Handle dashboard_secret_path - auto-generate if null/not set
        dashboard_path = dashboard.get("secret_path")
        if dashboard_path is None:
            dashboard_path = f"/{os.urandom(16).hex()}"
        else:
            # ensure the dashboard path starts with a /
            if dashboard_path[:1] != "/":
                dashboard_path = f"/{dashboard_path}"

        return cls(
            port=server.get("port", 5000),
            delay=server.get("delay", 100),
            server_header=server.get("server_header", ""),
            links_length_range=(
                links.get("min_length", 5),
                links.get("max_length", 15),
            ),
            links_per_page_range=(
                links.get("min_per_page", 10),
                links.get("max_per_page", 15),
            ),
            char_space=links.get(
                "char_space",
                "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789",
            ),
            max_counter=links.get("max_counter", 10),
            canary_token_url=canary.get("token_url"),
            canary_token_tries=canary.get("token_tries", 10),
            dashboard_secret_path=dashboard_path,
            api_server_url=api.get("server_url"),
            api_server_port=api.get("server_port", 8080),
            api_server_path=api.get("server_path", "/api/v2/users"),
            probability_error_codes=behavior.get("probability_error_codes", 0),
            database_path=database.get("path", "data/krawl.db"),
            database_retention_days=database.get("retention_days", 30),
            http_risky_methods_threshold=analyzer.get(
                "http_risky_methods_threshold", 0.1
            ),
            violated_robots_threshold=analyzer.get("violated_robots_threshold", 0.1),
            uneven_request_timing_threshold=analyzer.get(
                "uneven_request_timing_threshold", 0.5
            ),  # coefficient of variation
            uneven_request_timing_time_window_seconds=analyzer.get(
                "uneven_request_timing_time_window_seconds", 300
            ),
            user_agents_used_threshold=analyzer.get("user_agents_used_threshold", 2),
            attack_urls_threshold=analyzer.get("attack_urls_threshold", 1),
            infinite_pages_for_malicious=crawl.get(
                "infinite_pages_for_malicious", True
            ),
            max_pages_limit=crawl.get("max_pages_limit", 200),
            ban_duration_seconds=crawl.get("ban_duration_seconds", 60),
        )


def __get_env_from_config(config: str) -> str:

    env = config.upper().replace(".", "_").replace("-", "__").replace(" ", "_")

    return f"KRAWL_{env}"


def override_config_from_env(config: Config = None):
    """Initialize configuration from environment variables"""

    for field in config.__dataclass_fields__:

        env_var = __get_env_from_config(field)
        if env_var in os.environ:
            field_type = config.__dataclass_fields__[field].type
            env_value = os.environ[env_var]
            if field_type == int:
                setattr(config, field, int(env_value))
            elif field_type == float:
                setattr(config, field, float(env_value))
            elif field_type == Tuple[int, int]:
                parts = env_value.split(",")
                if len(parts) == 2:
                    setattr(config, field, (int(parts[0]), int(parts[1])))
            else:
                setattr(config, field, env_value)


_config_instance = None


def get_config() -> Config:
    """Get the singleton Config instance"""
    global _config_instance
    if _config_instance is None:
        _config_instance = Config.from_yaml()

        override_config_from_env(_config_instance)

    return _config_instance