domnitor/app/Services/TldRegistryService.php

<?php

namespace App\Services;

use App\Models\TldRegistry;
use App\Models\TldImportLog;
use App\Services\Logger;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\RequestException;

class TldRegistryService
{
    private Client $httpClient;
    private TldRegistry $tldModel;
    private TldImportLog $importLogModel;
    private Logger $logger;
    private static ?bool $brotliSupported = null;

    // IANA URLs
    private const IANA_RDAP_URL = 'https://data.iana.org/rdap/dns.json';
    private const IANA_TLD_BASE_URL = 'https://www.iana.org/domains/root/db/';
    private const IANA_TLD_LIST_URL = 'https://data.iana.org/TLD/tlds-alpha-by-domain.txt';
    private const IANA_RDAP_DOMAIN_URL = 'https://rdap.iana.org/domain/';

    /**
     * Check if Brotli compression is supported by the system
     */
    private static function isBrotliSupported(): bool
    {
        static $supported = null;
        if ($supported !== null) return $supported;
        
        $logger = new Logger('tld_import');
        $detectionResults = [];
        
        // Check for brotli extension
        $brotliExtension = extension_loaded('brotli');
        $detectionResults['brotli_extension'] = $brotliExtension;
        
        // Check for brotli function
        $brotliFunction = function_exists('brotli_compress');
        $detectionResults['brotli_function'] = $brotliFunction;
        
        if ($brotliExtension || $brotliFunction) {
            $logger->info("Brotli support detected", $detectionResults);
            return $supported = true;
        }
        
        // Check curl support for brotli
        try {
            $curlInfo = curl_version();
            $curlVersion = $curlInfo['version'] ?? 'unknown';
            $curlEncoding = $curlInfo['encoding'] ?? 'unknown';
            $curlFeatures = $curlInfo['features'] ?? 0;
            
            $detectionResults['curl_version'] = $curlVersion;
            $detectionResults['curl_encoding'] = $curlEncoding;
            $detectionResults['curl_features_raw'] = $curlFeatures;
            
            // Method 1: Check encoding field
            $encodingSupportsBrotli = stripos($curlEncoding, 'br') !== false;
            $detectionResults['curl_encoding_check'] = $encodingSupportsBrotli;
            
            // Method 2: Check features bitmask (CURL_VERSION_BROTLI = 8388608)
            $featuresSupportsBrotli = ($curlFeatures & 8388608) !== 0;
            $detectionResults['curl_features_check'] = $featuresSupportsBrotli;
            
            // Method 3: Check if curl version is recent enough (brotli support added in 7.57.0)
            $versionSupportsBrotli = version_compare($curlVersion, '7.57.0', '>=');
            $detectionResults['curl_version_check'] = $versionSupportsBrotli;
            
            if ($encodingSupportsBrotli || $featuresSupportsBrotli || $versionSupportsBrotli) {
                $detectionResults['final_decision'] = 'supported';
                $logger->info("Brotli support detected via cURL", $detectionResults);
                return $supported = true;
            }
        } catch (\Exception $e) {
            $detectionResults['curl_error'] = $e->getMessage();
        }
        
        $logger->info("Brotli support NOT detected - using gzip/deflate fallback", $detectionResults);
        return $supported = false;
    }

    /**
     * Log HTTP request and response details for debugging
     */
    private function logHttpRequest(string $method, string $url, array $headers, int $statusCode, array $responseHeaders, int $contentLength, float $requestTime): void
    {
        $acceptEncoding = $headers['Accept-Encoding'] ?? 'not-set';
        $contentEncoding = $responseHeaders['content-encoding'] ?? 'none';
        $contentType = $responseHeaders['content-type'] ?? 'unknown';
        
        $logData = [
            'method' => $method,
            'url' => $url,
            'request_accept_encoding' => $acceptEncoding,
            'response_status' => $statusCode,
            'response_content_encoding' => $contentEncoding,
            'response_content_type' => $contentType,
            'response_content_length' => $contentLength,
            'request_time_seconds' => round($requestTime, 3),
            'brotli_supported' => self::isBrotliSupported(),
            'compression_used' => $contentEncoding !== 'none' ? $contentEncoding : 'none'
        ];
        
        if ($statusCode >= 200 && $statusCode < 300) {
            $this->logger->info("HTTP request successful", $logData);
        } else {
            $this->logger->warning("HTTP request failed", $logData);
        }
    }

    public function __construct()
    {
        $acceptEncoding = self::isBrotliSupported() ? 'gzip, deflate, br' : 'gzip, deflate';
        
        $this->tldModel = new TldRegistry();
        $this->importLogModel = new TldImportLog();
        $this->logger = new Logger('tld_import');
        
        $this->logger->debug("Creating main HTTP client", [
            'accept_encoding' => $acceptEncoding,
            'brotli_supported' => self::isBrotliSupported()
        ]);
        
        $this->httpClient = new Client([
            'timeout' => 15, // Reduced for faster processing
            'connect_timeout' => 5, // Reduced for faster processing
            'verify' => true, // Enable SSL verification
            'allow_redirects' => [
                'max' => 5,
                'strict' => false,
                'referer' => true,
                'protocols' => ['http', 'https']
            ],
            'headers' => [
                'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
                'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
                'Accept-Language' => 'en-US,en;q=0.9',
                'Accept-Encoding' => $acceptEncoding,
                'DNT' => '1',
                'Connection' => 'keep-alive',
                'Upgrade-Insecure-Requests' => '1',
                'Sec-Fetch-Dest' => 'document',
                'Sec-Fetch-Mode' => 'navigate',
                'Sec-Fetch-Site' => 'none',
                'Cache-Control' => 'max-age=0'
            ],
            'on_stats' => function ($stats) {
                $this->logHttpRequest(
                    $stats->getRequest()->getMethod(),
                    (string) $stats->getRequest()->getUri(),
                    $stats->getRequest()->getHeaders(),
                    $stats->getResponse()->getStatusCode(),
                    $stats->getResponse()->getHeaders(),
                    $stats->getResponse()->getBody()->getSize() ?? 0,
                    $stats->getTransferTime()
                );
            }
        ]);
    }

    /**
     * Get HTTP client configured for JSON requests
     */
    private function getJsonClient(): Client
    {
        $acceptEncoding = self::isBrotliSupported() ? 'gzip, deflate, br' : 'gzip, deflate';
        
        $this->logger->debug("Creating JSON HTTP client", [
            'accept_encoding' => $acceptEncoding,
            'brotli_supported' => self::isBrotliSupported()
        ]);
        
        return new Client([
            'timeout' => 15, // Reduced for faster processing
            'connect_timeout' => 5, // Reduced for faster processing
            'verify' => true,
            'allow_redirects' => [
                'max' => 3,
                'strict' => true,
                'referer' => false,
                'protocols' => ['https']
            ],
            'headers' => [
                'User-Agent' => 'DomainMonitor/1.0 (TLD Registry Bot; compatible with IANA RDAP)',
                'Accept' => 'application/json, application/rdap+json, */*;q=0.8',
                'Accept-Language' => 'en-US,en;q=0.9',
                'Accept-Encoding' => $acceptEncoding,
                'Connection' => 'keep-alive',
                'Cache-Control' => 'no-cache'
            ],
            'http_errors' => false, // Don't throw exceptions on HTTP error codes
            'retry' => [
                'max' => 2, // Reduced retries for speed
                'delay' => 500, // 0.5 second delay between retries (reduced)
                'multiplier' => 1.5
            ],
            'on_stats' => function ($stats) {
                $this->logHttpRequest(
                    $stats->getRequest()->getMethod(),
                    (string) $stats->getRequest()->getUri(),
                    $stats->getRequest()->getHeaders(),
                    $stats->getResponse()->getStatusCode(),
                    $stats->getResponse()->getHeaders(),
                    $stats->getResponse()->getBody()->getSize() ?? 0,
                    $stats->getTransferTime()
                );
            }
        ]);
    }

    /**
     * Get HTTP client configured for HTML requests
     */
    private function getHtmlClient(): Client
    {
        $acceptEncoding = self::isBrotliSupported() ? 'gzip, deflate, br' : 'gzip, deflate';
        
        $this->logger->debug("Creating HTML HTTP client", [
            'accept_encoding' => $acceptEncoding,
            'brotli_supported' => self::isBrotliSupported()
        ]);
        
        return new Client([
            'timeout' => 8, // Further reduced for faster processing
            'connect_timeout' => 3, // Further reduced for faster processing
            'verify' => true,
            'allow_redirects' => [
                'max' => 3, // Reduced redirects
                'strict' => false,
                'referer' => true,
                'protocols' => ['http', 'https']
            ],
            'headers' => [
                'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
                'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
                'Accept-Language' => 'en-US,en;q=0.9',
                'Accept-Encoding' => $acceptEncoding,
                'DNT' => '1',
                'Connection' => 'keep-alive',
                'Upgrade-Insecure-Requests' => '1',
                'Sec-Fetch-Dest' => 'document',
                'Sec-Fetch-Mode' => 'navigate',
                'Sec-Fetch-Site' => 'none',
                'Cache-Control' => 'max-age=0'
            ],
            'http_errors' => false, // Don't throw exceptions on HTTP error codes
            'retry' => [
                'max' => 0, // No retries for HTML to avoid timeouts
                'delay' => 0,
                'multiplier' => 1
            ],
            'on_stats' => function ($stats) {
                $this->logHttpRequest(
                    $stats->getRequest()->getMethod(),
                    (string) $stats->getRequest()->getUri(),
                    $stats->getRequest()->getHeaders(),
                    $stats->getResponse()->getStatusCode(),
                    $stats->getResponse()->getHeaders(),
                    $stats->getResponse()->getBody()->getSize() ?? 0,
                    $stats->getTransferTime()
                );
            }
        ]);
    }

    /**
     * Import TLD list from IANA
     */
    public function importTldList(): array
    {
        $logId = $this->importLogModel->startImport('tld_list');
        $stats = [
            'total_tlds' => 0,
            'new_tlds' => 0,
            'updated_tlds' => 0,
            'failed_tlds' => 0
        ];

        try {
            // Fetch TLD list from IANA
            $jsonClient = $this->getJsonClient();
            $response = $jsonClient->get(self::IANA_TLD_LIST_URL);
            
            // Check response status
            if ($response->getStatusCode() !== 200) {
                throw new \Exception('Failed to fetch TLD list: HTTP ' . $response->getStatusCode());
            }
            
            $content = $response->getBody()->getContents();
            
            // Parse the content to extract version and TLDs
            $lines = explode("\n", $content);
            $version = null;
            $lastUpdated = null;
            $tlds = [];
            
            foreach ($lines as $line) {
                $line = trim($line);
                
                // Skip empty lines
                if (empty($line)) {
                    continue;
                }
                
                // Extract version and timestamp from header
                if (strpos($line, '# Version') === 0) {
                    if (preg_match('/# Version (\d+), Last Updated (.+)/', $line, $matches)) {
                        $version = $matches[1];
                        $lastUpdatedRaw = $matches[2];
                        
                        // Convert the timestamp to a proper format
                        try {
                            $lastUpdated = date('Y-m-d H:i:s', strtotime($lastUpdatedRaw));
                            if ($lastUpdated === '1970-01-01 00:00:00') {
                                // If strtotime fails, keep the raw value
                                $lastUpdated = $lastUpdatedRaw;
                            }
                        } catch (\Exception $e) {
                            $lastUpdated = $lastUpdatedRaw;
                        }
                    }
                    continue;
                }
                
                // Skip comment lines
                if (strpos($line, '#') === 0) {
                    continue;
                }
                
                // Add TLD to list (ensure it starts with dot)
                $tld = '.' . strtolower($line);
                $tlds[] = $tld;
            }
            
            if (empty($tlds)) {
                throw new \Exception('No TLDs found in the response');
            }
            
            $stats['total_tlds'] = count($tlds);
            
            // Normalize last updated date to UTC format
            $normalizedLastUpdated = $this->normalizeDate($lastUpdated);
            
            // Update log with version and timestamp
            $this->importLogModel->update($logId, [
                'iana_publication_date' => $normalizedLastUpdated,
                'version' => $version
            ]);
            
            // Process each TLD
            foreach ($tlds as $tld) {
                try {
                    $result = $this->processTldEntry($tld);
                    
                    if ($result['is_new']) {
                        $stats['new_tlds']++;
                    } else {
                        $stats['updated_tlds']++;
                    }
                } catch (\Exception $e) {
                    $stats['failed_tlds']++;
                    error_log("Failed to process TLD $tld: " . $e->getMessage());
                }
            }

            $this->importLogModel->completeImport($logId, $stats);
            
        } catch (\Exception $e) {
            $this->importLogModel->completeImport($logId, $stats, $e->getMessage());
            throw $e;
        }

        return $stats;
    }

    /**
     * Import RDAP data from IANA
     */
    public function importRdapData(): array
    {
        $logId = $this->importLogModel->startImport('rdap');
        $stats = [
            'total_tlds' => 0,
            'new_tlds' => 0,
            'updated_tlds' => 0,
            'failed_tlds' => 0
        ];

        try {
            // Fetch RDAP data from IANA using JSON client
            $jsonClient = $this->getJsonClient();
            $response = $jsonClient->get(self::IANA_RDAP_URL);
            
            // Check response status
            if ($response->getStatusCode() !== 200) {
                throw new \Exception('Failed to fetch RDAP data: HTTP ' . $response->getStatusCode());
            }
            
            $data = json_decode($response->getBody()->getContents(), true);

            if (!$data || !isset($data['services'])) {
                throw new \Exception('Invalid RDAP data format or empty response');
            }

            $publicationDate = $data['publication'] ?? null;
            $services = $data['services'] ?? [];

            // Normalize publication date to UTC format before saving
            $normalizedPublicationDate = $this->normalizeDate($publicationDate);

            // Update log with publication date
            $this->importLogModel->update($logId, ['iana_publication_date' => $normalizedPublicationDate]);

            foreach ($services as $service) {
                $tlds = $service[0] ?? []; // TLD patterns
                $rdapServers = $service[1] ?? []; // RDAP servers

                foreach ($tlds as $tld) {
                    $stats['total_tlds']++;
                    
                    try {
                        $result = $this->processTldRdapData($tld, $rdapServers, $normalizedPublicationDate);
                        
                        if ($result['is_new']) {
                            $stats['new_tlds']++;
                        } else {
                            $stats['updated_tlds']++;
                        }
                    } catch (\Exception $e) {
                        $stats['failed_tlds']++;
                        error_log("Failed to process TLD $tld: " . $e->getMessage());
                    }
                }
            }

            $this->importLogModel->completeImport($logId, $stats);
            
        } catch (\Exception $e) {
            $this->importLogModel->completeImport($logId, $stats, $e->getMessage());
            throw $e;
        }

        return $stats;
    }

    /**
     * Import WHOIS data for TLDs missing WHOIS servers or needing updates
     */
    public function importWhoisDataForMissingTlds(): array
    {
        $logId = $this->importLogModel->startImport('whois');
        $stats = [
            'total_tlds' => 0,
            'new_tlds' => 0,
            'updated_tlds' => 0,
            'failed_tlds' => 0
        ];

        try {
            // Get TLDs that need WHOIS data (missing WHOIS server or old data)
            $tldsNeedingWhois = $this->getTldsNeedingWhoisData();
            
            foreach ($tldsNeedingWhois as $index => $tld) {
                $stats['total_tlds']++;
                
                try {
                    $result = $this->fetchWhoisDataFromRdap($tld['tld']);
                    
                    if ($result) {
                        $this->tldModel->update($tld['id'], $result);
                        $stats['updated_tlds']++;
                    } else {
                        $stats['failed_tlds']++;
                    }
                } catch (\Exception $e) {
                    $stats['failed_tlds']++;
                    error_log("Failed to fetch WHOIS data for TLD {$tld['tld']}: " . $e->getMessage());
                }
                
                // Add delay between requests to be respectful to IANA servers
                if ($index < count($tldsNeedingWhois) - 1) {
                    usleep(500000); // 0.5 second delay
                }
            }

            $this->importLogModel->completeImport($logId, $stats);
            
        } catch (\Exception $e) {
            $this->importLogModel->completeImport($logId, $stats, $e->getMessage());
            throw $e;
        }

        return $stats;
    }

    /**
     * Get TLDs that need WHOIS data (missing or outdated)
     */
    private function getTldsNeedingWhoisData(int $limit = 100, int $startFromId = 0): array
    {
        // Process ALL TLDs systematically (A to Z, or ID 1 to last ID)
        // This ensures we get complete data for every TLD, even if some don't have WHOIS/RDAP data
        $sql = "SELECT * FROM tld_registry 
                WHERE is_active = 1 
                AND id > " . intval($startFromId) . "
                ORDER BY 
                    CASE 
                        WHEN (whois_server IS NULL OR whois_server = '') AND (registry_url IS NULL OR registry_url = '') THEN 0
                        WHEN whois_server IS NULL OR whois_server = '' THEN 1
                        WHEN registry_url IS NULL OR registry_url = '' THEN 2
                        WHEN registration_date IS NULL OR record_last_updated IS NULL THEN 3
                        ELSE 4
                    END,
                    id ASC
                LIMIT " . intval($limit);
        
        // Use the model's database connection through a public method
        return $this->tldModel->query($sql);
    }

    /**
     * Get the highest ID of processed TLDs for this import session
     */
    private function getLastProcessedTldId(int $logId): int
    {
        // Get the last processed TLD ID from the import log details
        $log = $this->importLogModel->find($logId);
        $details = $log['details'] ? json_decode($log['details'], true) : [];
        
        $lastId = $details['last_processed_id'] ?? 0;
        
        $this->logger->debug("Retrieved last_processed_id from database", [
            'log_id' => $logId,
            'last_processed_id' => $lastId,
            'details_raw' => $log['details'] ?? 'NULL',
            'details_parsed_keys' => array_keys($details)
        ]);
        
        return $lastId;
    }

    /**
     * Set the last processed TLD ID for this import session
     */
    private function setLastProcessedTldId(int $logId, int $lastId): void
    {
        $log = $this->importLogModel->find($logId);
        $details = $log['details'] ? json_decode($log['details'], true) : [];
        $details['last_processed_id'] = $lastId;
        
        $this->logger->debug("Updating last_processed_id", [
            'log_id' => $logId,
            'last_id' => $lastId,
            'full_details' => $details
        ]);
        
        // Fix: Pass details in the data array directly to avoid empty array issue
        $updateResult = $this->importLogModel->update($logId, ['details' => json_encode($details)]);
        
        // Verify the update worked
        if (!$updateResult) {
            $this->logger->error("Failed to update last_processed_id in database!", [
                'log_id' => $logId,
                'last_id' => $lastId
            ]);
        } else {
            // Verify by reading it back
            $verifyLog = $this->importLogModel->find($logId);
            $verifyDetails = $verifyLog['details'] ? json_decode($verifyLog['details'], true) : [];
            $verifiedId = $verifyDetails['last_processed_id'] ?? 0;
            
            if ($verifiedId !== $lastId) {
                $this->logger->critical("Database verification FAILED! last_processed_id mismatch", [
                    'expected' => $lastId,
                    'actual' => $verifiedId,
                    'log_id' => $logId
                ]);
            } else {
                $this->logger->debug("Database update verified successfully", [
                    'log_id' => $logId,
                    'verified_id' => $verifiedId
                ]);
            }
        }
    }

    /**
     * Fetch registry URL from IANA RDAP API
     */
    private function fetchRegistryUrlFromRdap(string $tld): ?string
    {
        $tldForUrl = ltrim($tld, '.');
        $url = self::IANA_RDAP_DOMAIN_URL . $tldForUrl;

        try {
            $jsonClient = $this->getJsonClient();
            $response = $jsonClient->get($url);
            
            if ($response->getStatusCode() !== 200) {
                return null;
            }
            
            $data = json_decode($response->getBody()->getContents(), true);
            
            if (isset($data['links']) && is_array($data['links'])) {
                foreach ($data['links'] as $link) {
                    if (isset($link['rel']) && $link['rel'] === 'related' && 
                        isset($link['title']) && $link['title'] === 'Registration URL') {
                        return $link['href'] ?? null;
                    }
                }
            }
            
        } catch (\Exception $e) {
            error_log("Failed to fetch RDAP registry URL for TLD $tld: " . $e->getMessage());
        }

        return null;
    }

    /**
     * Get count of TLDs that need WHOIS data
     */
    public function getTldsNeedingWhoisCount(int $logId = null): int
    {
        if ($logId) {
            // For a specific import session, count TLDs that haven't been processed yet
            $lastProcessedId = $this->getLastProcessedTldId($logId);
            $sql = "SELECT COUNT(*) as count FROM tld_registry WHERE is_active = 1 AND id > " . intval($lastProcessedId);
        } else {
            // Count ALL active TLDs since we process them all systematically
            $sql = "SELECT COUNT(*) as count FROM tld_registry WHERE is_active = 1";
        }
        
        $result = $this->tldModel->query($sql);
        return $result[0]['count'] ?? 0;
    }

    /**
     * Start progressive import for any type
     */
    public function startProgressiveImport(string $importType): array
    {
        $logId = $this->importLogModel->startImport($importType);
        
        switch ($importType) {
            case 'tld_list':
                $total = $this->getTotalTldsFromIana();
                $message = "Started TLD list import";
                break;
                
            case 'rdap':
                $total = $this->tldModel->getStatistics()['total'];
                $message = "Started RDAP import for {$total} TLDs";
                break;
                
            case 'whois':
                $total = $this->getTldsNeedingWhoisCount();
                if ($total === 0) {
                    return [
                        'status' => 'complete',
                        'message' => 'All TLDs already have WHOIS data',
                        'total' => 0,
                        'processed' => 0,
                        'remaining' => 0
                    ];
                }
                $message = "Started WHOIS import for {$total} TLDs";
                break;
                
            case 'check_updates':
                $total = 2; // TLD list + RDAP
                $message = "Started update check";
                break;
                
            case 'complete_workflow':
                $total = 4; // TLD list + RDAP + WHOIS + Registry URLs
                $message = "Started complete TLD import workflow";
                break;
                
            default:
                throw new \Exception("Unknown import type: {$importType}");
        }
        
        return [
            'status' => 'started',
            'log_id' => $logId,
            'import_type' => $importType,
            'total' => $total,
            'processed' => 0,
            'remaining' => $total,
            'message' => $message
        ];
    }

    /**
     * Get total TLDs from IANA (for TLD list import)
     */
    private function getTotalTldsFromIana(): int
    {
        try {
            $jsonClient = $this->getJsonClient();
            $response = $jsonClient->get(self::IANA_TLD_LIST_URL);
            
            if ($response->getStatusCode() !== 200) {
                return 0;
            }
            
            $content = $response->getBody()->getContents();
            $lines = explode("\n", $content);
            $count = 0;
            
            foreach ($lines as $line) {
                $line = trim($line);
                if (!empty($line) && strpos($line, '#') !== 0) {
                    $count++;
                }
            }
            
            return $count;
        } catch (\Exception $e) {
            return 0;
        }
    }

    /**
     * Process next batch of imports (universal)
     */
    public function processNextBatch(int $logId): array
    {
        // Get import type from log
        $log = $this->importLogModel->find($logId);
        if (!$log) {
            return ['status' => 'error', 'message' => 'Import log not found'];
        }
        
        $importType = $log['import_type'];
        
        switch ($importType) {
            case 'tld_list':
                return $this->processTldListBatch($logId);
            case 'rdap':
                return $this->processRdapBatch($logId);
            case 'whois':
                return $this->processWhoisBatch($logId);
            case 'check_updates':
                return $this->processCheckUpdatesBatch($logId);
            case 'complete_workflow':
                return $this->processCompleteWorkflowBatch($logId);
            default:
                return ['status' => 'error', 'message' => 'Unknown import type'];
        }
    }

    /**
     * Process TLD list batch
     */
    private function processTldListBatch(int $logId): array
    {
        // Get current progress from log
        $log = $this->importLogModel->find($logId);
        $currentProgress = $log['details'] ? json_decode($log['details'], true) : ['processed' => 0, 'failed' => 0];
        
        try {
            // Process TLD list in one go (it's already fast)
            $stats = $this->importTldList();
            
            // Update progress
            $currentProgress['processed'] += $stats['new_tlds'] + $stats['updated_tlds'];
            $currentProgress['failed'] += $stats['failed_tlds'];
            
            $this->importLogModel->completeImport($logId, $stats, 'completed', null, $currentProgress);
            
            return [
                'status' => 'complete',
                'log_id' => $logId,
                'total' => $stats['total_tlds'],
                'processed' => $currentProgress['processed'],
                'failed' => $currentProgress['failed'],
                'remaining' => 0,
                'message' => 'TLD list import completed'
            ];
        } catch (\Exception $e) {
            $this->importLogModel->completeImport($logId, [
                'total_tlds' => 0,
                'new_tlds' => 0,
                'updated_tlds' => 0,
                'failed_tlds' => 1
            ], 'failed', $e->getMessage());
            
            return [
                'status' => 'error',
                'message' => 'TLD list import failed: ' . $e->getMessage()
            ];
        }
    }

    /**
     * Process RDAP batch
     */
    private function processRdapBatch(int $logId): array
    {
        // Get current progress from log
        $log = $this->importLogModel->find($logId);
        $currentProgress = $log['details'] ? json_decode($log['details'], true) : ['processed' => 0, 'failed' => 0, 'total' => 0];
        
        // If this is the first batch, get total count
        if ($currentProgress['total'] == 0) {
            $currentProgress['total'] = $this->tldModel->getStatistics()['total'];
        }
        
        try {
            // Process RDAP data in one go (it's already fast)
            $stats = $this->importRdapData();
            
            // Update progress
            $currentProgress['processed'] += $stats['updated_tlds'];
            $currentProgress['failed'] += $stats['failed_tlds'];
            
            $this->importLogModel->completeImport($logId, $stats, 'completed', null, $currentProgress);
            
            return [
                'status' => 'complete',
                'log_id' => $logId,
                'total' => $currentProgress['total'],
                'processed' => $currentProgress['processed'],
                'failed' => $currentProgress['failed'],
                'remaining' => 0,
                'message' => 'RDAP import completed'
            ];
        } catch (\Exception $e) {
            $this->importLogModel->completeImport($logId, [
                'total_tlds' => 0,
                'new_tlds' => 0,
                'updated_tlds' => 0,
                'failed_tlds' => 1
            ], 'failed', $e->getMessage());
            
            return [
                'status' => 'error',
                'message' => 'RDAP import failed: ' . $e->getMessage()
            ];
        }
    }

    /**
     * Process WHOIS batch
     */
    private function processWhoisBatch(int $logId): array
    {
        $batchStartTime = microtime(true);
        $this->logger->startOperation("WHOIS Batch Processing", ['log_id' => $logId]);
        
        // Get current progress from log
        $log = $this->importLogModel->find($logId);
        $currentProgress = $log['details'] ? json_decode($log['details'], true) : ['processed' => 0, 'failed' => 0, 'total' => 0];
        
        $this->logger->info("Current progress retrieved", $currentProgress);
        
        // If this is the first batch, get total count
        if ($currentProgress['total'] == 0) {
            $currentProgress['total'] = $this->getTldsNeedingWhoisCount();
            $this->logger->info("First batch - Total TLDs to process: {$currentProgress['total']}");
        }
        
        // Get the last processed TLD ID to continue from where we left off
        $lastProcessedId = $this->getLastProcessedTldId($logId);
        $this->logger->info("Resuming from last processed ID: {$lastProcessedId}");
        
        // Get next batch of TLDs (increased to 50 for faster processing)
        $tldsNeedingWhois = $this->getTldsNeedingWhoisData(50, $lastProcessedId);
        $this->logger->info("Retrieved batch of " . count($tldsNeedingWhois) . " TLDs for processing");
        
        if (empty($tldsNeedingWhois)) {
            $this->logger->info("No more TLDs to process - Import complete!");
            $this->importLogModel->completeImport($logId, [
                'total_tlds' => $currentProgress['total'],
                'new_tlds' => 0,
                'updated_tlds' => $currentProgress['processed'],
                'failed_tlds' => $currentProgress['failed']
            ], 'completed', null, $currentProgress);
            
            $this->logger->endOperation("WHOIS Batch Processing", [
                'status' => 'complete',
                'total_processed' => $currentProgress['processed'],
                'total_failed' => $currentProgress['failed']
            ]);
            
            return [
                'status' => 'complete',
                'log_id' => $logId,
                'total' => $currentProgress['total'],
                'processed' => $currentProgress['processed'],
                'failed' => $currentProgress['failed'],
                'remaining' => 0,
                'message' => 'All TLDs processed successfully (ID 1 to last ID)'
            ];
        }

        $batchStats = [
            'total_tlds' => 0,
            'new_tlds' => 0,
            'updated_tlds' => 0,
            'failed_tlds' => 0
        ];

                $lastProcessedIdInBatch = 0;
                
                foreach ($tldsNeedingWhois as $index => $tld) {
                    $tldStartTime = microtime(true);
                    $batchStats['total_tlds']++;
                    $tldNumber = $index + 1;
                    $totalInBatch = count($tldsNeedingWhois);
                    
                    $this->logger->info("Processing TLD [{$tldNumber}/{$totalInBatch}]: {$tld['tld']} (ID: {$tld['id']})");
                    
                try {
                    $result = $this->fetchWhoisDataFromRdap($tld['tld']);
                    $fetchTime = round((microtime(true) - $tldStartTime) * 1000, 2);
                    
                    if ($result) {
                        $updateStartTime = microtime(true);
                        $this->tldModel->update($tld['id'], $result);
                        $updateTime = round((microtime(true) - $updateStartTime) * 1000, 2);
                        
                        $batchStats['updated_tlds']++;
                        $currentProgress['processed']++;
                        
                        // Log what data we found (or didn't find)
                        $foundData = [];
                        if (isset($result['whois_server'])) $foundData[] = 'WHOIS server';
                        if (isset($result['registry_url'])) $foundData[] = 'registry URL';
                        if (isset($result['registration_date'])) $foundData[] = 'registration date';
                        if (isset($result['record_last_updated'])) $foundData[] = 'last updated date';
                        
                        if (empty($foundData)) {
                            $this->logger->warning("TLD {$tld['tld']}: No data available from IANA", [
                                'fetch_time_ms' => $fetchTime,
                                'update_time_ms' => $updateTime
                            ]);
                        } else {
                            $this->logger->info("TLD {$tld['tld']}: SUCCESS - Found " . implode(', ', $foundData), [
                                'fetch_time_ms' => $fetchTime,
                                'update_time_ms' => $updateTime,
                                'data_fields' => count($foundData)
                            ]);
                        }
                    } else {
                        // Even if no data found, update the record to mark it as processed
                        $this->tldModel->update($tld['id'], ['updated_at' => date('Y-m-d H:i:s')]);
                        $batchStats['updated_tlds']++;
                        $currentProgress['processed']++;
                        $this->logger->warning("TLD {$tld['tld']}: No data returned, marked as processed", [
                            'fetch_time_ms' => $fetchTime
                        ]);
                    }
                    
                    // Track the highest ID processed in this batch
                    $lastProcessedIdInBatch = max($lastProcessedIdInBatch, $tld['id']);
                    
                } catch (\Exception $e) {
                    $batchStats['failed_tlds']++;
                    $currentProgress['failed']++;
                    $this->logger->error("TLD {$tld['tld']}: FAILED - " . $e->getMessage(), [
                        'exception_type' => get_class($e),
                        'file' => $e->getFile(),
                        'line' => $e->getLine()
                    ]);
                    
                    // Still track the ID even if it failed
                    $lastProcessedIdInBatch = max($lastProcessedIdInBatch, $tld['id']);
                }
                
                // Add minimal delay between requests (reduced for speed)
                if ($index < count($tldsNeedingWhois) - 1) {
                    usleep(25000); // 0.025 second delay for even faster processing
                }
            }
        
        $batchTime = round(microtime(true) - $batchStartTime, 2);
        
        // Update the last processed ID in currentProgress (CRITICAL - must be saved!)
        if ($lastProcessedIdInBatch > 0) {
            $currentProgress['last_processed_id'] = $lastProcessedIdInBatch;
            
            $this->logger->info("Updated last processed ID", [
                'previous_id' => $lastProcessedId,
                'new_id' => $lastProcessedIdInBatch,
                'jump' => $lastProcessedIdInBatch - $lastProcessedId
            ]);
        }

        $remainingCount = $this->getTldsNeedingWhoisCount($logId);
        
        $this->logger->info("Batch statistics", [
            'batch_time_seconds' => $batchTime,
            'tlds_in_batch' => count($tldsNeedingWhois),
            'successful' => $batchStats['updated_tlds'],
            'failed' => $batchStats['failed_tlds'],
            'avg_time_per_tld' => count($tldsNeedingWhois) > 0 ? round($batchTime / count($tldsNeedingWhois), 2) : 0,
            'remaining' => $remainingCount
        ]);
        
        if ($remainingCount === 0) {
            $this->logger->info("Import complete - No more TLDs remaining");
            $this->importLogModel->completeImport($logId, [
                'total_tlds' => $currentProgress['total'],
                'new_tlds' => 0,
                'updated_tlds' => $currentProgress['processed'],
                'failed_tlds' => $currentProgress['failed']
            ], 'completed', null, $currentProgress);
            $status = 'complete';
            $message = 'All TLDs processed successfully';
            
            $this->logger->endOperation("WHOIS Batch Processing", [
                'status' => 'complete',
                'total_processed' => $currentProgress['processed'],
                'total_failed' => $currentProgress['failed'],
                'total_time_seconds' => $batchTime
            ]);
        } else {
            $this->logger->info("Batch complete, more TLDs remaining", [
                'processed_in_batch' => count($tldsNeedingWhois),
                'remaining' => $remainingCount,
                'completion_percentage' => round((($currentProgress['total'] - $remainingCount) / $currentProgress['total']) * 100, 2)
            ]);
            
            $this->importLogModel->update($logId, [
                'total_tlds' => $currentProgress['total'],
                'updated_tlds' => $currentProgress['processed'],
                'failed_tlds' => $currentProgress['failed']
            ], null, null, $currentProgress);
            $status = 'in_progress';
            $message = "Processed batch of " . count($tldsNeedingWhois) . " TLDs, {$remainingCount} remaining";
        }

        return [
            'status' => $status,
            'log_id' => $logId,
            'total' => $currentProgress['total'],
            'processed' => $currentProgress['processed'],
            'failed' => $currentProgress['failed'],
            'remaining' => $remainingCount,
            'message' => $message
        ];
    }

    /**
     * Process check updates batch
     */
    private function processCheckUpdatesBatch(int $logId): array
    {
        try {
            $updateInfo = $this->checkForUpdates();
            $this->importLogModel->completeImport($logId, [
                'total_tlds' => 0,
                'new_tlds' => 0,
                'updated_tlds' => 0,
                'failed_tlds' => 0
            ]);
            
            // Build detailed message
            $messages = [];
            
            if ($updateInfo['tld_list']['needs_update'] ?? false) {
                $current = $updateInfo['tld_list']['current_version'] ?? 'Unknown';
                $last = $updateInfo['tld_list']['last_version'] ?? 'None';
                $messages[] = "TLD List: New version available (current: $current, previous: $last)";
            }
            
            if ($updateInfo['rdap']['needs_update'] ?? false) {
                $current = $updateInfo['rdap']['current_publication'] ?? 'Unknown';
                $last = $updateInfo['rdap']['last_publication'] ?? 'None';
                $messages[] = "RDAP Data: New publication available (current: $current, previous: $last)";
            }
            
            if ($updateInfo['overall_needs_update']) {
                $message = "🔔 Updates Available! " . implode(' • ', $messages) . " Click 'Import TLDs' to update your database.";
            } else {
                $tldVersion = $updateInfo['tld_list']['current_version'] ?? 'N/A';
                $rdapDate = $updateInfo['rdap']['current_publication'] ?? 'N/A';
                $message = "✅ TLD Registry is Up to Date! (TLD List version: $tldVersion, RDAP publication: $rdapDate)";
            }
            
            return [
                'status' => 'complete',
                'log_id' => $logId,
                'total' => 2,
                'processed' => 2,
                'failed' => 0,
                'remaining' => 0,
                'message' => $message,
                'update_info' => $updateInfo
            ];
        } catch (\Exception $e) {
            $this->importLogModel->completeImport($logId, [
                'total_tlds' => 0,
                'new_tlds' => 0,
                'updated_tlds' => 0,
                'failed_tlds' => 1
            ], 'failed', $e->getMessage());
            
            return [
                'status' => 'error',
                'message' => 'Update check failed: ' . $e->getMessage()
            ];
        }
    }

    /**
     * Process complete workflow batch (TLD list → RDAP → WHOIS → Registry URLs)
     */
    private function processCompleteWorkflowBatch(int $logId): array
    {
        $workflowStartTime = microtime(true);
        $this->logger->startOperation("Complete Workflow Batch", ['log_id' => $logId]);
        
        // Get current progress from log
        $log = $this->importLogModel->find($logId);
        $currentProgress = $log['details'] ? json_decode($log['details'], true) : [
            'current_step' => 0,
            'total_steps' => 3,
            'step_names' => ['Import TLD List', 'Import RDAP Servers', 'Import WHOIS & Registry Data'],
            'step_progress' => [0, 0, 0],
            'overall_processed' => 0,
            'overall_failed' => 0
        ];

        $this->logger->info("Workflow progress", [
            'current_step' => $currentProgress['current_step'],
            'total_steps' => $currentProgress['total_steps'],
            'step_name' => $currentProgress['step_names'][$currentProgress['current_step']] ?? 'Unknown'
        ]);

        try {
            // Step 1: Import TLD List
            if ($currentProgress['current_step'] == 0) {
                $stats = $this->importTldList();
                $currentProgress['step_progress'][0] = $stats['new_tlds'] + $stats['updated_tlds'];
                $currentProgress['overall_processed'] += $currentProgress['step_progress'][0];
                $currentProgress['overall_failed'] += $stats['failed_tlds'];
                $currentProgress['current_step'] = 1;
                
                $this->importLogModel->update($logId, [], 'running', null, $currentProgress);
                
                return [
                    'status' => 'in_progress',
                    'log_id' => $logId,
                    'total' => $currentProgress['total_steps'],
                    'processed' => $currentProgress['current_step'],
                    'failed' => $currentProgress['overall_failed'],
                    'remaining' => $currentProgress['total_steps'] - $currentProgress['current_step'],
                    'message' => "Step 1/3: {$currentProgress['step_names'][0]} completed. {$currentProgress['step_progress'][0]} TLDs processed."
                ];
            }

            // Step 2: Import RDAP Servers
            if ($currentProgress['current_step'] == 1) {
                $stats = $this->importRdapData();
                $currentProgress['step_progress'][1] = $stats['updated_tlds'];
                $currentProgress['overall_processed'] += $currentProgress['step_progress'][1];
                $currentProgress['overall_failed'] += $stats['failed_tlds'];
                $currentProgress['current_step'] = 2;
                
                $this->importLogModel->update($logId, [], 'running', null, $currentProgress);
                
                return [
                    'status' => 'in_progress',
                    'log_id' => $logId,
                    'total' => $currentProgress['total_steps'],
                    'processed' => $currentProgress['current_step'],
                    'failed' => $currentProgress['overall_failed'],
                    'remaining' => $currentProgress['total_steps'] - $currentProgress['current_step'],
                    'message' => "Step 2/3: {$currentProgress['step_names'][1]} completed. {$currentProgress['step_progress'][1]} TLDs updated."
                ];
            }

            // Step 3: Import WHOIS Data (in batches)
            if ($currentProgress['current_step'] == 2) {
                $this->logger->info("Step 3: Starting WHOIS batch processing");
                
                // Get the last processed TLD ID to continue from where we left off
                $lastProcessedId = $this->getLastProcessedTldId($logId);
                $this->logger->info("Resuming from last processed ID: {$lastProcessedId}");
                
                // Get next batch of TLDs needing WHOIS data (increased batch size)
                $tldsNeedingWhois = $this->getTldsNeedingWhoisData(50, $lastProcessedId);
                $this->logger->info("Retrieved " . count($tldsNeedingWhois) . " TLDs for WHOIS processing");
                
                if (empty($tldsNeedingWhois)) {
                    // No more TLDs to process, complete the workflow
                    $this->logger->info("Workflow complete - No more TLDs to process");
                    $this->importLogModel->completeImport($logId, [
                        'total_tlds' => $currentProgress['overall_processed'],
                        'new_tlds' => $currentProgress['step_progress'][0],
                        'updated_tlds' => $currentProgress['step_progress'][1] + $currentProgress['step_progress'][2],
                        'failed_tlds' => $currentProgress['overall_failed']
                    ], 'completed', null, $currentProgress);
                    
                    $this->logger->endOperation("Complete Workflow Batch", [
                        'status' => 'complete',
                        'total_processed' => $currentProgress['overall_processed'],
                        'total_failed' => $currentProgress['overall_failed']
                    ]);
                    
                    return [
                        'status' => 'complete',
                        'log_id' => $logId,
                        'total' => $currentProgress['total_steps'],
                        'processed' => $currentProgress['total_steps'],
                        'failed' => $currentProgress['overall_failed'],
                        'remaining' => 0,
                        'message' => "Complete workflow finished! All TLDs processed (ID 1 to last ID)."
                    ];
                }

                // Process WHOIS batch
                $batchProcessed = 0;
                $batchFailed = 0;
                $stepStartTime = microtime(true);
                
                $lastProcessedIdInBatch = 0;
                
                $this->logger->info("Starting to process batch of " . count($tldsNeedingWhois) . " TLDs");
                
                foreach ($tldsNeedingWhois as $index => $tld) {
                    $tldStartTime = microtime(true);
                    $tldNumber = $index + 1;
                    $totalInBatch = count($tldsNeedingWhois);
                    
                    $this->logger->info("Processing TLD [{$tldNumber}/{$totalInBatch}]: {$tld['tld']} (ID: {$tld['id']})");
                    
                    try {
                        $result = $this->fetchWhoisDataFromRdap($tld['tld']);
                        $fetchTime = round((microtime(true) - $tldStartTime) * 1000, 2);
                        
                        if ($result) {
                            $updateStartTime = microtime(true);
                            $this->tldModel->update($tld['id'], $result);
                            $updateTime = round((microtime(true) - $updateStartTime) * 1000, 2);
                            $batchProcessed++;
                            
                            // Log what data we found (or didn't find)
                            $foundData = [];
                            if (isset($result['whois_server'])) $foundData[] = 'WHOIS server';
                            if (isset($result['registry_url'])) $foundData[] = 'registry URL';
                            if (isset($result['registration_date'])) $foundData[] = 'registration date';
                            if (isset($result['record_last_updated'])) $foundData[] = 'last updated date';
                            
                            if (empty($foundData)) {
                                $this->logger->warning("TLD {$tld['tld']}: No data available", [
                                    'fetch_time_ms' => $fetchTime,
                                    'update_time_ms' => $updateTime
                                ]);
                            } else {
                                $this->logger->info("TLD {$tld['tld']}: SUCCESS - " . implode(', ', $foundData), [
                                    'fetch_time_ms' => $fetchTime,
                                    'update_time_ms' => $updateTime
                                ]);
                            }
                        } else {
                            // Even if no data found, update the record to mark it as processed
                            $this->tldModel->update($tld['id'], ['updated_at' => date('Y-m-d H:i:s')]);
                            $batchProcessed++;
                            $this->logger->warning("TLD {$tld['tld']}: No data returned, marked as processed", [
                                'fetch_time_ms' => $fetchTime
                            ]);
                        }
                        
                        // Track the highest ID processed in this batch
                        $lastProcessedIdInBatch = max($lastProcessedIdInBatch, $tld['id']);
                        
                    } catch (\Exception $e) {
                        $batchFailed++;
                        $this->logger->error("TLD {$tld['tld']}: FAILED - " . $e->getMessage(), [
                            'exception_type' => get_class($e),
                            'file' => $e->getFile(),
                            'line' => $e->getLine()
                        ]);
                        
                        // Still track the ID even if it failed
                        $lastProcessedIdInBatch = max($lastProcessedIdInBatch, $tld['id']);
                    }
                    
                    // Add minimal delay between requests (reduced for speed)
                    if ($index < count($tldsNeedingWhois) - 1) {
                        usleep(25000); // 0.025 second delay for even faster processing
                    }
                }
                
                $stepTime = round(microtime(true) - $stepStartTime, 2);
                
                // Update progress counters
                $currentProgress['step_progress'][2] += $batchProcessed;
                $currentProgress['overall_processed'] += $batchProcessed;
                $currentProgress['overall_failed'] += $batchFailed;
                
                // Update the last processed ID in currentProgress (CRITICAL - must be saved!)
                if ($lastProcessedIdInBatch > 0) {
                    $currentProgress['last_processed_id'] = $lastProcessedIdInBatch;
                    
                    $this->logger->info("Updated last processed ID", [
                        'previous_id' => $lastProcessedId,
                        'new_id' => $lastProcessedIdInBatch,
                        'jump' => $lastProcessedIdInBatch - $lastProcessedId
                    ]);
                }
                
                $this->logger->info("Step 3 batch statistics", [
                    'batch_time_seconds' => $stepTime,
                    'processed' => $batchProcessed,
                    'failed' => $batchFailed,
                    'avg_time_per_tld' => $batchProcessed > 0 ? round($stepTime / $batchProcessed, 2) : 0
                ]);
                
                // Update the import log with all progress including last_processed_id
                $this->importLogModel->update($logId, [], 'running', null, $currentProgress);
                
                $remainingWhois = $this->getTldsNeedingWhoisCount($logId);
                $this->logger->info("Remaining TLDs to process: {$remainingWhois}");
                
                if ($remainingWhois > 0) {
                    // Still TLDs to process - return in_progress status
                    $completionPercent = round((($currentProgress['step_progress'][2]) / ($currentProgress['step_progress'][2] + $remainingWhois)) * 100, 2);
                    $totalTldsInStep3 = $currentProgress['step_progress'][2] + $remainingWhois;
                    
                    $this->logger->info("Step 3 in progress", [
                        'completion_percentage' => $completionPercent,
                        'processed_so_far' => $currentProgress['step_progress'][2],
                        'remaining' => $remainingWhois,
                        'batch_processed' => $batchProcessed
                    ]);
                    
                    return [
                        'status' => 'in_progress',
                        'log_id' => $logId,
                        'total' => $totalTldsInStep3, // Total TLDs to process in step 3
                        'processed' => $currentProgress['step_progress'][2], // TLDs processed so far in step 3
                        'failed' => $currentProgress['overall_failed'],
                        'remaining' => $remainingWhois, // Fixed: Use actual remaining TLDs, not remaining steps
                        'message' => "Step 3/3: {$currentProgress['step_names'][2]} - Processed {$currentProgress['step_progress'][2]} of {$totalTldsInStep3} TLDs ({$completionPercent}% complete, {$remainingWhois} remaining)"
                    ];
                } else {
                    // No more TLDs - complete the workflow!
                    $this->logger->info("Step 3 complete - All TLDs processed!");
                    
                    $this->importLogModel->completeImport($logId, [
                        'total_tlds' => $currentProgress['overall_processed'],
                        'new_tlds' => $currentProgress['step_progress'][0],
                        'updated_tlds' => $currentProgress['step_progress'][1] + $currentProgress['step_progress'][2],
                        'failed_tlds' => $currentProgress['overall_failed']
                    ], 'completed', null, $currentProgress);
                    
                    $this->logger->endOperation("Complete Workflow Batch", [
                        'status' => 'complete',
                        'total_processed' => $currentProgress['overall_processed'],
                        'total_failed' => $currentProgress['overall_failed']
                    ]);
                    
                    return [
                        'status' => 'complete',
                        'log_id' => $logId,
                        'total' => $currentProgress['step_progress'][2],
                        'processed' => $currentProgress['step_progress'][2],
                        'failed' => $currentProgress['overall_failed'],
                        'remaining' => 0,
                        'message' => "Complete workflow finished! All {$currentProgress['step_progress'][2]} TLDs processed successfully."
                    ];
                }
            }

        } catch (\Exception $e) {
            $this->logger->critical("Complete workflow failed", [
                'error' => $e->getMessage(),
                'exception_type' => get_class($e),
                'file' => $e->getFile(),
                'line' => $e->getLine(),
                'trace' => $e->getTraceAsString()
            ]);
            
            $this->importLogModel->completeImport($logId, [
                'total_tlds' => 0,
                'new_tlds' => 0,
                'updated_tlds' => 0,
                'failed_tlds' => 1
            ], 'failed', $e->getMessage());
            
            $this->logger->endOperation("Complete Workflow Batch", [
                'status' => 'failed',
                'error' => $e->getMessage()
            ]);
            
            return [
                'status' => 'error',
                'message' => 'Complete workflow failed: ' . $e->getMessage()
            ];
        }
    }

    /**
     * Import registry URLs for TLDs missing them
     */
    private function importRegistryUrls(): array
    {
        $stats = [
            'total_tlds' => 0,
            'new_tlds' => 0,
            'updated_tlds' => 0,
            'failed_tlds' => 0
        ];

        // Get TLDs missing registry URLs
        $sql = "SELECT * FROM tld_registry 
                WHERE is_active = 1 
                AND (registry_url IS NULL OR registry_url = '')
                LIMIT 50";
        
        $tlds = $this->tldModel->query($sql);
        
        foreach ($tlds as $tld) {
            $stats['total_tlds']++;
            
            try {
                // Try to fetch registry URL from RDAP API
                $registryUrl = $this->fetchRegistryUrlFromRdap($tld['tld']);
                
                if ($registryUrl) {
                    $this->tldModel->update($tld['id'], [
                        'registry_url' => $registryUrl,
                        'updated_at' => date('Y-m-d H:i:s')
                    ]);
                    $stats['updated_tlds']++;
                } else {
                    $stats['failed_tlds']++;
                }
            } catch (\Exception $e) {
                $stats['failed_tlds']++;
                error_log("Failed to fetch registry URL for TLD {$tld['tld']}: " . $e->getMessage());
            }
            
            // Add small delay
            usleep(100000); // 0.1 second delay
        }

        return $stats;
    }

    /**
     * Process a single TLD entry from the TLD list
     */
    private function processTldEntry(string $tld): array
    {
        // Ensure TLD starts with dot
        if (!str_starts_with($tld, '.')) {
            $tld = '.' . $tld;
        }

        $data = [
            'tld' => $tld,
            'is_active' => 1,
            'created_at' => date('Y-m-d H:i:s'),
            'updated_at' => date('Y-m-d H:i:s')
        ];

        // Check if TLD already exists
        $existing = $this->tldModel->getByTld($tld);
        $isNew = !$existing;

        if ($existing) {
            // Update existing record (just update the timestamp)
            $this->tldModel->update($existing['id'], [
                'updated_at' => date('Y-m-d H:i:s')
            ]);
        } else {
            // Create new record
            $this->tldModel->create($data);
        }

        return ['is_new' => $isNew];
    }

    /**
     * Process RDAP data for a single TLD
     */
    private function processTldRdapData(string $tld, array $rdapServers, ?string $publicationDate): array
    {
        // Ensure TLD starts with dot
        if (!str_starts_with($tld, '.')) {
            $tld = '.' . $tld;
        }

        $data = [
            'tld' => $tld,
            'rdap_servers' => json_encode($rdapServers),
            'iana_publication_date' => $publicationDate,
            'iana_last_updated' => date('Y-m-d H:i:s'),
            'is_active' => 1
        ];

        // Check if TLD already exists
        $existing = $this->tldModel->getByTld($tld);
        $isNew = !$existing;

        if ($existing) {
            // Update existing record
            $this->tldModel->update($existing['id'], $data);
        } else {
            // Create new record
            $this->tldModel->create($data);
        }

        return ['is_new' => $isNew];
    }

    /**
     * Fetch WHOIS and registry data using hybrid approach: RDAP API first, HTML fallback
     */
    private function fetchWhoisDataFromRdap(string $tld): ?array
    {
        $tldForUrl = ltrim($tld, '.');
        $rdapUrl = self::IANA_RDAP_DOMAIN_URL . $tldForUrl;

        try {
            // Step 1: Try RDAP API first (fast, structured data)
            $jsonClient = $this->getJsonClient();
            $response = $jsonClient->get($rdapUrl);
            
            if ($response->getStatusCode() !== 200) {
                error_log("Failed to fetch RDAP data for TLD $tld: HTTP " . $response->getStatusCode());
                return null;
            }
            
            $responseBody = $response->getBody()->getContents();
            
            // Check if response is HTML instead of JSON (common when servers are down)
            if (strpos($responseBody, '<html') !== false || strpos($responseBody, '<!DOCTYPE') !== false) {
                error_log("Received HTML instead of JSON for TLD $tld - server may be down");
                return null;
            }
            
            $data = json_decode($responseBody, true);
            
            if (!$data) {
                error_log("Invalid JSON response for TLD $tld: " . substr($responseBody, 0, 200));
                return null;
            }

            $result = [
                'updated_at' => date('Y-m-d H:i:s')
            ];

            // Extract WHOIS server from RDAP data (port43 field)
            if (isset($data['port43']) && !empty($data['port43'])) {
                $result['whois_server'] = $data['port43'];
            }

            // Extract registry URL from links array
            if (isset($data['links']) && is_array($data['links'])) {
                foreach ($data['links'] as $link) {
                    if (isset($link['rel']) && $link['rel'] === 'related' && 
                        isset($link['title']) && $link['title'] === 'Registration URL' &&
                        isset($link['href'])) {
                        $result['registry_url'] = $link['href'];
                        break;
                    }
                }
            }

            // Extract dates from events array
            if (isset($data['events']) && is_array($data['events'])) {
                foreach ($data['events'] as $event) {
                    if (isset($event['eventAction']) && isset($event['eventDate'])) {
                        switch ($event['eventAction']) {
                            case 'registration':
                                $result['registration_date'] = $this->normalizeDate($event['eventDate']);
                                break;
                            case 'last changed':
                                $result['record_last_updated'] = $this->normalizeDate($event['eventDate']);
                                break;
                        }
                    }
                }
            }

            // Step 2: If WHOIS server is missing, try HTML fallback
            if (!isset($result['whois_server']) || empty($result['whois_server'])) {
                $htmlData = $this->fetchWhoisDataFromHtml($tld);
                if ($htmlData) {
                    // Merge HTML data, prioritizing RDAP data but filling gaps with HTML data
                    if (isset($htmlData['whois_server']) && !empty($htmlData['whois_server'])) {
                        $result['whois_server'] = $htmlData['whois_server'];
                    }
                    if (!isset($result['registry_url']) && isset($htmlData['registry_url'])) {
                        $result['registry_url'] = $htmlData['registry_url'];
                    }
                    if (!isset($result['registration_date']) && isset($htmlData['registration_date'])) {
                        $result['registration_date'] = $htmlData['registration_date'];
                    }
                    if (!isset($result['record_last_updated']) && isset($htmlData['record_last_updated'])) {
                        $result['record_last_updated'] = $htmlData['record_last_updated'];
                    }
                }
            }

            // Always return the result, even if some fields are missing
            // This ensures we update the TLD record with whatever data we found
            return $result;

        } catch (\Exception $e) {
            error_log("Failed to fetch RDAP data for TLD $tld: " . $e->getMessage());
        }

        return null;
    }

    /**
     * Fallback: Fetch WHOIS data from IANA HTML page
     */
    private function fetchWhoisDataFromHtml(string $tld): ?array
    {
        $tldForUrl = ltrim($tld, '.');
        $url = self::IANA_TLD_BASE_URL . $tldForUrl . '.html';

        try {
            $htmlClient = $this->getHtmlClient();
            $response = $htmlClient->get($url);
            
            if ($response->getStatusCode() !== 200) {
                error_log("HTML fetch failed for TLD $tld: HTTP " . $response->getStatusCode());
                return null;
            }
            
            $html = $response->getBody()->getContents();
            
            if (empty($html) || strlen($html) < 100) {
                error_log("HTML content too short for TLD $tld: " . strlen($html) . " bytes");
                return null;
            }

            $result = [];

            // Parse HTML to extract WHOIS server and other data
            $whoisServer = $this->extractWhoisServer($html);
            $lastUpdated = $this->extractLastUpdated($html);
            $registryUrl = $this->extractRegistryUrl($html);
            $registrationDate = $this->extractRegistrationDate($html);

            if ($whoisServer) {
                $result['whois_server'] = $whoisServer;
            }
            if ($lastUpdated) {
                $result['record_last_updated'] = $this->normalizeDate($lastUpdated);
            }
            if ($registryUrl) {
                $result['registry_url'] = $registryUrl;
            }
            if ($registrationDate) {
                $result['registration_date'] = $this->normalizeDate($registrationDate);
            }

            return !empty($result) ? $result : null;

        } catch (\Exception $e) {
            error_log("Failed to fetch HTML data for TLD $tld: " . $e->getMessage());
        }

        return null;
    }

    /**
     * Extract WHOIS server from IANA HTML
     */
    private function extractWhoisServer(string $html): ?string
    {
        // Look for WHOIS Server pattern with HTML tags
        if (preg_match('/<b>WHOIS Server:<\/b>\s*([^\s<]+)/i', $html, $matches)) {
            return trim($matches[1]);
        }
        
        // Fallback: Look for WHOIS Server pattern without HTML tags
        if (preg_match('/WHOIS Server:\s*([^\s<]+)/i', $html, $matches)) {
            return trim($matches[1]);
        }
        
        return null;
    }

    /**
     * Extract last updated date from IANA HTML
     */
    private function extractLastUpdated(string $html): ?string
    {
        // Look for "Record last updated" pattern
        if (preg_match('/Record last updated\s+(\d{4}-\d{2}-\d{2})/i', $html, $matches)) {
            return $matches[1] . ' 00:00:00';
        }
        return null;
    }

    /**
     * Extract registry URL from IANA HTML
     */
    private function extractRegistryUrl(string $html): ?string
    {
        // Look for "URL for registration services" pattern with <a> tag
        if (preg_match('/<b>URL for registration services:<\/b>\s*<a[^>]*href="([^"]+)"[^>]*>/i', $html, $matches)) {
            return trim($matches[1]);
        }
        
        // Look for "URL for registration services" pattern with HTML tags
        if (preg_match('/<b>URL for registration services:<\/b>\s*([^\s<]+)/i', $html, $matches)) {
            return trim($matches[1]);
        }
        
        // Fallback: Look for "URL for registration services" pattern without HTML tags
        if (preg_match('/URL for registration services:\s*([^\s<]+)/i', $html, $matches)) {
            return trim($matches[1]);
        }
        
        return null;
    }

    /**
     * Extract registration date from IANA HTML
     */
    private function extractRegistrationDate(string $html): ?string
    {
        // Look for "Registration date" pattern
        if (preg_match('/Registration date\s+(\d{4}-\d{2}-\d{2})/i', $html, $matches)) {
            return $matches[1];
        }
        return null;
    }

    /**
     * Check for IANA updates in both TLD list and RDAP data
     */
    public function checkForUpdates(): array
    {
        $updates = [
            'tld_list' => ['needs_update' => false, 'current_version' => null, 'last_version' => null],
            'rdap' => ['needs_update' => false, 'current_publication' => null, 'last_publication' => null],
            'overall_needs_update' => false,
            'errors' => []
        ];

        try {
            // Check TLD list for updates
            $tldListUpdate = $this->checkTldListUpdates();
            $updates['tld_list'] = $tldListUpdate;
            
            // Check RDAP data for updates
            $rdapUpdate = $this->checkRdapUpdates();
            $updates['rdap'] = $rdapUpdate;
            
            // Determine if any updates are needed
            $updates['overall_needs_update'] = $tldListUpdate['needs_update'] || $rdapUpdate['needs_update'];
            
        } catch (\Exception $e) {
            $updates['errors'][] = $e->getMessage();
        }

        return $updates;
    }

    /**
     * Check for TLD list updates
     */
    private function checkTldListUpdates(): array
    {
        try {
            $jsonClient = $this->getJsonClient();
            $response = $jsonClient->get(self::IANA_TLD_LIST_URL);
            
            if ($response->getStatusCode() !== 200) {
                return [
                    'needs_update' => false,
                    'current_version' => null,
                    'last_version' => null,
                    'error' => 'Failed to fetch TLD list: HTTP ' . $response->getStatusCode()
                ];
            }
            
            $content = $response->getBody()->getContents();
            $lines = explode("\n", $content);
            
            $currentVersion = null;
            $currentLastUpdated = null;
            
            // Extract version and timestamp from header
            foreach ($lines as $line) {
                $line = trim($line);
                if (strpos($line, '# Version') === 0) {
                    if (preg_match('/# Version (\d+), Last Updated (.+)/', $line, $matches)) {
                        $currentVersion = $matches[1];
                        $currentLastUpdatedRaw = $matches[2];
                        
                        // Convert the timestamp to a proper format
                        try {
                            $currentLastUpdated = date('Y-m-d H:i:s', strtotime($currentLastUpdatedRaw));
                            if ($currentLastUpdated === '1970-01-01 00:00:00') {
                                // If strtotime fails, keep the raw value
                                $currentLastUpdated = $currentLastUpdatedRaw;
                            }
                        } catch (\Exception $e) {
                            $currentLastUpdated = $currentLastUpdatedRaw;
                        }
                    }
                    break;
                }
            }
            
            // Get last TLD list import
            $lastTldImport = $this->importLogModel->query(
                "SELECT version, iana_publication_date FROM tld_import_logs 
                 WHERE import_type = 'tld_list' AND status = 'completed' 
                 ORDER BY started_at DESC LIMIT 1"
            );
            
            $lastVersion = $lastTldImport[0]['version'] ?? null;
            $lastPublication = $lastTldImport[0]['iana_publication_date'] ?? null;
            
            $needsUpdate = ($currentVersion !== $lastVersion) || ($currentLastUpdated !== $lastPublication);
            
            return [
                'needs_update' => $needsUpdate,
                'current_version' => $currentVersion,
                'current_last_updated' => $currentLastUpdated,
                'last_version' => $lastVersion,
                'last_publication' => $lastPublication
            ];
            
        } catch (\Exception $e) {
            return [
                'needs_update' => false,
                'current_version' => null,
                'last_version' => null,
                'error' => $e->getMessage()
            ];
        }
    }

    /**
     * Check for RDAP data updates
     */
    private function checkRdapUpdates(): array
    {
        try {
            $jsonClient = $this->getJsonClient();
            $response = $jsonClient->get(self::IANA_RDAP_URL);
            
            if ($response->getStatusCode() !== 200) {
                return [
                    'needs_update' => false,
                    'current_publication' => null,
                    'last_publication' => null,
                    'error' => 'Failed to fetch RDAP data: HTTP ' . $response->getStatusCode()
                ];
            }
            
            $data = json_decode($response->getBody()->getContents(), true);
            $currentPublication = $data['publication'] ?? null;
            
            // Get last RDAP import using database directly
            $db = \Core\Database::getConnection();
            $stmt = $db->prepare(
                "SELECT iana_publication_date FROM tld_import_logs 
                 WHERE import_type = 'rdap' AND status = 'completed' 
                 ORDER BY started_at DESC LIMIT 1"
            );
            $stmt->execute();
            $lastRdapImport = $stmt->fetch();
            
            $lastPublication = $lastRdapImport['iana_publication_date'] ?? null;
            
            // Normalize date formats for comparison (ISO 8601 vs MySQL datetime)
            $currentNormalized = $this->normalizeDate($currentPublication);
            $lastNormalized = $this->normalizeDate($lastPublication);
            
            $needsUpdate = ($currentNormalized !== $lastNormalized) && ($currentNormalized !== null);
            
            return [
                'needs_update' => $needsUpdate,
                'current_publication' => $currentNormalized ?: $currentPublication,
                'last_publication' => $lastNormalized ?: $lastPublication
            ];
            
        } catch (\Exception $e) {
            return [
                'needs_update' => false,
                'current_publication' => null,
                'last_publication' => null,
                'error' => $e->getMessage()
            ];
        }
    }

    /**
     * Get TLD registry information for a domain
     */
    public function getTldInfo(string $domain): ?array
    {
        // Extract TLD from domain
        $tld = $this->extractTldFromDomain($domain);
        if (!$tld) {
            return null;
        }

        return $this->tldModel->getByTld($tld);
    }

    /**
     * Extract TLD from domain name
     */
    private function extractTldFromDomain(string $domain): ?string
    {
        $domain = strtolower(trim($domain));
        
        // Remove protocol if present
        $domain = preg_replace('/^https?:\/\//', '', $domain);
        
        // Remove www if present
        $domain = preg_replace('/^www\./', '', $domain);
        
        // Remove path if present
        $domain = explode('/', $domain)[0];
        
        // Split by dots and get the last part (TLD)
        $parts = explode('.', $domain);
        if (count($parts) < 2) {
            return null;
        }
        
        // For domains like example.co.uk, we want .co.uk
        if (count($parts) > 2) {
            // Check if it's a known multi-part TLD
            $lastTwo = '.' . $parts[count($parts) - 2] . '.' . $parts[count($parts) - 1];
            $lastOne = '.' . $parts[count($parts) - 1];
            
            // Try to find the TLD in our registry
            $tldInfo = $this->tldModel->getByTld($lastTwo);
            if ($tldInfo) {
                return $lastTwo;
            }
            
            return $lastOne;
        }
        
        return '.' . $parts[count($parts) - 1];
    }

    /**
     * Get RDAP servers for a TLD
     */
    public function getRdapServers(string $tld): array
    {
        $tldInfo = $this->tldModel->getByTld($tld);
        if (!$tldInfo || empty($tldInfo['rdap_servers'])) {
            return [];
        }

        $servers = json_decode($tldInfo['rdap_servers'], true);
        return is_array($servers) ? $servers : [];
    }

    /**
     * Get WHOIS server for a TLD
     */
    public function getWhoisServer(string $tld): ?string
    {
        $tldInfo = $this->tldModel->getByTld($tld);
        return $tldInfo['whois_server'] ?? null;
    }

    /**
     * Import WHOIS data for specific TLDs that are known to be missing from RDAP
     */
    public function importWhoisForSpecificTlds(array $tlds): array
    {
        $logId = $this->importLogModel->startImport('whois');
        $stats = [
            'total_tlds' => 0,
            'new_tlds' => 0,
            'updated_tlds' => 0,
            'failed_tlds' => 0
        ];

        try {
            foreach ($tlds as $index => $tld) {
                $stats['total_tlds']++;
                
                try {
                    // Ensure TLD starts with dot
                    if (!str_starts_with($tld, '.')) {
                        $tld = '.' . $tld;
                    }

                    // Check if TLD exists in our registry
                    $existing = $this->tldModel->getByTld($tld);
                    
                    if (!$existing) {
                        // Create new TLD entry first
                        $this->tldModel->create([
                            'tld' => $tld,
                            'is_active' => 1,
                            'created_at' => date('Y-m-d H:i:s'),
                            'updated_at' => date('Y-m-d H:i:s')
                        ]);
                        $existing = $this->tldModel->getByTld($tld);
                        $stats['new_tlds']++;
                    }

                    // Fetch WHOIS data
                    $result = $this->fetchWhoisDataFromRdap($tld);
                    
                    if ($result && $existing) {
                        $this->tldModel->update($existing['id'], $result);
                        $stats['updated_tlds']++;
                    } else {
                        $stats['failed_tlds']++;
                    }
                } catch (\Exception $e) {
                    $stats['failed_tlds']++;
                    error_log("Failed to fetch WHOIS data for TLD $tld: " . $e->getMessage());
                }
                
                // Add delay between requests to be respectful to IANA servers
                if ($index < count($tlds) - 1) {
                    usleep(500000); // 0.5 second delay
                }
            }

            $this->importLogModel->completeImport($logId, $stats);
            
        } catch (\Exception $e) {
            $this->importLogModel->completeImport($logId, $stats, $e->getMessage());
            throw $e;
        }

        return $stats;
    }

    /**
     * Normalize date string for comparison
     * Converts both ISO 8601 and MySQL datetime to same format (UTC)
     *
     * @param string|null $date Date string to normalize
     * @return string|null Normalized date in UTC (Y-m-d H:i:s) or null
     */
    private function normalizeDate(?string $date): ?string
    {
        if (empty($date)) {
            return null;
        }

        try {
            // If date has timezone info (ISO 8601 with Z or +00:00), parse it correctly
            // Otherwise assume it's UTC (for dates from database)
            if (strpos($date, 'T') !== false || strpos($date, 'Z') !== false || strpos($date, '+') !== false) {
                // ISO 8601 format with timezone - parse as-is
                $dateTime = new \DateTime($date);
            } else {
                // Plain datetime (from database) - explicitly parse as UTC
                $dateTime = new \DateTime($date, new \DateTimeZone('UTC'));
            }
            
            // Convert to UTC to ensure consistent comparison
            $dateTime->setTimezone(new \DateTimeZone('UTC'));
            
            // Return in MySQL datetime format (Y-m-d H:i:s) in UTC
            return $dateTime->format('Y-m-d H:i:s');
        } catch (\Exception $e) {
            // Fallback to null if date parsing fails
            return null;
        }
    }
}