Add TLD registry import/export/create & logging

Add CSV/JSON export and import endpoints and UI for the TLD registry, plus a manual Create TLD modal and drag-and-drop import UX. Standardize import/export logging by adding Logger('import'/'export') calls to Domains, Tags, Notification Groups and TLD flows. Add TldRegistry model helpers (findByTld, getAll) used for deduplication and exports. Update routes for /tld-registry export/import/create and add a migration to bump app_version to 1.1.4. Also update default app_version, enhance WhoisService parsing (registrar regex and ISO-8601 date handling), and adjust the TLD registry index view to include IANA and Export dropdowns, import modal, create modal, and related JS behavior.
2026-03-02 11:17:58 +02:00
parent aca4c14b8b
commit ed3e5739f4
13 changed files with 792 additions and 29 deletions
--- a/app/Controllers/DomainController.php
+++ b/app/Controllers/DomainController.php
@@ -224,16 +224,26 @@ class DomainController extends Controller

        $this->verifyCsrf('/domains/bulk-add');

+        $logger = new \App\Services\Logger('import');
+        $userId = \Core\Auth::id();
+        $logger->info('Domains import started', ['user_id' => $userId]);
+
        if (!isset($_FILES['import_file']) || $_FILES['import_file']['error'] !== UPLOAD_ERR_OK) {
+            $logger->warning('No valid file uploaded for domains import');
            $_SESSION['error'] = 'Please select a valid file to import';
            $this->redirect('/domains/bulk-add');
            return;
        }

        $file = $_FILES['import_file'];
+        $logger->info('Import file received', [
+            'filename' => $file['name'],
+            'size' => $file['size']
+        ]);

        // Validate file size (5MB max for domains)
        if ($file['size'] > 5242880) {
+            $logger->warning('Import file too large', ['size' => $file['size']]);
            $_SESSION['error'] = 'File is too large. Maximum size is 5MB';
            $this->redirect('/domains/bulk-add');
            return;
@@ -241,6 +251,7 @@ class DomainController extends Controller

        $ext = strtolower(pathinfo($file['name'], PATHINFO_EXTENSION));
        if (!in_array($ext, ['csv', 'json'])) {
+            $logger->warning('Invalid file type for domains import', ['extension' => $ext]);
            $_SESSION['error'] = 'Invalid file type. Please upload a CSV or JSON file';
            $this->redirect('/domains/bulk-add');
            return;
@@ -252,6 +263,7 @@ class DomainController extends Controller
        if ($ext === 'json') {
            $parsed = json_decode($content, true);
            if (!is_array($parsed)) {
+                $logger->error('Invalid JSON file for domains import');
                $_SESSION['error'] = 'Invalid JSON file';
                $this->redirect('/domains/bulk-add');
                return;
@@ -275,12 +287,14 @@ class DomainController extends Controller
        }

        if (empty($domainsData)) {
+            $logger->warning('No domains found in import file');
            $_SESSION['error'] = 'No domains found in file';
            $this->redirect('/domains/bulk-add');
            return;
        }

-        $userId = \Core\Auth::id();
+        $logger->info('Domains data parsed from file', ['entries' => count($domainsData)]);
+
        $settingModel = new \App\Models\Setting();
        $isolationMode = $settingModel->getValue('user_isolation_mode', 'shared');
        $tagModel = new \App\Models\Tag();
@@ -291,7 +305,6 @@ class DomainController extends Controller
        $added = 0;
        $skipped = 0;
        $errors = [];
-        $logger = new \App\Services\Logger();

        foreach ($domainsData as $row) {
            $domainName = strtolower(trim($row['domain_name'] ?? ''));
@@ -367,6 +380,12 @@ class DomainController extends Controller
            }
        }

+        $logger->info('Domains import completed', [
+            'added' => $added,
+            'skipped' => $skipped,
+            'failed' => count($errors)
+        ]);
+
        $msg = "{$added} domain(s) imported successfully";
        if ($skipped > 0) $msg .= ", {$skipped} skipped (already exist)";
        if (!empty($errors)) $msg .= ", " . count($errors) . " failed";
--- a/app/Controllers/InstallerController.php
+++ b/app/Controllers/InstallerController.php
@@ -56,6 +56,7 @@ class InstallerController extends Controller
            '023_update_app_version_to_1.1.1.sql',
            '024_add_status_notifications_v1.1.2.sql',
            '025_add_update_system_v1.1.3.sql',
+            '026_update_app_version_v1.1.4.sql',
        ];
        
        try {
@@ -198,6 +199,7 @@ class InstallerController extends Controller
                    '023_update_app_version_to_1.1.1.sql',
                    '024_add_status_notifications_v1.1.2.sql',
                    '025_add_update_system_v1.1.3.sql',
+                    '026_update_app_version_v1.1.4.sql',
                ];
            }
            
@@ -420,6 +422,7 @@ class InstallerController extends Controller
                    '023_update_app_version_to_1.1.1.sql',
                    '024_add_status_notifications_v1.1.2.sql',
                    '025_add_update_system_v1.1.3.sql',
+                    '026_update_app_version_v1.1.4.sql',
                ];
                
                $stmt = $pdo->prepare("INSERT INTO migrations (migration) VALUES (?) ON DUPLICATE KEY UPDATE migration=migration");
@@ -655,7 +658,9 @@ class InstallerController extends Controller
                    
                    // Fallback: detect "to" version from which migrations were run
                    if ($toVersion === $fromVersion) {
-                        if (in_array('025_add_update_system_v1.1.3.sql', $executed)) {
+                        if (in_array('026_update_app_version_v1.1.4.sql', $executed)) {
+                            $toVersion = '1.1.4';
+                        } elseif (in_array('025_add_update_system_v1.1.3.sql', $executed)) {
                            $toVersion = '1.1.3';
                        } elseif (in_array('024_add_status_notifications_v1.1.2.sql', $executed)) {
                            $toVersion = '1.1.2';
--- a/app/Controllers/NotificationGroupController.php
+++ b/app/Controllers/NotificationGroupController.php
@@ -194,17 +194,27 @@ class NotificationGroupController extends Controller

        $this->verifyCsrf('/groups');

+        $logger = new \App\Services\Logger('import');
+        $userId = \Core\Auth::id();
+        $logger->info('Notification groups import started', ['user_id' => $userId]);
+
        $validChannelTypes = ['email', 'telegram', 'discord', 'slack', 'mattermost', 'webhook', 'pushover'];

        if (!isset($_FILES['import_file']) || $_FILES['import_file']['error'] !== UPLOAD_ERR_OK) {
+            $logger->warning('No valid file uploaded for groups import');
            $_SESSION['error'] = 'Please select a valid file to import';
            $this->redirect('/groups');
            return;
        }

        $file = $_FILES['import_file'];
+        $logger->info('Import file received', [
+            'filename' => $file['name'],
+            'size' => $file['size']
+        ]);

        if ($file['size'] > 2097152) {
+            $logger->warning('Import file too large', ['size' => $file['size']]);
            $_SESSION['error'] = 'File is too large. Maximum size is 2MB';
            $this->redirect('/groups');
            return;
@@ -212,13 +222,13 @@ class NotificationGroupController extends Controller

        $ext = strtolower(pathinfo($file['name'], PATHINFO_EXTENSION));
        if (!in_array($ext, ['csv', 'json'])) {
+            $logger->warning('Invalid file type for groups import', ['extension' => $ext]);
            $_SESSION['error'] = 'Invalid file type. Please upload a CSV or JSON file';
            $this->redirect('/groups');
            return;
        }

        $content = file_get_contents($file['tmp_name']);
-        $userId = \Core\Auth::id();
        $settingModel = new \App\Models\Setting();
        $isolationMode = $settingModel->getValue('user_isolation_mode', 'shared');

@@ -229,11 +239,14 @@ class NotificationGroupController extends Controller
        if ($ext === 'json') {
            $parsed = json_decode($content, true);
            if (!is_array($parsed)) {
+                $logger->error('Invalid JSON file for groups import');
                $_SESSION['error'] = 'Invalid JSON file';
                $this->redirect('/groups');
                return;
            }

+            $logger->info('Groups data parsed from file', ['entries' => count($parsed)]);
+
            foreach ($parsed as $groupData) {
                $groupName = trim($groupData['group_name'] ?? '');
                if (empty($groupName)) continue;
@@ -338,6 +351,12 @@ class NotificationGroupController extends Controller
            }
        }

+        $logger->info('Notification groups import completed', [
+            'groups_created' => $groupsCreated,
+            'channels_created' => $channelsCreated,
+            'groups_skipped' => $groupsSkipped
+        ]);
+
        $msg = "{$groupsCreated} group(s) imported ({$channelsCreated} channels)";
        if ($groupsSkipped > 0) $msg .= ", {$groupsSkipped} skipped (already exist)";
        $_SESSION['success'] = $msg;
--- a/app/Controllers/TagController.php
+++ b/app/Controllers/TagController.php
@@ -176,16 +176,26 @@ class TagController extends Controller

        $this->verifyCsrf('/tags');

+        $logger = new \App\Services\Logger('import');
+        $userId = \Core\Auth::id();
+        $logger->info('Tags import started', ['user_id' => $userId]);
+
        if (!isset($_FILES['import_file']) || $_FILES['import_file']['error'] !== UPLOAD_ERR_OK) {
+            $logger->warning('No valid file uploaded for tags import');
            $_SESSION['error'] = 'Please select a valid file to import';
            $this->redirect('/tags');
            return;
        }

        $file = $_FILES['import_file'];
+        $logger->info('Import file received', [
+            'filename' => $file['name'],
+            'size' => $file['size']
+        ]);

        // Validate file size (1MB max)
        if ($file['size'] > 1048576) {
+            $logger->warning('Import file too large', ['size' => $file['size']]);
            $_SESSION['error'] = 'File is too large. Maximum size is 1MB';
            $this->redirect('/tags');
            return;
@@ -194,6 +204,7 @@ class TagController extends Controller
        // Detect format from extension
        $ext = strtolower(pathinfo($file['name'], PATHINFO_EXTENSION));
        if (!in_array($ext, ['csv', 'json'])) {
+            $logger->warning('Invalid file type for tags import', ['extension' => $ext]);
            $_SESSION['error'] = 'Invalid file type. Please upload a CSV or JSON file';
            $this->redirect('/tags');
            return;
@@ -205,6 +216,7 @@ class TagController extends Controller
        if ($ext === 'json') {
            $parsed = json_decode($content, true);
            if (!is_array($parsed)) {
+                $logger->error('Invalid JSON file for tags import');
                $_SESSION['error'] = 'Invalid JSON file';
                $this->redirect('/tags');
                return;
@@ -228,12 +240,13 @@ class TagController extends Controller
        }

        if (empty($tagsData)) {
+            $logger->warning('No tags found in import file');
            $_SESSION['error'] = 'No tags found in file';
            $this->redirect('/tags');
            return;
        }

-        $userId = \Core\Auth::id();
+        $logger->info('Tags data parsed from file', ['entries' => count($tagsData)]);
        $colorMap = $this->tagModel->getAvailableColors(); // cssClass => 'Name'
        $availableColorClasses = array_keys($colorMap);
        // Build reverse map: lowercase name => cssClass (e.g. 'blue' => 'bg-blue-100 ...')
@@ -284,6 +297,11 @@ class TagController extends Controller
            $created++;
        }

+        $logger->info('Tags import completed', [
+            'created' => $created,
+            'skipped' => $skipped
+        ]);
+
        $_SESSION['success'] = "{$created} tag(s) imported successfully" . ($skipped > 0 ? ", {$skipped} skipped (already exist or invalid)" : '');
        $this->redirect('/tags');
    }
--- a/app/Controllers/TldRegistryController.php
+++ b/app/Controllers/TldRegistryController.php
@@ -640,6 +640,359 @@ class TldRegistryController extends Controller
        }
    }

+    /**
+     * Export TLD registry as CSV or JSON
+     */
+    public function export()
+    {
+        Auth::requireAdmin();
+
+        $logger = new \App\Services\Logger('export');
+
+        try {
+            $format = $_GET['format'] ?? 'csv';
+            $logger->info('TLD registry export started', [
+                'format' => $format,
+                'user_id' => $_SESSION['user_id'] ?? 'unknown'
+            ]);
+
+            if (!in_array($format, ['csv', 'json'])) {
+                $_SESSION['error'] = 'Invalid export format';
+                $this->redirect('/tld-registry');
+                return;
+            }
+
+            $allTlds = $this->tldModel->getAll();
+
+            $tlds = array_map(fn($t) => [
+                'tld' => $t['tld'],
+                'whois_server' => $t['whois_server'] ?? '',
+                'rdap_servers' => $t['rdap_servers'] ?? '',
+                'registry_url' => $t['registry_url'] ?? '',
+                'is_active' => $t['is_active'] ? 'yes' : 'no',
+            ], $allTlds);
+
+            $logger->info('TLD registry export data prepared', ['count' => count($tlds)]);
+
+            $date = date('Y-m-d');
+            $filename = "tld_registry_export_{$date}";
+
+            while (ob_get_level()) {
+                ob_end_clean();
+            }
+
+            if ($format === 'json') {
+                header('Content-Type: application/json');
+                header("Content-Disposition: attachment; filename=\"{$filename}.json\"");
+                header('Cache-Control: no-cache, must-revalidate');
+                header('Pragma: no-cache');
+                echo json_encode($tlds, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE);
+            } else {
+                $csvContent = $this->buildCsv($tlds, ['tld', 'whois_server', 'rdap_servers', 'registry_url', 'is_active']);
+                $logger->info('CSV content built', ['bytes' => strlen($csvContent)]);
+
+                header('Content-Type: text/csv; charset=utf-8');
+                header("Content-Disposition: attachment; filename=\"{$filename}.csv\"");
+                header('Content-Length: ' . strlen($csvContent));
+                header('Cache-Control: no-cache, must-revalidate');
+                header('Pragma: no-cache');
+                echo $csvContent;
+            }
+
+            $logger->info('TLD registry export completed successfully');
+            exit;
+
+        } catch (\Throwable $e) {
+            $logger->error('TLD registry export failed', [
+                'error' => $e->getMessage(),
+                'file' => $e->getFile(),
+                'line' => $e->getLine()
+            ]);
+            $_SESSION['error'] = 'Export failed: ' . $e->getMessage();
+            $this->redirect('/tld-registry');
+        }
+    }
+
+    private function buildCsv(array $rows, array $headers): string
+    {
+        $handle = fopen('php://temp', 'r+');
+        fputcsv($handle, $headers, ',', '"', '\\');
+        foreach ($rows as $row) {
+            fputcsv($handle, array_values($row), ',', '"', '\\');
+        }
+        rewind($handle);
+        $csv = stream_get_contents($handle);
+        fclose($handle);
+        return $csv;
+    }
+
+    /**
+     * Import TLDs from CSV or JSON file
+     */
+    public function import()
+    {
+        Auth::requireAdmin();
+
+        if ($_SERVER['REQUEST_METHOD'] !== 'POST') {
+            $this->redirect('/tld-registry');
+            return;
+        }
+
+        $this->verifyCsrf('/tld-registry');
+
+        $logger = new \App\Services\Logger('import');
+        $logger->info('TLD registry import started', [
+            'user_id' => $_SESSION['user_id'] ?? 'unknown',
+            'username' => $_SESSION['username'] ?? 'unknown'
+        ]);
+
+        if (!isset($_FILES['import_file']) || $_FILES['import_file']['error'] !== UPLOAD_ERR_OK) {
+            $logger->warning('No valid file uploaded for TLD import');
+            $_SESSION['error'] = 'Please select a valid file to import';
+            $this->redirect('/tld-registry');
+            return;
+        }
+
+        $file = $_FILES['import_file'];
+        $logger->info('Import file received', [
+            'filename' => $file['name'],
+            'size' => $file['size']
+        ]);
+
+        if ($file['size'] > 5242880) {
+            $logger->warning('Import file too large', ['size' => $file['size']]);
+            $_SESSION['error'] = 'File is too large. Maximum size is 5MB';
+            $this->redirect('/tld-registry');
+            return;
+        }
+
+        $ext = strtolower(pathinfo($file['name'], PATHINFO_EXTENSION));
+        if (!in_array($ext, ['csv', 'json'])) {
+            $logger->warning('Invalid file type for TLD import', ['extension' => $ext]);
+            $_SESSION['error'] = 'Invalid file type. Please upload a CSV or JSON file';
+            $this->redirect('/tld-registry');
+            return;
+        }
+
+        $content = file_get_contents($file['tmp_name']);
+        $tldsData = [];
+
+        if ($ext === 'json') {
+            $parsed = json_decode($content, true);
+            if (!is_array($parsed)) {
+                $logger->error('Invalid JSON file for TLD import');
+                $_SESSION['error'] = 'Invalid JSON file';
+                $this->redirect('/tld-registry');
+                return;
+            }
+            $tldsData = $parsed;
+        } else {
+            $lines = array_filter(explode("\n", $content));
+            $header = null;
+            foreach ($lines as $line) {
+                $row = str_getcsv(trim($line), ',', '"', '\\');
+                if (!$header) {
+                    $header = array_map('strtolower', array_map('trim', $row));
+                    continue;
+                }
+                $item = [];
+                foreach ($header as $i => $col) {
+                    $item[$col] = $row[$i] ?? '';
+                }
+                $tldsData[] = $item;
+            }
+        }
+
+        if (empty($tldsData)) {
+            $logger->warning('No TLD data found in import file');
+            $_SESSION['error'] = 'No TLD data found in the file';
+            $this->redirect('/tld-registry');
+            return;
+        }
+
+        $logger->info('TLD data parsed from file', ['entries' => count($tldsData)]);
+
+        $imported = 0;
+        $updated = 0;
+        $skipped = 0;
+
+        foreach ($tldsData as $tldRow) {
+            $tldName = trim($tldRow['tld'] ?? '');
+            if (empty($tldName)) {
+                $skipped++;
+                continue;
+            }
+
+            if (!str_starts_with($tldName, '.')) {
+                $tldName = '.' . $tldName;
+            }
+
+            $existing = $this->tldModel->findByTld($tldName);
+
+            $data = [
+                'tld' => $tldName,
+                'updated_at' => date('Y-m-d H:i:s'),
+            ];
+
+            if (!empty($tldRow['whois_server'])) {
+                $data['whois_server'] = trim($tldRow['whois_server']);
+            }
+            if (!empty($tldRow['rdap_servers'])) {
+                $rdap = trim($tldRow['rdap_servers']);
+                if (str_starts_with($rdap, '[')) {
+                    $data['rdap_servers'] = $rdap;
+                } else {
+                    $servers = array_filter(array_map('trim', preg_split('/[,;]+/', $rdap)));
+                    $data['rdap_servers'] = json_encode(array_values($servers));
+                }
+            }
+            if (!empty($tldRow['registry_url'])) {
+                $data['registry_url'] = trim($tldRow['registry_url']);
+            }
+            if (isset($tldRow['is_active'])) {
+                $val = strtolower(trim($tldRow['is_active']));
+                $data['is_active'] = in_array($val, ['yes', '1', 'true', 'active']) ? 1 : 0;
+            }
+
+            if ($existing) {
+                unset($data['tld']);
+                $this->tldModel->update($existing['id'], $data);
+                $updated++;
+            } else {
+                if (!isset($data['is_active'])) {
+                    $data['is_active'] = 1;
+                }
+                $data['created_at'] = date('Y-m-d H:i:s');
+                $this->tldModel->create($data);
+                $imported++;
+            }
+        }
+
+        $logger->info('TLD registry import completed', [
+            'imported' => $imported,
+            'updated' => $updated,
+            'skipped' => $skipped
+        ]);
+
+        $message = "Import completed: {$imported} new, {$updated} updated";
+        if ($skipped > 0) {
+            $message .= ", {$skipped} skipped";
+        }
+        $_SESSION['success'] = $message;
+        $this->redirect('/tld-registry');
+    }
+
+    /**
+     * Create a new TLD registry entry manually
+     */
+    public function createTld()
+    {
+        Auth::requireAdmin();
+
+        if ($_SERVER['REQUEST_METHOD'] !== 'POST') {
+            $this->redirect('/tld-registry');
+            return;
+        }
+
+        $this->verifyCsrf('/tld-registry');
+
+        $tldName = trim($_POST['tld'] ?? '');
+        $whoisServer = trim($_POST['whois_server'] ?? '');
+        $rdapServersInput = trim($_POST['rdap_servers'] ?? '');
+        $registryUrl = trim($_POST['registry_url'] ?? '');
+
+        $this->logger->info('Manual TLD creation requested', [
+            'tld' => $tldName,
+            'user_id' => $_SESSION['user_id'] ?? 'unknown',
+            'username' => $_SESSION['username'] ?? 'unknown'
+        ]);
+
+        if (empty($tldName)) {
+            $_SESSION['error'] = 'TLD name is required';
+            $this->redirect('/tld-registry');
+            return;
+        }
+
+        if (!str_starts_with($tldName, '.')) {
+            $tldName = '.' . $tldName;
+        }
+
+        $tldName = strtolower($tldName);
+
+        if (!preg_match('/^\.[a-z0-9\-]+(\.[a-z0-9\-]+)*$/', $tldName)) {
+            $this->logger->warning('Invalid TLD format provided', ['tld' => $tldName]);
+            $_SESSION['error'] = 'Invalid TLD format. Use only letters, numbers, hyphens, and dots for multi-level TLDs (e.g., .co.uk).';
+            $this->redirect('/tld-registry');
+            return;
+        }
+
+        $existing = $this->tldModel->findByTld($tldName);
+        if ($existing) {
+            $this->logger->warning('Attempted to create duplicate TLD', ['tld' => $tldName]);
+            $_SESSION['error'] = "TLD {$tldName} already exists in the registry";
+            $this->redirect('/tld-registry');
+            return;
+        }
+
+        if (!empty($whoisServer) && !preg_match('/^[a-z0-9.-]+\.[a-z]{2,}$/i', $whoisServer)) {
+            $_SESSION['error'] = 'Invalid WHOIS server format';
+            $this->redirect('/tld-registry');
+            return;
+        }
+
+        $rdapServers = [];
+        if (!empty($rdapServersInput)) {
+            $servers = preg_split('/[,\n\r]+/', $rdapServersInput);
+            foreach ($servers as $server) {
+                $server = trim($server);
+                if (!empty($server)) {
+                    $server = rtrim($server, '/') . '/';
+                    $rdapServers[] = $server;
+                }
+            }
+        }
+
+        try {
+            $data = [
+                'tld' => $tldName,
+                'is_active' => 1,
+                'created_at' => date('Y-m-d H:i:s'),
+                'updated_at' => date('Y-m-d H:i:s'),
+            ];
+
+            if (!empty($whoisServer)) {
+                $data['whois_server'] = $whoisServer;
+            }
+            if (!empty($rdapServers)) {
+                $data['rdap_servers'] = json_encode($rdapServers);
+            }
+            if (!empty($registryUrl)) {
+                $data['registry_url'] = $registryUrl;
+            }
+
+            $this->tldModel->create($data);
+
+            $this->logger->info('TLD created successfully', [
+                'tld' => $tldName,
+                'whois_server' => $whoisServer ?: null,
+                'rdap_servers_count' => count($rdapServers),
+                'registry_url' => $registryUrl ?: null
+            ]);
+
+            $_SESSION['success'] = "TLD {$tldName} created successfully";
+        } catch (\Exception $e) {
+            $_SESSION['error'] = 'Failed to create TLD: ' . $e->getMessage();
+            $this->logger->error('Failed to create TLD', [
+                'tld' => $tldName,
+                'error' => $e->getMessage(),
+                'file' => $e->getFile(),
+                'line' => $e->getLine()
+            ]);
+        }
+
+        $this->redirect('/tld-registry');
+    }
+
    /**
     * Extract WHOIS server from HTML
     */