feat: add full Zonemaster stack with Docker and Spanish UI

- Clone all 5 Zonemaster component repos (LDNS, Engine, CLI, Backend, GUI)
- Dockerfile.backend: 8-stage multi-stage build LDNS→Engine→CLI→Backend
- Dockerfile.gui: Astro static build served via nginx
- docker-compose.yml: backend (internal) + frontend (port 5353)
- nginx.conf: root redirects to /es/, /api/ proxied to backend
- zonemaster-gui/config.ts: defaultLanguage set to 'es' (Spanish)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-04-21 08:19:24 +02:00
commit 8d4eaa1489
1567 changed files with 204155 additions and 0 deletions

View File

@@ -0,0 +1,56 @@
.POSIX:
.SUFFIXES: .po .mo
.PHONY: all check-msg-args dist extract-pot tidy-po show-fuzzy update-po new-po check-po
POFILES := $(shell find . -maxdepth 1 -type f -name '*.po' -exec basename {} \;)
MOFILES := $(POFILES:%.po=%.mo)
POTFILE = Zonemaster-Backend.pot
PMFILES := $(shell find ../lib -type f -name '*.pm' | sort)
all: $(MOFILES)
@echo
@echo Remember to make sure all of the above names are in the
@echo MANIFEST file, or they will not be installed.
@echo
# Tidy the formatting of all PO files
tidy-po:
@tmpdir="`mktemp -d tidy-po.XXXXXXXX`" ;\
trap 'rm -rf "$$tmpdir"' EXIT ;\
for f in $(POFILES) ; do msgcat $$f -o $$tmpdir/$$f && mv -f $$tmpdir/$$f $$f ; done
update-po: extract-pot
@for f in $(POFILES) ; do msgmerge --update --backup=none --quiet --no-location $(MSGMERGE_OPTS) $$f $(POTFILE) ; done
# Create a new empty PO file with basename provided with the POLANG variable
# Update the Language field in the header
new-po: extract-pot
@[ -n "$(POLANG)" ] || ( echo "Usage: make POLANG=xx new-po" && exit 1 )
@cp $(POTFILE) $(POLANG).po
@perl -pi -e 's/^("Project-Id-Version:) .+(\\n)/$$1 1.0.0$$2/;' \
-e 's/^("Language-Team:) .+(\\n)/$$1 Zonemaster Team$$2/;' \
-e 's/^"Language: /$$&$(POLANG)/;' \
-e 's/^("Content-Type:.+charset=)CHARSET/$${1}UTF-8/;' $(POLANG).po
@perl -ni -e 'print unless /^#( |$$)/' $(POLANG).po
# Check the msgid/msgstr pair for some inconsistencies between them in the
# selected PO file and report on standard error any errors found. The PO file
# is not updated.
check-po:
@for f in $(POFILES) ; do msgfmt -c $$f ; done
extract-pot:
@xgettext --output $(POTFILE) --sort-by-file --add-comments --language=Perl --from-code=UTF-8 -k__ -k\$$__ -k%__ -k__x -k__n:1,2 -k__nx:1,2 -k__xn:1,2 -kN__ -kN__n:1,2 -k__p:1c,2 -k__np:1c,2,3 -kN__p:1c,2 -kN__np:1c,2,3 $(PMFILES)
$(POTFILE): extract-pot
.po.mo:
@msgfmt -o $@ $<
@mkdir -p locale/`basename $@ .mo`/LC_MESSAGES
@ln -vf $@ locale/`basename $@ .mo`/LC_MESSAGES/Zonemaster-Backend.mo
show-fuzzy:
@for f in $(POFILES) ; do msgattrib --only-fuzzy $$f ; done
check-msg-args:
@for f in $(POFILES) ; do ../util/check-msg-args $$f ; done

View File

@@ -0,0 +1,14 @@
# This is a wrapper for BSD Make (FreeBSD) to execute
# GNU Make (gmake) and the primary makefile GNUmakefile.
GNUMAKE ?= gmake
FILES != ls *
# File targets should be evaluated by gmake.
.PHONY: all $(FILES)
all:
@${GNUMAKE} $@
.DEFAULT:
@${GNUMAKE} $@

View File

@@ -0,0 +1,20 @@
[DB]
engine=MySQL
polling_interval=0.5
#seconds
[MYSQL]
host=localhost
database=zonemaster
user=ci
password=password
[ZONEMASTER]
max_zonemaster_execution_time=300
number_of_processes_for_frontend_testing=20
number_of_processes_for_batch_testing=20
#seconds
[LANGUAGE]
locale = da_DK en_US es_ES fi_FI fr_FR nb_NO sv_SE

View File

@@ -0,0 +1,20 @@
[DB]
engine=PostgreSQL
polling_interval=0.5
#seconds
[POSTGRESQL]
host=localhost
database=zonemaster
user=ci
password=password
[ZONEMASTER]
max_zonemaster_execution_time=300
number_of_processes_for_frontend_testing=20
number_of_processes_for_batch_testing=20
#seconds
[LANGUAGE]
locale = da_DK en_US es_ES fi_FI fr_FR nb_NO sv_SE

View File

@@ -0,0 +1,17 @@
[DB]
engine=SQLite
polling_interval=0.5
#seconds
[SQLITE]
database_file=/tmp/zonemaster.sqlite
[ZONEMASTER]
max_zonemaster_execution_time=300
number_of_processes_for_frontend_testing=20
number_of_processes_for_batch_testing=20
#seconds
[LANGUAGE]
locale = da_DK en_US es_ES fi_FI fr_FR nb_NO sv_SE

View File

@@ -0,0 +1,50 @@
# For documentation of the backend_config.ini file see
# https://github.com/zonemaster/zonemaster/blob/master/docs/public/configuration/backend.md
[DB]
engine = SQLite
polling_interval = 0.5
[MYSQL]
host = localhost
user = zonemaster
password = zonemaster
database = zonemaster
[POSTGRESQL]
host = localhost
user = zonemaster
password = zonemaster
database = zonemaster
[SQLITE]
database_file = /var/lib/zonemaster/db.sqlite
[ZONEMASTER]
#max_zonemaster_execution_time = 600
#number_of_processes_for_frontend_testing = 20
#number_of_processes_for_batch_testing = 20
#lock_on_queue = 0
#age_reuse_previous_test = 600
[RPCAPI]
# Uncomment to enable API method "add_api_user"
#enable_add_api_user = yes
# Uncomment to disable API method "add_batch_job"
#enable_add_batch_job = no
[LANGUAGE]
locale = da_DK en_US es_ES fi_FI fr_FR nb_NO sl_SI sv_SE
[PUBLIC PROFILES]
#example_profile_1=/example/directory/test1_profile.json
#default=/example/directory/default_profile.json
[PRIVATE PROFILES]
#example_profile_2=/example/directory/test2_profile.json
[METRICS]
# Uncoment the following option to enable the metrics feature
#statsd_host = localhost
#statsd_port = 8125

View File

@@ -0,0 +1,3 @@
-- Remove Zonemaster data from database
DROP DATABASE zonemaster;
DROP USER 'zonemaster'@'localhost';

View File

@@ -0,0 +1,3 @@
-- Remove Zonemaster data from database
DROP DATABASE zonemaster;
DROP USER zonemaster;

View File

@@ -0,0 +1,15 @@
#!/usr/bin/env perl
use strict;
use warnings;
use Zonemaster::Backend::Config;
use Zonemaster::Backend::DB;
my $config = Zonemaster::Backend::Config->load_config();
my $db_engine = $config->DB_engine;
my $db_class = Zonemaster::Backend::DB->get_db_class( $db_engine );
my $db = $db_class->from_config( $config );
$db->create_schema();

View File

@@ -0,0 +1,89 @@
msgid ""
msgstr ""
"Project-Id-Version: 1.0.0\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2023-05-26 14:36+0000\n"
"PO-Revision-Date: 2023-05-26 14:33+0000\n"
"Last-Translator: haarbo@dk-hostmaster.dk\n"
"Language-Team: Zonemaster project\n"
"Language: da\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
msgid "Invalid method parameter(s)."
msgstr "Invalid metodeparametre"
msgid "Missing property"
msgstr "Manglende egenskab"
msgid ""
"Warning: Zonemaster::LDNS not compiled with IDN support, cannot handle non-"
"ASCII names correctly."
msgstr ""
"Advarsel: Zonemaster::LDNS ikke kompileret med IDN-support, og kan derfor "
"ikke håndtere ikke-ascii-navne korrekt"
#. BACKEND_TEST_AGENT:TEST_DIED
msgid "An error occured and Zonemaster could not start or finish the test."
msgstr ""
"Der opstod en fejl, og Zonemaster kunne ikke starte eller afslutte testen."
#. BACKEND_TEST_AGENT:UNABLE_TO_FINISH_TEST
#, perl-brace-format
msgid ""
"The test took too long to run (the current limit is {max_execution_time} "
"seconds). Maybe there are too many name servers or the name servers are "
"either unreachable or not responsive enough."
msgstr ""
"Testen tog for lang tid at afvikle (den aktuelle grænse er "
"{max_execution_time} sekunder). Måske er der for mange navneservere, eller "
"også er navneserverne enten ikke tilgængelige eller ikke responsive nok."
msgid "Invalid digest format"
msgstr "Invalid digest-format"
msgid "Algorithm must be a positive integer"
msgstr "Algoritmen skal være et positivt heltal"
msgid "Digest type must be a positive integer"
msgstr "Digest type skal være et positivt heltal"
msgid "Keytag must be a positive integer"
msgstr "Keytag skal være et positivt heltal"
msgid "Domain name required"
msgstr "Domænenavn påkrævet"
msgid "The domain name is IDNA invalid"
msgstr "Domænenavnet er IDNA ugyldigt"
msgid ""
"The domain name contains non-ascii characters and IDNA support is not "
"installed"
msgstr ""
"Domænenavnet indeholder ikke-ascii-tegn og IDNA-support er ikke installeret"
msgid "The domain name character(s) are not supported"
msgstr "Domænenavnets tegn er ikke understøttet"
msgid "The domain name contains consecutive dots"
msgstr "Domænenavnet indeholder flere dots (.) efter hinanden"
msgid "The domain name or label is too long"
msgstr "Domænenavnet eller labelen er for lang"
msgid "Invalid language tag format"
msgstr "Invalidt sprogkode-format"
msgid "Unkown language string"
msgstr "Ukendt sprogstreng"
msgid "Invalid IP address"
msgstr "Invalid IP-adresse"
msgid "Invalid profile format"
msgstr "Invalidt profil-format"
msgid "Unknown profile"
msgstr "Ukendt profil"

View File

@@ -0,0 +1,91 @@
msgid ""
msgstr ""
"Project-Id-Version: 1.0.0\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2023-05-21 21:33+0000\n"
"PO-Revision-Date: 2023-05-21 21:32+0000\n"
"Last-Translator: hsalgado@vulcano.cl\n"
"Language-Team: Zonemaster project\n"
"Language: es\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"X-Generator: Poedit 3.0\n"
msgid "Invalid method parameter(s)."
msgstr "Parámetro(s) de método inválido."
msgid "Missing property"
msgstr "Propiedad no incluída"
msgid ""
"Warning: Zonemaster::LDNS not compiled with IDN support, cannot handle non-"
"ASCII names correctly."
msgstr ""
"Advertencia: Zonemaster::LDNS no fue compilado con soporte IDN, no se puede "
"manejar correctamente los nombres no-ASCII."
#. BACKEND_TEST_AGENT:TEST_DIED
msgid "An error occured and Zonemaster could not start or finish the test."
msgstr ""
"Ha ocurrido un error y Zonemaster no pudo comenzar o finalizar la prueba."
#. BACKEND_TEST_AGENT:UNABLE_TO_FINISH_TEST
#, perl-brace-format
msgid ""
"The test took too long to run (the current limit is {max_execution_time} "
"seconds). Maybe there are too many name servers or the name servers are "
"either unreachable or not responsive enough."
msgstr ""
"La prueba tomó demasiado tiempo en terminar (el límite máximo actual es "
"{max_execution_time} segundos). Quizás hay demasiados servidores de nombres, "
"o son inalcanzables, o no lo suficientemente rápidos para responder."
msgid "Invalid digest format"
msgstr "Formato de resumen (digest) inválido"
msgid "Algorithm must be a positive integer"
msgstr "El algoritmo debe ser un entero positivo"
msgid "Digest type must be a positive integer"
msgstr "El tipo de resumen (digest) debe ser un entero positivo"
msgid "Keytag must be a positive integer"
msgstr "El tag de llave debe ser un entero positivo"
msgid "Domain name required"
msgstr "Se necesita el nombre de dominio"
msgid "The domain name is IDNA invalid"
msgstr "El nombre de dominio es inválido según IDNA"
msgid ""
"The domain name contains non-ascii characters and IDNA support is not "
"installed"
msgstr ""
"El nombre de dominio contiene caracteres no-ascii, y el soporte IDNA no está "
"instalado"
msgid "The domain name character(s) are not supported"
msgstr "Los caracteres del nombre de dominio no están soportados"
msgid "The domain name contains consecutive dots"
msgstr "El nombre de dominio contiene puntos consecutivos"
msgid "The domain name or label is too long"
msgstr "El nombre de dominio o la etiqueta es muy larga"
msgid "Invalid language tag format"
msgstr "Formato de descriptor de idioma inválido"
msgid "Unkown language string"
msgstr "Descriptor de idioma desconocido"
msgid "Invalid IP address"
msgstr "Dirección IP inválida"
msgid "Invalid profile format"
msgstr "Formato de perfil (profile) inválido"
msgid "Unknown profile"
msgstr "Perfil desconocido"

View File

@@ -0,0 +1,91 @@
msgid ""
msgstr ""
"Project-Id-Version: 1.0.0\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2023-05-23 14:04+0000\n"
"PO-Revision-Date: 2023-05-23 14:03+0000\n"
"Last-Translator: mats.dufberg@iis.se\n"
"Language-Team: Zonemaster project\n"
"Language: fi\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"X-Generator: Poedit 3.0\n"
msgid "Invalid method parameter(s)."
msgstr "Virheelliset asetukset"
msgid "Missing property"
msgstr "Kenttä puuttuu"
msgid ""
"Warning: Zonemaster::LDNS not compiled with IDN support, cannot handle non-"
"ASCII names correctly."
msgstr ""
"Varoitus: Zonemaster::LDNS ei ole käännetty IDN tuella, joten se ei pysty "
"käsittelemään ei-ASCII-nimiä oikein."
#. BACKEND_TEST_AGENT:TEST_DIED
msgid "An error occured and Zonemaster could not start or finish the test."
msgstr ""
"Tapahtui virhe, eikä Zonemaster voinut aloittaa tai suorittaa testiä loppuun."
#. BACKEND_TEST_AGENT:UNABLE_TO_FINISH_TEST
#, perl-brace-format
msgid ""
"The test took too long to run (the current limit is {max_execution_time} "
"seconds). Maybe there are too many name servers or the name servers are "
"either unreachable or not responsive enough."
msgstr ""
"Testin suorittaminen kesti liian kauan (nykyinen raja on "
"{max_execution_time} sekuntia). Ehkä nimipalvelimia on liian monta, "
"nimipalvelimiin ei saada yhteyttä, tai ne eivät vastaa tarpeeksi nopeasti."
msgid "Invalid digest format"
msgstr "Virheellinen tiivisteen muoto"
msgid "Algorithm must be a positive integer"
msgstr "Algoritmin on oltava positiivinen kokonaisluku"
msgid "Digest type must be a positive integer"
msgstr "Tiivistetyypin (Digest type) on oltava positiivinen kokonaisluku"
msgid "Keytag must be a positive integer"
msgstr "Tunnisteen (Keytag) on oltava positiivinen kokonaisluku"
msgid "Domain name required"
msgstr "Vaaditaan verkkotunnus"
msgid "The domain name is IDNA invalid"
msgstr "Verkkotunnus on IDNA virheellinen"
msgid ""
"The domain name contains non-ascii characters and IDNA support is not "
"installed"
msgstr ""
"Verkkotunnuksen nimi sisältää muita kuin ascii-merkkejä, eikä IDNA tukea ole "
"asennettu"
msgid "The domain name character(s) are not supported"
msgstr "Verkkotunnuksen sisältämiä merkkejä ei tueta"
msgid "The domain name contains consecutive dots"
msgstr "Verkkotunnus sisältää peräkkäisiä pisteitä"
msgid "The domain name or label is too long"
msgstr "Verkkotunnus tai sen tunnisteet ovat liian pitkiä"
msgid "Invalid language tag format"
msgstr "Virheellinen kielitunnisteen muoto"
msgid "Unkown language string"
msgstr "Tuntematon kielitunniste"
msgid "Invalid IP address"
msgstr "Virheellinen IP-osoite"
msgid "Invalid profile format"
msgstr "Virheellinen profiilin muoto"
msgid "Unknown profile"
msgstr "Tuntematon profiili"

View File

@@ -0,0 +1,71 @@
msgid ""
msgstr ""
"Project-Id-Version: 1.0.0\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2025-02-18 11:20+0100\n"
"PO-Revision-Date: 2023-05-22 07:17+0200\n"
"Last-Translator: thomas.green@afnic.fr\n"
"Language-Team: Zonemaster project\n"
"Language: fr\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
msgid "Invalid method parameter(s)."
msgstr "Paramètre(s) incorrect(s)."
msgid "Missing property"
msgstr "Propriété manquante"
msgid ""
"Warning: Zonemaster::LDNS not compiled with IDN support, cannot handle non-"
"ASCII names correctly."
msgstr ""
"Attention : Zonemaster::LDNS n'est pas compilé avec le support IDN, "
"impossible de traiter correctement les noms non ASCII."
#. BACKEND_TEST_AGENT:TEST_DIED
msgid "An error occured and Zonemaster could not start or finish the test."
msgstr ""
"Une erreur est survenue et Zonemaster na pas pu commencer ou finir le test."
#. BACKEND_TEST_AGENT:UNABLE_TO_FINISH_TEST
#, perl-brace-format
msgid ""
"The test took too long to run (the current limit is {max_execution_time} "
"seconds). Maybe there are too many name servers or the name servers are "
"either unreachable or not responsive enough."
msgstr ""
"Le test a mis trop de temps à sexécuter (la limite actuelle est de "
"{max_execution_time} secondes). Il y a peut-être trop de serveurs de noms, "
"ou les serveurs de noms sont injoignables ou trop peu réactifs."
msgid "Invalid digest format"
msgstr "Format du condensat non valide"
msgid "Algorithm must be a positive integer"
msgstr "L'algorithme doit être un entier positif"
msgid "Digest type must be a positive integer"
msgstr "Le type d'empreinte doit être un entier positif"
msgid "Keytag must be a positive integer"
msgstr "L'identifiant de clef doit être un entier positif"
msgid "Domain name required"
msgstr "Nom de domaine requis"
msgid "Invalid language tag format"
msgstr "Format de l'étiquette d'identification de langue incorrect"
msgid "Unkown language string"
msgstr "Étiquette d'identification de langue inconnue"
msgid "Invalid IP address"
msgstr "Adresse IP non valide"
msgid "Invalid profile format"
msgstr "Format du profil non valide"
msgid "Unknown profile"
msgstr "Profil inconnu"

View File

@@ -0,0 +1,7 @@
# Range of free system UIDs that can be used for Zonemaster user
minuid = 736
maxuid = 769
# Range of free system GIDs that can be used for Zonemaster group
mingid = 736
maxgid = 769

View File

@@ -0,0 +1,71 @@
msgid ""
msgstr ""
"Project-Id-Version: 1.0.0\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2023-05-24 09:36+0000\n"
"PO-Revision-Date: 2025-04-25 08:30+0200\n"
"Last-Translator: richard.persson@norid.no\n"
"Language-Team: Zonemaster project\n"
"Language: nb\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
msgid "Invalid method parameter(s)."
msgstr "Ugyldig metodeparameter."
msgid "Missing property"
msgstr "Mangler verdi"
msgid ""
"Warning: Zonemaster::LDNS not compiled with IDN support, cannot handle non-"
"ASCII names correctly."
msgstr ""
"Advarsel: Zonemaster::LDNS er ikke kompilert med IDN-støtte. Kan bare "
"håndtere ASCII-navn."
#. BACKEND_TEST_AGENT:TEST_DIED
msgid "An error occured and Zonemaster could not start or finish the test."
msgstr ""
"Det oppstod en feil og Zonemaster kunne ikke starte eller fullføre testen."
#. BACKEND_TEST_AGENT:UNABLE_TO_FINISH_TEST
#, perl-brace-format
msgid ""
"The test took too long to run (the current limit is {max_execution_time} "
"seconds). Maybe there are too many name servers or the name servers are "
"either unreachable or not responsive enough."
msgstr ""
"Testen tok for lang tid å kjøre (gjeldende grense er {max_execution_time} "
"sekunder). Kanskje er det for mange navnetjenere eller navnetjenerne er "
"enten utilgjengelige eller ikke responsive nok."
msgid "Invalid digest format"
msgstr "Ugyldig format på digest"
msgid "Algorithm must be a positive integer"
msgstr "Algoritme må være et positivt tall"
msgid "Digest type must be a positive integer"
msgstr "Algoritme-type må være et positivt tall"
msgid "Keytag must be a positive integer"
msgstr "Keytag må være et positivt tall"
msgid "Domain name required"
msgstr "Domenenavn påkrevd"
msgid "Invalid language tag format"
msgstr "Ugyldig format på språk-tag"
msgid "Unkown language string"
msgstr "Ukjent språk-tag"
msgid "Invalid IP address"
msgstr "Ugylding IP-adresse"
msgid "Invalid profile format"
msgstr "Ugyldig format på profil"
msgid "Unknown profile"
msgstr "Ukjent profil"

View File

@@ -0,0 +1,2 @@
Find instructions on patching (upgrading) the Zonemaster database
on https://github.com/zonemaster/zonemaster/blob/master/docs/public/upgrading/backend.md

View File

@@ -0,0 +1,330 @@
use strict;
use warnings;
use List::MoreUtils qw(zip_unflatten);
use JSON::PP;
use Try::Tiny;
use File::Temp qw(tempfile);
use Encode qw(find_encoding);
use Zonemaster::Backend::Config;
use Zonemaster::Engine;
my $config = Zonemaster::Backend::Config->load_config();
my %module_mapping;
for my $module ( Zonemaster::Engine->modules ) {
$module_mapping{lc $module} = $module;
}
my %patch = (
mysql => \&patch_db_mysql,
postgresql => \&patch_db_postgresql,
sqlite => \&patch_db_sqlite,
);
my $db_engine = $config->DB_engine;
print "Configured database engine: $db_engine\n";
if ( $db_engine =~ /^(MySQL|PostgreSQL|SQLite)$/ ) {
print( "Starting database migration\n" );
$patch{ lc $db_engine }();
print( "\nMigration done\n" );
}
else {
die "Unknown database engine configured: $db_engine\n";
}
# depending on the resources available to select all data in database
# update $row_count to your needs
sub _update_data_result_entries {
my ( $dbh, $row_count ) = @_;
my $json = JSON::PP->new->allow_blessed->convert_blessed->canonical;
# update only jobs with results
my ( $row_total ) = $dbh->selectrow_array( 'SELECT count(*) FROM test_results WHERE results IS NOT NULL' );
print "Will update $row_total rows\n";
my %levels = Zonemaster::Engine::Logger::Entry->levels();
my $row_done = 0;
while ( $row_done < $row_total ) {
print "Progress update: $row_done / $row_total\n";
my $row_updated = 0;
my $sth1 = $dbh->prepare( 'SELECT hash_id, results FROM test_results WHERE results IS NOT NULL ORDER BY id ASC LIMIT ? OFFSET ?' );
$sth1->execute( $row_count, $row_done );
while ( my $row = $sth1->fetchrow_arrayref ) {
my ( $hash_id, $results ) = @$row;
next unless $results;
my @records;
my $entries = $json->decode( $results );
foreach my $m ( @$entries ) {
my $module = $module_mapping{ lc $m->{module} } // ucfirst lc $m->{module};
my $testcase =
( !defined $m->{testcase} or $m->{testcase} eq 'UNSPECIFIED' )
? 'Unspecified'
: $m->{testcase} =~ s/[a-z_]*/$module/ir;
if ($testcase eq 'Delegation01' and $m->{tag} =~ /^(NOT_)?ENOUGH_IPV[46]_NS_(CHILD|DEL)$/) {
my @ips = split( /;/, delete $m->{args}{ns_ip_list} );
my @names = split( /;/, delete $m->{args}{nsname_list} );
my @ns_list = map { join( '/', @$_ ) } zip_unflatten(@names, @ips);
$m->{args}{ns_list} = join( ';', @ns_list );
}
my $r = [
$hash_id,
$levels{ $m->{level} },
$module,
$testcase,
$m->{tag},
$m->{timestamp},
$json->encode( $m->{args} // {} ),
];
push @records, $r;
}
my $query_values = join ", ", ("(?, ?, ?, ?, ?, ?, ?)") x @records;
my $query = "INSERT INTO result_entries (hash_id, level, module, testcase, tag, timestamp, args) VALUES $query_values";
my $sth = $dbh->prepare( $query );
$sth = $sth->execute( map { @$_ } @records );
$row_updated += $dbh->do( "UPDATE test_results SET results = NULL WHERE hash_id = ?", undef, $hash_id );
}
# increase by min(row_updated, row_count)
$row_done += ( $row_updated < $row_count ) ? $row_updated : $row_count;
}
print "Progress update: $row_done / $row_total\n";
}
sub _update_data_normalize_domains {
my ( $db ) = @_;
my ( $row_total ) = $db->dbh->selectrow_array( 'SELECT count(*) FROM test_results' );
print "Will update $row_total rows\n";
my $sth1 = $db->dbh->prepare( 'SELECT hash_id, params FROM test_results' );
$sth1->execute;
my $row_done = 0;
my $progress = 0;
while ( my $row = $sth1->fetchrow_hashref ) {
my $hash_id = $row->{hash_id};
eval {
my $raw_params = decode_json($row->{params});
my $domain = $raw_params->{domain};
# This has never been cleaned
delete $raw_params->{user_ip};
my $params = $db->encode_params( $raw_params );
my $fingerprint = $db->generate_fingerprint( $raw_params );
$domain = Zonemaster::Backend::DB::_normalize_domain( $domain );
$db->dbh->do('UPDATE test_results SET domain = ?, params = ?, fingerprint = ? where hash_id = ?', undef, $domain, $params, $fingerprint, $hash_id);
};
if ($@) {
warn "Caught error while updating record with hash id $hash_id, ignoring: $@\n";
}
$row_done += 1;
my $new_progress = int(($row_done / $row_total) * 100);
if ( $new_progress != $progress ) {
$progress = $new_progress;
print("$progress%\n");
}
}
}
sub patch_db_mysql {
use Zonemaster::Backend::DB::MySQL;
my $db = Zonemaster::Backend::DB::MySQL->from_config( $config );
my $dbh = $db->dbh;
$dbh->{AutoCommit} = 0;
try {
$db->create_schema();
print( "\n-> (1/2) Populating new result_entries table\n" );
_update_data_result_entries( $dbh, 50000 );
print( "\n-> (2/2) Normalizing domain names\n" );
_update_data_normalize_domains( $db );
$dbh->commit();
} catch {
print( "\nCould not upgrade database: " . $_ );
$dbh->rollback();
};
}
sub _patch_db_postgresql_step1 {
my ($dbh, $chunk_size) = @_;
$chunk_size //= 100_000;
# This is used later for backslash-escaping data supplied to COPY … FROM
# STDIN commands.
my %conv = ( 8 => '\b', 9 => '\t', 10 => '\n', 11 => '\v', 12 => '\f', 13 => '\r', 92 => '\\\\' );
my $utf8 = find_encoding('utf8');
# Why a cursor instead of a plain SELECT statement? Because DBD::Pg does
# not use server-side cursors itself when reading the result of a SELECT
# query.
#
# And why is that a problem? Thats because the DBMS will try to compute
# the entire result set before handing it to the client. With large
# Zonemaster setups with years of history and millions of tests, this
# SELECT statement will generate hundreds of millions of rows. So without
# the appropriate precautions, a plain SELECT query like this one will
# definitely take out the machine it is running on!
print("Starting up\n");
$dbh->do(q[
DECLARE curs NO SCROLL CURSOR FOR
SELECT
test_results.hash_id,
log_level.value AS level,
CASE res.module
WHEN 'DNSSEC' THEN res.module
ELSE initcap(res.module)
END AS module,
CASE
WHEN res.testcase IS NULL THEN ''
WHEN res.testcase LIKE 'DNSSEC%' THEN res.testcase
ELSE initcap(res.testcase)
END AS testcase,
res.tag AS tag,
res.timestamp AS timestamp,
COALESCE(migrated_args.args, '{}') AS args
FROM test_results,
json_to_recordset(results)
AS res(module TEXT, testcase TEXT, tag TEXT, level TEXT, timestamp REAL, args JSONB)
LEFT JOIN log_level ON (res.level = log_level.level)
LEFT JOIN LATERAL (
SELECT CASE WHEN res.testcase = 'DELEGATION01'
AND res.tag ~ '^(NOT_)?ENOUGH_IPV[46]_NS_(CHILD|DEL)$'
AND (NOT res.args ? 'ns_list')
THEN (
SELECT res.args
- ARRAY['ns_ip_list', 'nsname_list']
|| jsonb_build_object('ns_list', string_agg(name || '/' || ip, ';'))
FROM unnest(
string_to_array(res.args->>'ns_ip_list', ';'),
string_to_array(res.args->>'nsname_list', ';'))
AS unnest(ip, name))
ELSE res.args
END) AS migrated_args(args) ON TRUE]);
# Ive tried to avoid hardcoding numbers but FETCH statements somehow
# dont like being parameterized with placeholders. This will have to do.
my $read_sth = $dbh->prepare(sprintf(q[FETCH FORWARD %d FROM curs], $chunk_size));
my $row_inserted = 0;
while ($read_sth->execute(), (my $row_count = $read_sth->rows()) > 0) {
my @copydata = ();
print("Progress update: ${row_inserted} rows inserted\n");
$row_inserted += $row_count;
$dbh->do(q[COPY result_entries FROM STDIN]);
while (my $row = $read_sth->fetchrow_arrayref) {
my @columns = map {
if (defined $_) {
# Replaces invalid UTF-8 sequences with U+FFFD and escapes
# characters as required by PostgreSQLs text COPY data
# format.
$utf8->encode($utf8->decode($_) =~ s/[\x08-\x0D\\]/$conv{ord $&}/aegr);
} else {
'\N';
}
} @$row;
my $line = join("\t", @columns) . "\n";
push @copydata, $line;
$dbh->pg_putcopydata( $line );
}
try {
$dbh->pg_putcopyend();
}
catch {
print("An error occurred while trying to copy some data.\n");
my ($fh, $filename) = tempfile();
print $fh @copydata;
close $fh;
print("The data supplied to COPY causing the failure has been ",
"stored in $filename for inspection\n");
die $_;
}
}
$dbh->do(q[CLOSE curs]);
print("Done inserting ${row_inserted} rows\n");
}
sub patch_db_postgresql {
use Zonemaster::Backend::DB::PostgreSQL;
my $db = Zonemaster::Backend::DB::PostgreSQL->from_config( $config );
my $dbh = $db->dbh;
$dbh->{AutoCommit} = 0;
try {
$db->create_schema();
# Make sure the planner knows that log_level is a small table
# so it can optimize step 1 appropriately
$dbh->do(q[ANALYZE log_level]);
print( "\n-> (1/2) Populating new result_entries table\n" );
_patch_db_postgresql_step1( $dbh );
$dbh->do(
'UPDATE test_results SET results = NULL WHERE results IS NOT NULL'
);
print( "\n-> (2/2) Normalizing domain names\n" );
_update_data_normalize_domains( $db );
$dbh->commit();
} catch {
print( "\nCould not upgrade database: " . $_ );
$dbh->rollback();
};
}
sub patch_db_sqlite {
use Zonemaster::Backend::DB::SQLite;
my $db = Zonemaster::Backend::DB::SQLite->from_config( $config );
my $dbh = $db->dbh;
$dbh->{AutoCommit} = 0;
try {
$db->create_schema();
print( "\n-> (1/2) Populating new result_entries table\n" );
_update_data_result_entries( $dbh, 142 );
print( "\n-> (2/2) Normalizing domain names\n" );
_update_data_normalize_domains( $db );
$dbh->commit();
} catch {
print( "\nError while upgrading database: " . $_ );
$dbh->rollback();
};
}

View File

@@ -0,0 +1,32 @@
use strict;
use warnings;
use Zonemaster::Backend::Config;
use Zonemaster::Engine;
my $config = Zonemaster::Backend::Config->load_config();
my $db_engine = $config->DB_engine;
print "Configured database engine: $db_engine\n";
if ( $db_engine =~ /^(MySQL|PostgreSQL|SQLite)$/ ) {
print( "Starting database migration\n" );
_update_result_entries( $config->new_DB()->dbh() );
print( "\nMigration done\n" );
}
else {
die "Unknown database engine configured: $db_engine\n";
}
sub _update_result_entries {
my ( $dbh ) = @_;
$dbh->do(<<SQL) or die 'Migration failed';
UPDATE result_entries
SET module = 'Backend'
WHERE upper(module) = 'BACKEND_TEST_AGENT';
SQL
}

View File

@@ -0,0 +1,253 @@
use strict;
use warnings;
use Try::Tiny;
use Zonemaster::Backend::Config;
my $config = Zonemaster::Backend::Config->load_config();
my %patch = (
mysql => \&patch_db_mysql,
postgresql => \&patch_db_postgresql,
sqlite => \&patch_db_sqlite,
);
my $db_engine = $config->DB_engine;
if ( $db_engine =~ /^(MySQL|PostgreSQL|SQLite)$/ ) {
$patch{ lc $db_engine }();
}
else {
die "Unknown database engine configured: $db_engine\n";
}
sub patch_db_mysql {
use Zonemaster::Backend::DB::MySQL;
my $db = Zonemaster::Backend::DB::MySQL->from_config( $config );
my $dbh = $db->dbh;
# add table constraints
$dbh->do( 'ALTER TABLE users ADD CONSTRAINT UNIQUE (username)' );
$dbh->do( 'ALTER TABLE test_results ADD CONSTRAINT UNIQUE (hash_id)' );
# update columns names, data type and default value
$dbh->do( 'ALTER TABLE test_results MODIFY COLUMN id BIGINT AUTO_INCREMENT' );
$dbh->do( 'ALTER TABLE test_results CHANGE COLUMN creation_time created_at DATETIME NOT NULL' );
$dbh->do( 'ALTER TABLE test_results CHANGE COLUMN test_start_time started_at DATETIME DEFAULT NULL' );
$dbh->do( 'ALTER TABLE test_results CHANGE COLUMN test_end_time ended_at DATETIME DEFAULT NULL' );
$dbh->do( 'ALTER TABLE batch_jobs CHANGE COLUMN creation_time created_at DATETIME NOT NULL' );
$dbh->{AutoCommit} = 0;
try {
# normalize "domain" column
$dbh->do(
q[
UPDATE test_results
SET domain = LOWER(domain)
WHERE CAST(domain AS BINARY) RLIKE '[A-Z]'
]
);
$dbh->do(
q[
UPDATE test_results
SET domain = '.'
WHERE domain = '..' OR domain = '...' OR domain = '....'
]
);
$dbh->do(
q[
UPDATE test_results
SET domain = TRIM( TRAILING '.' FROM domain )
WHERE domain != '.' AND domain LIKE '%.'
]
);
$dbh->commit();
} catch {
print( "Could not upgrade database: " . $_ );
eval { $dbh->rollback() };
};
}
sub patch_db_postgresql {
use Zonemaster::Backend::DB::PostgreSQL;
my $db = Zonemaster::Backend::DB::PostgreSQL->from_config( $config );
my $dbh = $db->dbh;
$dbh->{AutoCommit} = 0;
try {
# update sequence data type to BIGINT
$dbh->do( 'ALTER SEQUENCE test_results_id_seq AS BIGINT' );
$dbh->do( 'ALTER TABLE test_results ALTER COLUMN id SET DATA TYPE BIGINT' );
# remove default value for "creation_time"
$dbh->do( 'ALTER TABLE test_results ALTER COLUMN creation_time DROP DEFAULT' );
$dbh->do( 'ALTER TABLE batch_jobs ALTER COLUMN creation_time DROP DEFAULT' );
# rename columns
$dbh->do( 'ALTER TABLE test_results RENAME COLUMN creation_time TO created_at' );
$dbh->do( 'ALTER TABLE test_results RENAME COLUMN test_start_time TO started_at' );
$dbh->do( 'ALTER TABLE test_results RENAME COLUMN test_end_time TO ended_at' );
$dbh->do( 'ALTER TABLE batch_jobs RENAME COLUMN creation_time TO created_at' );
# add table constraints
$dbh->do( 'ALTER TABLE test_results ADD UNIQUE (hash_id)' );
$dbh->do( 'ALTER TABLE users ADD UNIQUE (username)' );
# normalize "domain" column
$dbh->do(
q[
UPDATE test_results
SET domain = LOWER(domain)
WHERE domain != LOWER(domain)
]
);
$dbh->do(
q[
UPDATE test_results
SET domain = '.'
WHERE domain = '..' OR domain = '...' OR domain = '....'
]
);
$dbh->do(
q[
UPDATE test_results
SET domain = RTRIM(domain, '.')
WHERE domain != '.' AND domain LIKE '%.'
]
);
$dbh->commit();
} catch {
print( "Could not upgrade database: " . $_ );
eval { $dbh->rollback() };
};
}
sub patch_db_sqlite {
use Zonemaster::Backend::DB::SQLite;
my $db = Zonemaster::Backend::DB::SQLite->from_config( $config );
my $dbh = $db->dbh;
$dbh->{AutoCommit} = 0;
# since we change the default value for a column, the whole table needs to
# be recreated
# 1. rename the table to "<table>_old"
# 2. recreate a clean table schema
# 3. populate it with the values from "<table>_old"
# 4. remove "<table>_old" and indexes
# 5. recreate the indexes
try {
$dbh->do('ALTER TABLE test_results RENAME TO test_results_old');
$dbh->do('ALTER TABLE batch_jobs RENAME TO batch_jobs_old');
$dbh->do('ALTER TABLE users RENAME TO users_old');
# create the tables
$db->create_schema();
# populate the tables
$dbh->do(
q[
INSERT INTO test_results
(
id,
hash_id,
domain,
batch_id,
created_at,
started_at,
ended_at,
priority,
queue,
progress,
fingerprint,
params,
results,
undelegated
)
SELECT
id,
hash_id,
lower(domain),
batch_id,
creation_time,
test_start_time,
test_end_time,
priority,
queue,
progress,
fingerprint,
params,
results,
undelegated
FROM test_results_old
]
);
$dbh->do(
q[
UPDATE test_results
SET domain = '.'
WHERE domain = '..' OR domain = '...' OR domain = '....'
]
);
$dbh->do(
q[
UPDATE test_results
SET domain = RTRIM(domain, '.')
WHERE domain != '.' AND domain LIKE '%.'
]
);
$dbh->do('
INSERT INTO batch_jobs
(
id,
username,
created_at
)
SELECT
id,
username,
creation_time
FROM batch_jobs_old
');
$dbh->do('
INSERT INTO users
(
id,
username,
api_key
)
SELECT
id,
username,
api_key
FROM users_old
');
# delete old tables
$dbh->do('DROP TABLE test_results_old');
$dbh->do('DROP TABLE batch_jobs_old');
$dbh->do('DROP TABLE users_old');
# recreate indexes
$db->create_schema();
$dbh->commit();
} catch {
print( "Error while upgrading database: " . $_ );
eval { $dbh->rollback() };
};
}

View File

@@ -0,0 +1,40 @@
use strict;
use warnings;
use DBI qw(:utils);
use Zonemaster::Backend::Config;
use Zonemaster::Backend::DB::MySQL;
my $config = Zonemaster::Backend::Config->load_config();
if ( $config->DB_engine ne 'MySQL' ) {
die "The configuration file does not contain the MySQL backend";
}
my $dbh = Zonemaster::Backend::DB::MySQL->from_config( $config )->dbh;
sub patch_db {
####################################################################
# TEST RESULTS
####################################################################
$dbh->do( 'ALTER TABLE test_results ADD COLUMN hash_id VARCHAR(16) NULL' );
$dbh->do( 'UPDATE test_results SET hash_id = (SELECT SUBSTRING(MD5(CONCAT(RAND(), UUID())) from 1 for 16))' );
$dbh->do( 'ALTER TABLE test_results MODIFY hash_id VARCHAR(16) DEFAULT NULL NOT NULL' );
$dbh->do(
'CREATE TRIGGER before_insert_test_results
BEFORE INSERT ON test_results
FOR EACH ROW
BEGIN
IF new.hash_id IS NULL OR new.hash_id=\'\'
THEN
SET new.hash_id = SUBSTRING(MD5(CONCAT(RAND(), UUID())) from 1 for 16);
END IF;
END;
'
);
}
patch_db();

View File

@@ -0,0 +1,22 @@
use strict;
use warnings;
use DBI qw(:utils);
use Zonemaster::Backend::Config;
use Zonemaster::Backend::DB::MySQL;
my $config = Zonemaster::Backend::Config->load_config();
if ( $config->DB_engine ne 'MySQL' ) {
die "The configuration file does not contain the MySQL backend";
}
my $dbh = Zonemaster::Backend::DB::MySQL->from_config( $config )->dbh;
sub patch_db {
####################################################################
# TEST RESULTS
####################################################################
$dbh->do( 'ALTER TABLE test_results ADD COLUMN nb_retries INTEGER NOT NULL DEFAULT 0' );
}
patch_db();

View File

@@ -0,0 +1,22 @@
use strict;
use warnings;
use DBI qw(:utils);
use Zonemaster::Backend::Config;
use Zonemaster::Backend::DB::MySQL;
my $config = Zonemaster::Backend::Config->load_config();
if ( $config->DB_engine ne 'MySQL' ) {
die "The configuration file does not contain the MySQL backend";
}
my $dbh = Zonemaster::Backend::DB::MySQL->from_config( $config )->dbh;
sub patch_db {
############################################################################
# Convert column "results" to MEDIUMBLOB so that it can hold larger results
############################################################################
$dbh->do( 'ALTER TABLE test_results MODIFY results mediumblob' );
}
patch_db();

View File

@@ -0,0 +1,76 @@
use strict;
use warnings;
use JSON::PP;
use DBI qw(:utils);
use Zonemaster::Backend::Config;
use Zonemaster::Backend::DB::MySQL;
my $config = Zonemaster::Backend::Config->load_config();
if ( $config->DB_engine ne 'MySQL' ) {
die "The configuration file does not contain the MySQL backend";
}
my $db = Zonemaster::Backend::DB::MySQL->from_config( $config );
my $dbh = $db->dbh;
sub patch_db {
# Remove the trigger
$dbh->do( 'DROP TRIGGER IF EXISTS before_insert_test_results' );
# Set the "hash_id" field to NOT NULL
eval {
$dbh->do( 'ALTER TABLE test_results MODIFY COLUMN hash_id VARCHAR(16) NOT NULL' );
};
print( "Error while changing DB schema: " . $@ ) if ($@);
# Rename column "params_deterministic_hash" into "fingerprint"
# Since MariaDB 10.5.2 (2020-03-26) <https://mariadb.com/kb/en/mariadb-1052-release-notes/>
# ALTER TABLE t1 RENAME COLUMN old_col TO new_col;
# Before that we need to use CHANGE COLUMN <https://mariadb.com/kb/en/alter-table/#change-column>
eval {
$dbh->do('ALTER TABLE test_results CHANGE COLUMN params_deterministic_hash fingerprint CHARACTER VARYING(32)');
};
print( "Error while changing DB schema: " . $@ ) if ($@);
# Update index
eval {
# retrieve all indexes by key name
my $indexes = $dbh->selectall_hashref( 'SHOW INDEXES FROM test_results', 'Key_name' );
if ( exists($indexes->{test_results__params_deterministic_hash}) ) {
$dbh->do( "DROP INDEX test_results__params_deterministic_hash ON test_results" );
}
$dbh->do( "CREATE INDEX test_results__fingerprint ON test_results (fingerprint)" );
};
print( "Error while updating the index: " . $@ ) if ($@);
# Update the "undelegated" column
my $sth1 = $dbh->prepare('SELECT id, params from test_results', undef);
$sth1->execute;
while ( my $row = $sth1->fetchrow_hashref ) {
my $id = $row->{id};
my $raw_params = decode_json($row->{params});
my $ds_info_values = scalar grep !/^$/, map { values %$_ } @{$raw_params->{ds_info}};
my $nameservers_values = scalar grep !/^$/, map { values %$_ } @{$raw_params->{nameservers}};
my $undelegated = $ds_info_values > 0 || $nameservers_values > 0 || 0;
$dbh->do('UPDATE test_results SET undelegated = ? where id = ?', undef, $undelegated, $id);
}
# remove the "user_info" column from the "users" table
# the IF EXISTS clause is available with MariaDB but not MySQL
eval {
$dbh->do( "ALTER TABLE users DROP COLUMN user_info" );
};
print( "Error while dropping the column: " . $@ ) if ($@);
# remove the "nb_retries" column from the "test_results" table
eval {
$dbh->do( "ALTER TABLE test_results DROP COLUMN nb_retries" );
};
print( "Error while dropping the column: " . $@ ) if ($@);
}
patch_db();

View File

@@ -0,0 +1,23 @@
use strict;
use warnings;
use DBI qw(:utils);
use Zonemaster::Backend::Config;
use Zonemaster::Backend::DB::MySQL;
my $config = Zonemaster::Backend::Config->load_config();
if ( $config->DB_engine ne 'MySQL' ) {
die "The configuration file does not contain the MySQL backend";
}
my $dbh = Zonemaster::Backend::DB::MySQL->from_config( $config )->dbh;
sub patch_db {
####################################################################
# TEST RESULTS
####################################################################
$dbh->do( 'ALTER TABLE test_results ADD COLUMN hash_id VARCHAR(16) DEFAULT substring(md5(random()::text || clock_timestamp()::text) from 1 for 16) NOT NULL' );
}
patch_db();

View File

@@ -0,0 +1,23 @@
use strict;
use warnings;
use DBI qw(:utils);
use Zonemaster::Backend::Config;
use Zonemaster::Backend::DB::PostgreSQL;
my $config = Zonemaster::Backend::Config->load_config();
if ( $config->DB_engine ne 'PostgreSQL' ) {
die "The configuration file does not contain the PostgreSQL backend";
}
my $dbh = Zonemaster::Backend::DB::PostgreSQL->from_config( $config )->dbh;
sub patch_db {
####################################################################
# TEST RESULTS
####################################################################
$dbh->do( 'ALTER TABLE test_results ADD COLUMN nb_retries INTEGER NOT NULL DEFAULT 0' );
}
patch_db();

View File

@@ -0,0 +1,109 @@
use strict;
use warnings;
use JSON::PP;
use Encode;
use DBI qw(:utils);
use Zonemaster::Backend::Config;
use Zonemaster::Backend::DB::PostgreSQL;
my $config = Zonemaster::Backend::Config->load_config();
if ( $config->DB_engine ne 'PostgreSQL' ) {
die "The configuration file does not contain the PostgreSQL backend";
}
my $db = Zonemaster::Backend::DB::PostgreSQL->from_config( $config );
my $dbh = $db->dbh;
sub patch_db {
# Drop default value for the "hash_id" field
$dbh->do( 'ALTER TABLE test_results ALTER COLUMN hash_id DROP DEFAULT' );
# Rename column "params_deterministic_hash" into "fingerprint"
eval {
$dbh->do( 'ALTER TABLE test_results RENAME COLUMN params_deterministic_hash TO fingerprint' );
};
print( "Error while changing DB schema: " . $@ ) if ($@);
# Update index
eval {
$dbh->do( "DROP INDEX IF EXISTS test_results__params_deterministic_hash" );
$dbh->do( "CREATE INDEX test_results__fingerprint ON test_results (fingerprint)" );
};
print( "Error while updating the index: " . $@ ) if ($@);
# test_start_time and test_end_time default to NULL
eval {
$dbh->do('ALTER TABLE test_results ALTER COLUMN test_start_time SET DEFAULT NULL');
$dbh->do('ALTER TABLE test_results ALTER COLUMN test_end_time SET DEFAULT NULL');
};
print( "Error while changing DB schema: " . $@ ) if ($@);
# Add missing "domain" and "undelegated" columns
eval {
$dbh->do( "ALTER TABLE test_results ADD COLUMN domain VARCHAR(255) NOT NULL DEFAULT ''" );
$dbh->do( 'ALTER TABLE test_results ADD COLUMN undelegated integer NOT NULL DEFAULT 0' );
};
print( "Error while changing DB schema: " . $@ ) if ($@);
# Update index
eval {
$dbh->do( "DROP INDEX IF EXISTS test_results__domain_undelegated" );
$dbh->do( "CREATE INDEX test_results__domain_undelegated ON test_results (domain, undelegated)" );
};
print( "Error while updating the index: " . $@ ) if ($@);
# New index
eval {
$dbh->do( 'CREATE INDEX IF NOT EXISTS test_results__progress_priority_id ON test_results (progress, priority DESC, id) WHERE (progress = 0)' );
};
print( "Error while creating the index: " . $@ ) if ($@);
# Update the "domain" column
$dbh->do( "UPDATE test_results SET domain = (params->>'domain')" );
# remove default value to "domain" column
$dbh->do( "ALTER TABLE test_results ALTER COLUMN domain DROP DEFAULT" );
# Update the "undelegated" column
my $sth1 = $dbh->prepare('SELECT id, params from test_results', undef);
$sth1->execute;
while ( my $row = $sth1->fetchrow_hashref ) {
my $id = $row->{id};
my $raw_params;
if (utf8::is_utf8($row->{params}) ) {
$raw_params = decode_json( encode_utf8 ( $row->{params} ) );
} else {
$raw_params = decode_json( $row->{params} );
}
my $ds_info_values = scalar grep !/^$/, map { values %$_ } @{$raw_params->{ds_info}};
my $nameservers_values = scalar grep !/^$/, map { values %$_ } @{$raw_params->{nameservers}};
my $undelegated = $ds_info_values > 0 || $nameservers_values > 0 || 0;
$dbh->do('UPDATE test_results SET undelegated = ? where id = ?', undef, $undelegated, $id);
}
# add "username" and "api_key" columns to the "users" table
eval {
$dbh->do( 'ALTER TABLE users ADD COLUMN username VARCHAR(128)' );
$dbh->do( 'ALTER TABLE users ADD COLUMN api_key VARCHAR(512)' );
};
print( "Error while changing DB schema: " . $@ ) if ($@);
# update the columns
eval {
$dbh->do( "UPDATE users SET username = (user_info->>'username'), api_key = (user_info->>'api_key')" );
};
print( "Error while updating the users table: " . $@ ) if ($@);
# remove the "user_info" column from the "users" table
$dbh->do( "ALTER TABLE users DROP COLUMN IF EXISTS user_info" );
# remove the "nb_retries" column from the "test_results" table
$dbh->do( "ALTER TABLE test_results DROP COLUMN IF EXISTS nb_retries" );
}
patch_db();

View File

@@ -0,0 +1,95 @@
use strict;
use warnings;
use JSON::PP;
use DBI qw(:utils);
use Zonemaster::Backend::Config;
use Zonemaster::Backend::DB::SQLite;
my $config = Zonemaster::Backend::Config->load_config();
if ( $config->DB_engine ne 'SQLite' ) {
die "The configuration file does not contain the SQLite backend";
}
my $db = Zonemaster::Backend::DB::SQLite->from_config( $config );
my $dbh = $db->dbh;
sub patch_db {
# since we change the default value for a column, the whole table needs to
# be recreated
# 1. rename the "test_results" table to "test_results_old"
# 2. create the new "test_results" table
# 3. populate it with the values from "test_results_old"
# 4. remove old table and indexes
# 5. recreate the indexes
eval {
$dbh->do('ALTER TABLE test_results RENAME TO test_results_old');
# create the table
$db->create_schema();
# populate it
# - nb_retries is omitted as we remove this column
# - params_deterministic_hash is renamed to fingerprint
$dbh->do('
INSERT INTO test_results
SELECT id,
hash_id,
domain,
batch_id,
creation_time,
test_start_time,
test_end_time,
priority,
queue,
progress,
params_deterministic_hash,
params,
results,
undelegated
FROM test_results_old
');
$dbh->do('DROP TABLE test_results_old');
# recreate indexes
$db->create_schema();
};
print( "Error while updating the 'test_results' table schema: " . $@ ) if ($@);
# Update the "undelegated" column
my $sth1 = $dbh->prepare('SELECT id, params from test_results', undef);
$sth1->execute;
while ( my $row = $sth1->fetchrow_hashref ) {
my $id = $row->{id};
my $raw_params = decode_json($row->{params});
my $ds_info_values = scalar grep !/^$/, map { values %$_ } @{$raw_params->{ds_info}};
my $nameservers_values = scalar grep !/^$/, map { values %$_ } @{$raw_params->{nameservers}};
my $undelegated = $ds_info_values > 0 || $nameservers_values > 0 || 0;
$dbh->do('UPDATE test_results SET undelegated = ? where id = ?', undef, $undelegated, $id);
}
# in order to properly drop a column, the whole table needs to be recreated
# 1. rename the "users" table to "users_old"
# 2. create the new "users" table
# 3. populate it with the values from "users_old"
# 4. remove old table
eval {
$dbh->do('ALTER TABLE users RENAME TO users_old');
# create the table
$db->create_schema();
# populate it
$dbh->do('INSERT INTO users SELECT id, username, api_key FROM users_old');
$dbh->do('DROP TABLE users_old');
};
print( "Error while updating the 'users' table schema: " . $@ ) if ($@);
}
patch_db();

View File

@@ -0,0 +1,85 @@
msgid ""
msgstr ""
"Project-Id-Version: 1.0.0\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2024-09-18 08:43+0200\n"
"PO-Revision-Date: 2024-09-18 10:05+0200\n"
"Last-Translator: milijan@arnes.si\n"
"Language-Team: Zonemaster project\n"
"Language: sl\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"X-Generator: Poedit 3.5\n"
#: ../lib/Zonemaster/Backend/RPCAPI.pm:858
msgid "Invalid method parameter(s)."
msgstr "Nepravilni parametri."
#: ../lib/Zonemaster/Backend/RPCAPI.pm:888
msgid "Missing property"
msgstr "Manjkajoče polje"
#: ../lib/Zonemaster/Backend/TestAgent.pm:213
msgid ""
"Warning: Zonemaster::LDNS not compiled with IDN support, cannot handle non-"
"ASCII names correctly."
msgstr ""
"Opozorilo: Zonemaster::LDNS ne podpira IDN, ni mogoče obdelati ne-ASCII "
"imena."
#. BACKEND_TEST_AGENT:TEST_DIED
#: ../lib/Zonemaster/Backend/Translator.pm:23
msgid "An error occured and Zonemaster could not start or finish the test."
msgstr "Zgodila se je napaka, Zonemaster ne more začeti ali končati testa."
#. BACKEND_TEST_AGENT:UNABLE_TO_FINISH_TEST
#: ../lib/Zonemaster/Backend/Translator.pm:27
#, perl-brace-format
msgid ""
"The test took too long to run (the current limit is {max_execution_time} "
"seconds). Maybe there are too many name servers or the name servers are "
"either unreachable or not responsive enough."
msgstr ""
"Test traja predolgo (trenutna meja je {max_execution_time} sekund). Mogoče "
"je preveč strežnikov za preveriti, ali so neodzivni ali pa počasni."
#: ../lib/Zonemaster/Backend/Validator.pm:162
msgid "Invalid digest format"
msgstr "Nepravilen digest format"
#: ../lib/Zonemaster/Backend/Validator.pm:167
msgid "Algorithm must be a positive integer"
msgstr "Algoritem mora biti pozitivno število"
#: ../lib/Zonemaster/Backend/Validator.pm:172
msgid "Digest type must be a positive integer"
msgstr "Tip izvlečka za DS mora biti pozitivno število"
#: ../lib/Zonemaster/Backend/Validator.pm:177
msgid "Keytag must be a positive integer"
msgstr "Oznaka za ključ mora biti pozitivno število"
#: ../lib/Zonemaster/Backend/Validator.pm:282
msgid "Domain name required"
msgstr "Domena je obvezna"
#: ../lib/Zonemaster/Backend/Validator.pm:314
msgid "Invalid language tag format"
msgstr "Nepravilen format zastavice za jezik"
#: ../lib/Zonemaster/Backend/Validator.pm:317
msgid "Unkown language string"
msgstr "Neznan jezik"
#: ../lib/Zonemaster/Backend/Validator.pm:332
msgid "Invalid IP address"
msgstr "Neveljaven IP naslov"
#: ../lib/Zonemaster/Backend/Validator.pm:356
msgid "Invalid profile format"
msgstr "Neveljaven format profila"
#: ../lib/Zonemaster/Backend/Validator.pm:360
msgid "Unknown profile"
msgstr "Neznan profil"

View File

@@ -0,0 +1,92 @@
msgid ""
msgstr ""
"Project-Id-Version: 1.0.0\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2023-05-21 21:29+0000\n"
"PO-Revision-Date: 2023-05-21 21:29+0000\n"
"Last-Translator: mats.dufberg@iis.se\n"
"Language-Team: Zonemaster project\n"
"Language: sv\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
msgid "Invalid method parameter(s)."
msgstr "Ogiltig metodparameter."
msgid "Missing property"
msgstr "Attribut saknas"
msgid ""
"Warning: Zonemaster::LDNS not compiled with IDN support, cannot handle non-"
"ASCII names correctly."
msgstr ""
"Varning: Zonemaster::LDNS är inte kompilerad med IDNA-stöd, så enbart ASCII-"
"namn kan hanteras."
#. BACKEND_TEST_AGENT:TEST_DIED
msgid "An error occured and Zonemaster could not start or finish the test."
msgstr ""
"Ett fel har inträffat så att Zonemaster inte kunde starta eller slutföra "
"testet."
#. BACKEND_TEST_AGENT:UNABLE_TO_FINISH_TEST
#, perl-brace-format
msgid ""
"The test took too long to run (the current limit is {max_execution_time} "
"seconds). Maybe there are too many name servers or the name servers are "
"either unreachable or not responsive enough."
msgstr ""
"Det tog för lång tid att köra testet (övre tidsgränsen är f.n. "
"{max_execution_time} sekunder). Kanske har domänen för många namnservrar "
"eller så är namnservrarna oåtkomliga eller så tar namnservrarna för lång på "
"att svara."
msgid "Invalid digest format"
msgstr "Ogiltigt format på digest-data"
msgid "Algorithm must be a positive integer"
msgstr "Algoritm måste vara ett positivt heltal"
msgid "Digest type must be a positive integer"
msgstr "Digest-typ måste vara ett positivt heltal"
msgid "Keytag must be a positive integer"
msgstr "Keytag måste vara ett positivt heltal"
msgid "Domain name required"
msgstr "Domännamn är obligatoriskt"
msgid "The domain name is IDNA invalid"
msgstr "Domännamnet är ogiltigt enligt IDN-standarden"
msgid ""
"The domain name contains non-ascii characters and IDNA support is not "
"installed"
msgstr ""
"Domännamnet innehåller icke-ASCII-tecken, men stöd för IDN är inte "
"installerat"
msgid "The domain name character(s) are not supported"
msgstr "Domännamnstecken stöds inte"
msgid "The domain name contains consecutive dots"
msgstr "Domännamnet innehåller flera punkter i följd"
msgid "The domain name or label is too long"
msgstr "Domännamnet eller en domännamnsdel är för långt"
msgid "Invalid language tag format"
msgstr "Ogiltigt format på språkkoden"
msgid "Unkown language string"
msgstr "Okänd språksträng"
msgid "Invalid IP address"
msgstr "Ogiltig IP-adress"
msgid "Invalid profile format"
msgstr "Ogiltigt profilformat"
msgid "Unknown profile"
msgstr "Okänd profil"

View File

@@ -0,0 +1,2 @@
#Type Path Mode UID GID Age Argument
d /run/zonemaster 0755 zonemaster zonemaster - -

View File

@@ -0,0 +1,9 @@
#!/bin/sh
if [ -z "$1" ] ; then
echo "error: No PO file specified." >&2
exit 2
fi
po_file="$1" ; shift
make update-po POFILES="$po_file"

View File

@@ -0,0 +1,73 @@
#!/bin/sh
#
### BEGIN INIT INFO
# Provides: zm-rpcapi
# Required-Start: $network $local_fs
# Required-Stop: $network $local_fs
# Should-Start: mysql postgresql
# Should-Stop: mysql postgresql
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: A JSON-RPC frontend for Zonemaster Backend
# Description: zm-rpcapi lets you add new tests and check for results in
# the the Zonemaster Backend database
### END INIT INFO
BINDIR=${ZM_BACKEND_BINDIR:-/usr/local/bin}
LOGFILE=${ZM_BACKEND_LOGFILE:-/var/log/zonemaster/zm-rpcapi.log}
PIDFILE=${ZM_BACKEND_PIDFILE:-/var/run/zonemaster/zm-rpcapi.pid}
LISTENIP=${ZM_BACKEND_LISTENIP:-127.0.0.1}
LISTENPORT=${ZM_BACKEND_LISTENPORT:-5000}
USER=${ZM_BACKEND_USER:-zonemaster}
GROUP=${ZM_BACKEND_GROUP:-zonemaster}
STARMAN=`PATH="$PATH:/usr/local/bin" /usr/bin/which starman`
#export ZM_BACKEND_RPCAPI_LOGLEVEL='warning' # Set this variable to override the default log level
. /lib/lsb/init-functions
start () {
$STARMAN --listen=$LISTENIP:$LISTENPORT --preload-app --user=$USER --group=$GROUP --pid=$PIDFILE --error-log=$LOGFILE --daemonize $BINDIR/zonemaster_backend_rpcapi.psgi || exit 1
}
stop () {
if [ -f $PIDFILE ]
then
kill `cat $PIDFILE`
fi
}
status () {
status="0"
pidofproc -p "$PIDFILE" starman >/dev/null || status="$?"
if [ "$status" = 0 ]; then
log_success_msg "zm-rpcapi is running"
return 0
elif [ "$status" = 4 ]; then
log_failure_msg "could not access PID file for zm-rpcapi"
return $status
else
log_failure_msg "zm-rpcapi is not running"
return $status
fi
}
case "$1" in
start)
start
;;
stop)
stop
;;
restart|force-reload)
stop
start
;;
status)
status
;;
*)
echo "usage: $0 [start|stop|restart|force-reload|status]"
exit 1
esac
exit 0

View File

@@ -0,0 +1,13 @@
[Unit]
Description=RPC server for Zonemaster Backend
After=network.target mariadb.service postgresql.service
Wants=mariadb.service postgresql.service
[Service]
Type=simple
ExecStart=/usr/local/bin/starman --listen=127.0.0.1:5000 --preload-app --user=zonemaster --group=zonemaster --pid=/run/zonemaster/zm-rpcapi.pid --error-log=/var/log/zonemaster/zm-rpcapi.log --daemonize /usr/local/bin/zonemaster_backend_rpcapi.psgi
KillSignal=SIGQUIT
PIDFile=/run/zonemaster/zm-rpcapi.pid
[Install]
WantedBy=multi-user.target

View File

@@ -0,0 +1,60 @@
#!/bin/sh
#
### BEGIN INIT INFO
# Provides: zm-testagent
# Required-Start: $network $local_fs
# Required-Stop: $network $local_fs
# Should-Start: mysql postgresql
# Should-Stop: mysql postgresql
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: An asynchronous execution backend for Zonemaster Backend
# Description: zm-testagent checks the Zonemaster Backend database for new
# tests, executes them and writes back progress and results.
### END INIT INFO
BINDIR=${ZM_BACKEND_BINDIR:-/usr/local/bin}
LOGFILE=${ZM_BACKEND_LOGFILE:-/var/log/zonemaster/zm-testagent.log}
OUTFILE=${ZM_BACKEND_OUTFILE:-/var/log/zonemaster/zm-testagent.out}
PIDFILE=${ZM_BACKEND_PIDFILE:-/var/run/zonemaster/zm-testagent.pid}
USER=${ZM_BACKEND_USER:-zonemaster}
GROUP=${ZM_BACKEND_GROUP:-zonemaster}
#ZM_BACKEND_TESTAGENT_LOGLEVEL='info' # Set this variable to override the default log level
testagent_args="--logfile=$LOGFILE --outfile=$OUTFILE --pidfile=$PIDFILE --user=$USER --group=$GROUP"
if [ -n "$ZM_BACKEND_TESTAGENT_LOGLEVEL" ] ; then
testagent_args="$testagent_args --loglevel=$ZM_BACKEND_TESTAGENT_LOGLEVEL"
fi
start () {
$BINDIR/zonemaster_backend_testagent $testagent_args start || exit 1
}
stop () {
$BINDIR/zonemaster_backend_testagent $testagent_args stop
}
status () {
$BINDIR/zonemaster_backend_testagent $testagent_args status
}
case "$1" in
start)
start
;;
stop)
stop
;;
restart|force-reload)
stop
start
;;
status)
status
;;
*)
echo "usage: $0 [start|stop|restart|status]"
exit 1
esac
exit 0

View File

@@ -0,0 +1,13 @@
[Unit]
Description=test agent for Zonemaster Backend
After=network.target mariadb.service postgresql.service
Wants=mariadb.service postgresql.service
[Service]
Type=simple
ExecStart=/usr/local/bin/zonemaster_backend_testagent --logfile=/var/log/zonemaster/zm-testagent.log --outfile=/var/log/zonemaster/zm-testagent.out --pidfile=/run/zonemaster/zm-testagent.pid --user=zonemaster --group=zonemaster start
ExecStop=/usr/local/bin/zonemaster_backend_testagent --logfile=/var/log/zonemaster/zm-testagent.log --outfile=/var/log/zonemaster/zm-testagent.out --pidfile=/run/zonemaster/zm-testagent.pid --user=zonemaster --group=zonemaster stop
PIDFile=/run/zonemaster/zm-testagent.pid
[Install]
WantedBy=multi-user.target

View File

@@ -0,0 +1,28 @@
#!/bin/sh
# PROVIDE: zm_rpcapi
# REQUIRE: NETWORKING mysql postgresql
# KEYWORD: shutdown
. /etc/rc.subr
name="zm_rpcapi"
rcvar="${name}_enable"
load_rc_config $name
: ${zm_rpcapi_enable="NO"}
: ${zm_rpcapi_user="zonemaster"}
: ${zm_rpcapi_group="zonemaster"}
: ${zm_rpcapi_pidfile="/var/run/zonemaster/${name}.pid"}
: ${zm_rpcapi_logfile="/var/log/zonemaster/${name}.log"}
: ${zm_rpcapi_listen="127.0.0.1:5000"}
export ZONEMASTER_BACKEND_CONFIG_FILE="/usr/local/etc/zonemaster/backend_config.ini"
#export ZM_BACKEND_RPCAPI_LOGLEVEL='warning' # Set this variable to override the default log level
command="/usr/local/bin/starman"
command_args="--listen=${zm_rpcapi_listen} --preload-app --user=${zm_rpcapi_user} --group=${zm_rpcapi_group} --pid=${zm_rpcapi_pidfile} --error-log=${zm_rpcapi_logfile} --daemonize /usr/local/bin/zonemaster_backend_rpcapi.psgi"
pidfile="${zm_rpcapi_pidfile}"
required_files="/usr/local/etc/zonemaster/backend_config.ini /usr/local/bin/zonemaster_backend_rpcapi.psgi"
run_rc_command "$1"

View File

@@ -0,0 +1,46 @@
#!/bin/sh
# PROVIDE: zm_testagent
# REQUIRE: NETWORKING mysql postgresql
# KEYWORD: shutdown
. /etc/rc.subr
name="zm_testagent"
rcvar="${name}_enable"
load_rc_config $name
: ${zm_testagent_enable="NO"}
: ${zm_testagent_user="zonemaster"}
: ${zm_testagent_group="zonemaster"}
: ${zm_testagent_pidfile="/var/run/zonemaster/${name}.pid"}
export ZONEMASTER_BACKEND_CONFIG_FILE="/usr/local/etc/zonemaster/backend_config.ini"
#ZM_BACKEND_TESTAGENT_LOGLEVEL='info' # Set this variable to override the default log level
# Make Perl available for service() when executed via env() in script
export PATH="$PATH:/usr/local/bin"
command="/usr/local/bin/zonemaster_backend_testagent"
command_args="--user=${zm_testagent_user} --group=${zm_testagent_group} --pidfile=${zm_testagent_pidfile}"
if [ -n "$ZM_BACKEND_TESTAGENT_LOGLEVEL" ] ; then
command_args="$testagent_args --loglevel=$ZM_BACKEND_TESTAGENT_LOGLEVEL"
fi
pidfile="${zm_testagent_pidfile}"
procname="/usr/local/bin/perl"
required_files="/usr/local/etc/zonemaster/backend_config.ini"
start_precmd="${name}_prestart"
stop_precmd="${name}_prestop"
zm_testagent_prestart()
{
rc_flags="${rc_flags} start"
}
zm_testagent_prestop()
{
rc_flags="${rc_flags} stop"
}
run_rc_command "$1"