From fe193f44f47bed15aa632685e1aba819320375f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Fri, 28 Jun 2019 20:10:23 +0200
Subject: [PATCH 01/92] Add experimental CLI tools for backend

---
 MANIFEST.SKIP |   4 +
 script/zmb    | 424 ++++++++++++++++++++++++++++++++++++++++++++++++++
 script/zmtest |  46 ++++++
 3 files changed, 474 insertions(+)
 create mode 100755 script/zmb
 create mode 100755 script/zmtest

diff --git a/MANIFEST.SKIP b/MANIFEST.SKIP
index d588b8c47..7321e5032 100644
--- a/MANIFEST.SKIP
+++ b/MANIFEST.SKIP
@@ -6,6 +6,10 @@ maint
 ^.*\.swp$
 ^MANIFEST\.SKIP$
 
+# Avoid experimental tools
+script/zmb
+script/zmtest
+
 #!start included /usr/share/perl/5.20/ExtUtils/MANIFEST.SKIP
 # Avoid version control files.
 \bRCS\b
diff --git a/script/zmb b/script/zmb
new file mode 100755
index 000000000..dda65c5b9
--- /dev/null
+++ b/script/zmb
@@ -0,0 +1,424 @@
+#!/usr/bin/env perl
+use strict;
+use warnings;
+use feature 'say';
+
+use Getopt::Long qw( GetOptionsFromArray :config require_order );
+
+use Pod::Usage;
+
+use JSON::PP;
+use LWP::UserAgent;
+
+=head1 NAME
+
+B<zmb> - Shell bindings for the Zonemaster::Backend RPC API
+
+Zmb is meant to be pronounced I<Zimba>.
+
+=head1 SYNOPSIS
+
+zmb COMMAND [OPTIONS]
+
+=head1 GLOBAL OPTIONS
+
+ --help        Show usage
+ --verbose     Show RPC query
+
+=cut
+
+sub main {
+    my @argv = @_;
+
+    my $opt_help;
+    my $opt_verbose;
+    GetOptionsFromArray(
+        \@argv,
+        'help'    => \$opt_help,
+        'verbose' => \$opt_verbose,
+    ) or pod2usage( 2 );
+    if ( !@argv ) {
+        pod2usage( -verbose => 99, -sections => ['SYNOPSIS', 'GLOBAL OPTIONS'], -exitval => 'NOEXIT' );
+        show_commands();
+        exit 1;
+    }
+    my $cmd = shift @argv;
+    pod2usage( 1 ) if !defined $cmd;
+    my $cmd_sub = \&{ "cmd_" . $cmd };
+    pod2usage( "'$cmd' is not a command" ) if !defined &$cmd_sub;
+    pod2usage( -verbose => 99, -sections => ["COMMANDS/$cmd"] ) if $opt_help;
+
+    my $json = &$cmd_sub( @argv );
+
+    if ( $json ) {
+        say $json if $opt_verbose;
+        my $request  = to_request( $json );
+        my $response = submit( $request );
+        say $response;
+    }
+}
+
+
+=head1 COMMANDS
+
+=head2 man
+
+Show the full manual page.
+
+    zmb man
+
+=cut
+
+sub cmd_man {
+    pod2usage( -verbose => 2 );
+}
+
+
+=head2 non_existing_method
+
+Call a non-existing RPC method.
+
+    zmb non_existing_method
+
+=cut
+
+sub cmd_non_existing_method {
+    return to_jsonrpc(
+        id     => 1,
+        method => 'non_existing_method',
+    );
+}
+
+
+=head2 version_info
+
+    zmb version_info
+
+=cut
+
+sub cmd_version_info {
+    return to_jsonrpc(
+        id     => 1,
+        method => 'version_info',
+    );
+}
+
+
+=head2 start_domain_test
+
+Call start_domain_test.
+
+    zmb start_domain_test [OPTIONS]
+
+Options:
+
+    --domain DOMAIN_NAME
+    --nameserver DOMAIN_NAME:IP_ADDRESS
+    --client-id CLIENT_ID
+    --client-version CLIENT_VERSION
+    --ds-info DS_INFO
+
+DS_INFO is a comma separated list of key-value pairs. The expected pairs are:
+
+    keytag=UNSIGNED_INTEGER
+    algorithm=UNSIGNED_INTEGER
+    digtype=UNSIGNED_INTEGER
+    digest=HEX_STRING
+
+=cut
+
+sub cmd_start_domain_test {
+    my @opts = @_;
+
+    my @opt_nameserver;
+    my $opt_domain;
+    my $opt_client_id;
+    my $opt_client_version;
+    my @opt_ds_info;
+    GetOptionsFromArray(
+        \@opts,
+        'domain|d=s'       => \$opt_domain,
+        'nameserver|n=s'   => \@opt_nameserver,
+        'client-id=s'      => \$opt_client_id,
+        'client-version=s' => \$opt_client_version,
+        'ds-info=s'        => \@opt_ds_info,
+    ) or pod2usage( 2 );
+
+    my %params = ( domain => $opt_domain, );
+
+    if ( $opt_client_id ) {
+        $params{client_id} = $opt_client_id,
+    }
+
+    if ( $opt_client_version ) {
+        $params{client_version} = $opt_client_version,
+    }
+
+    if ( @opt_ds_info ) {
+        my @info_objects;
+        for my $property_value_pairs ( @opt_ds_info ) {
+            my %info_object;
+            for my $pair ( split /,/, $property_value_pairs ) {
+                my ( $property, $value ) = split /=/, $pair;
+                if ( $property =~ /^(?:keytag|algorithm|digtype)$/ ) {
+                    $value = 0 + $value;
+                }
+                $info_object{$property} = $value;
+            }
+            push @info_objects, \%info_object;
+        }
+        $params{ds_info} = \@info_objects;
+    }
+
+    if ( @opt_nameserver ) {
+        my @nameserver_objects;
+        for my $domain_ip_pair ( @opt_nameserver ) {
+            my ( $domain, $ip ) = split /:/, $domain_ip_pair, 2;
+            push @nameserver_objects,
+              {
+                ns => $domain,
+                ip => $ip,
+              };
+        }
+        $params{nameservers} = \@nameserver_objects;
+    }
+
+    return to_jsonrpc(
+        id     => 1,
+        method => 'start_domain_test',
+        params => \%params,
+    );
+}
+
+
+=head2 test_progress
+
+ zmb test_progress [OPTIONS]
+
+ Options:
+   --testid TEST_ID
+
+=cut
+
+sub cmd_test_progress {
+    my @opts = @_;
+
+    my $opt_lang;
+    my $opt_testid;
+    GetOptionsFromArray( \@opts, 'testid|t=s' => \$opt_testid, )
+      or pod2usage( 2 );
+
+    return to_jsonrpc(
+        id     => 1,
+        method => 'test_progress',
+        params => {
+            test_id => $opt_testid,
+        },
+    );
+}
+
+
+=head2 get_test_results
+
+ zmb get_test_results [OPTIONS]
+
+ Options:
+   --testid TEST_ID
+   --lang LANGUAGE
+
+=cut
+
+sub cmd_get_test_results {
+    my @opts = @_;
+
+    my $opt_lang;
+    my $opt_testid;
+    GetOptionsFromArray(
+        \@opts,
+        'testid|t=s' => \$opt_testid,
+        'lang|l=s'   => \$opt_lang,
+    ) or pod2usage( 2 );
+
+    return to_jsonrpc(
+        id     => 1,
+        method => 'get_test_results',
+        params => {
+            id       => $opt_testid,
+            language => $opt_lang,
+        },
+    );
+}
+
+
+=head2 get_test_history
+
+ zmb get_test_history [OPTIONS]
+
+ Options:
+   --domain DOMAIN_NAME
+   --nameserver true|false|null
+   --offset COUNT
+   --limit COUNT
+
+=cut
+
+sub cmd_get_test_history {
+    my @opts = @_;
+    my $opt_nameserver;
+    my $opt_domain;
+    my $opt_offset;
+    my $opt_limit;
+
+    GetOptionsFromArray(
+        \@opts,
+        'domain|d=s'     => \$opt_domain,
+        'nameserver|n=s' => \$opt_nameserver,
+        'offset|o=i'     => \$opt_offset,
+        'limit|l=i'      => \$opt_limit,
+    ) or pod2usage( 2 );
+
+    my %params = (
+        frontend_params => {
+            domain => $opt_domain,
+        },
+    );
+
+    if ( $opt_nameserver ) {
+        $params{frontend_params}{nameservers} = json_tern( $opt_nameserver );
+    }
+
+    if ( defined $opt_offset ) {
+        $params{offset} = $opt_offset;
+    }
+
+    if ( defined $opt_limit ) {
+        $params{limit} = $opt_limit;
+    }
+
+    return to_jsonrpc(
+        id     => 1,
+        method => 'get_test_history',
+        params => \%params,
+    );
+}
+
+
+=head2 add_api_user
+
+ zmb add_api_user [OPTIONS]
+
+ Options:
+   --username USERNAME
+   --api-key API_KEY
+
+=cut
+
+sub cmd_add_api_user {
+    my @opts = @_;
+
+    my $opt_username;
+    my $opt_api_key;
+    GetOptionsFromArray(
+        \@opts,
+        'username|u=s' => \$opt_username,
+        'api-key|a=s'  => \$opt_api_key,
+    ) or pod2usage( 2 );
+
+    return to_jsonrpc(
+        id     => 1,
+        method => 'add_api_user',
+        params => {
+            username => $opt_username,
+            api_key  => $opt_api_key,
+        },
+    );
+}
+
+
+sub show_commands {
+    my %specials = (
+        man                 => 'Show the full manual page.',
+        non_existing_method => 'Call a non-existing RPC method.',
+    );
+    my @commands  = get_commands();
+    my $max_width = 0;
+    for my $command ( @commands ) {
+        $max_width = length $command if length $command > $max_width;
+    }
+    say "Commands:";
+    for my $command ( @commands ) {
+        if ( exists $specials{$command} ) {
+            printf "     %-*s     %s\n", $max_width, $command, $specials{$command};
+        }
+        else {
+            say "     ", $command;
+        }
+    }
+}
+
+
+sub get_commands {
+    no strict 'refs';
+
+    return sort
+      map { $_ =~ s/^cmd_//r }
+      grep { $_ =~ /^cmd_/ } grep { defined &{"main\::$_"} } keys %{"main\::"};
+}
+
+sub json_tern {
+    my $value = shift;
+    if ( $value eq 'true' ) {
+        return JSON::PP::true;
+    }
+    elsif ( $value eq 'false' ) {
+        return JSON::PP::false;
+    }
+    elsif ( $value eq 'null' ) {
+        return undef;
+    }
+    else {
+        die "unknown ternary value";
+    }
+}
+
+sub to_jsonrpc {
+    my %args   = @_;
+    my $id     = $args{id};
+    my $method = $args{method};
+
+    my $request = {
+        jsonrpc => "2.0",
+        method  => $method,
+        id      => $id,
+    };
+    if ( exists $args{params} ) {
+        $request->{params} = $args{params};
+    }
+    return encode_json( $request );
+}
+
+sub to_request {
+    my $json = shift;
+
+    my $req = HTTP::Request->new( POST => 'http://localhost:5000/' );
+    $req->content_type( 'application/json' );
+    $req->content( $json );
+
+    return $req;
+}
+
+sub submit {
+    my $req = shift;
+
+    my $ua  = LWP::UserAgent->new;
+    my $res = $ua->request( $req );
+
+    if ( $res->is_success ) {
+        return $res->decoded_content;
+    }
+    else {
+        die $res->status_line;
+    }
+}
+
+main( @ARGV );
diff --git a/script/zmtest b/script/zmtest
new file mode 100755
index 000000000..e30a9d29f
--- /dev/null
+++ b/script/zmtest
@@ -0,0 +1,46 @@
+#!/bin/sh
+
+bindir="`dirname $0`"
+
+ZMB="${bindir}/zmb"
+JQ="`which jq`"
+
+error () {
+    status="$1"
+    message="$2"
+    echo "error: ${message}" >&2
+    exit "${status}"
+}
+
+zmb () {
+    json="`"${ZMB}" "$@"`"
+    if echo "${json}" | "${JQ}" -e .error > /dev/null ; then
+        error 1 "method $1: ${json}"
+    else
+        echo "${json}"
+    fi
+}
+
+[ -n "${JQ}" ] || error 2 "Dependency not found: jq"
+
+[ $# -ge 1 ] || error 2 "No domain specified"
+domain="$1" ; shift
+
+# Start test
+testid="`zmb start_domain_test --domain "${domain}" | "${JQ}" -r .result`" || exit 1
+
+# Wait for test to finish
+while true
+do
+    progress="`zmb test_progress --testid "${testid}" | "${JQ}" -r .result`" || exit 1
+    printf "\r${progress}%% done" >&2
+    if [ "${progress}" -eq 100 ] ; then
+        echo >&2
+        break
+    fi
+    sleep 1
+done
+
+# Get test results
+zmb get_test_results --testid "${testid}" --lang en
+echo "testid:" ${testid} >&2

From 8c64fe4c1f96665358a4272bf3e1322bf6551365 Mon Sep 17 00:00:00 2001
From: Michal TOMA <toma@afnic.fr>
Date: Tue, 8 Oct 2019 18:58:18 +0200
Subject: [PATCH 02/92] Fixes issue #213 - Internal error details are leaked

---
 lib/Zonemaster/Backend/RPCAPI.pm | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/Zonemaster/Backend/RPCAPI.pm b/lib/Zonemaster/Backend/RPCAPI.pm
index e7433e090..0e62c46d7 100644
--- a/lib/Zonemaster/Backend/RPCAPI.pm
+++ b/lib/Zonemaster/Backend/RPCAPI.pm
@@ -41,7 +41,7 @@ sub new {
             die "$@ \n" if $@;
             $self->{db} = "$params->{db}"->new();
         };
-        die "$@ \n" if $@;
+        die "Failed to initialize the SQLite database backend module \n" if $@;
     }
     else {
         eval {
@@ -50,7 +50,7 @@ sub new {
             die "$@ \n" if $@;
             $self->{db} = $backend_module->new();
         };
-        die "$@ \n" if $@;
+        die "Failed to initialize the database backend module \n" if $@;
     }
 
     return ( $self );

From a855558e4fa41779bb57ac8ebea09299ef46a491 Mon Sep 17 00:00:00 2001
From: Michal TOMA <toma@afnic.fr>
Date: Wed, 9 Oct 2019 09:04:49 +0200
Subject: [PATCH 03/92] Added detailed info to log file

---
 lib/Zonemaster/Backend/RPCAPI.pm | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/lib/Zonemaster/Backend/RPCAPI.pm b/lib/Zonemaster/Backend/RPCAPI.pm
index 0e62c46d7..8adc1cc73 100644
--- a/lib/Zonemaster/Backend/RPCAPI.pm
+++ b/lib/Zonemaster/Backend/RPCAPI.pm
@@ -41,7 +41,10 @@ sub new {
             die "$@ \n" if $@;
             $self->{db} = "$params->{db}"->new();
         };
-        die "Failed to initialize the SQLite database backend module \n" if $@;
+        if $@ {
+            warn "Failed to initialize the [$params->{db}] database backend module: [$@] \n";
+            die "Failed to initialize the [$params->{db}] database backend module \n";
+        }
     }
     else {
         eval {
@@ -50,7 +53,9 @@ sub new {
             die "$@ \n" if $@;
             $self->{db} = $backend_module->new();
         };
-        die "Failed to initialize the database backend module \n" if $@;
+        if $@ {
+            warn "Failed to initialize the database backend module: [$@] \n";
+            die "Failed to initialize the database backend module \n" if $@;
     }
 
     return ( $self );
@@ -167,13 +172,13 @@ sub _check_domain {
     }
 
     if( $dn !~ m/^[\-a-zA-Z0-9\.\_]+$/ ) {
-	    return (
-		   $dn,
-		   {
-			   status  => 'nok',
-			   message => encode_entities( "The domain name contains unauthorized character(s)")
-                   }
-            );
+        return (
+            $dn,
+            {
+               status  => 'nok',
+               message => encode_entities( "The domain name contains unauthorized character(s)")
+            }
+        );
     }
 
     my @res;

From d61fe2cc4d1bf9c8b329f8640846795652918ae3 Mon Sep 17 00:00:00 2001
From: Michal TOMA <toma@afnic.fr>
Date: Wed, 9 Oct 2019 09:10:30 +0200
Subject: [PATCH 04/92] Fixed missing parentheses in if caluse

---
 lib/Zonemaster/Backend/RPCAPI.pm | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/Zonemaster/Backend/RPCAPI.pm b/lib/Zonemaster/Backend/RPCAPI.pm
index 8adc1cc73..4aa31e962 100644
--- a/lib/Zonemaster/Backend/RPCAPI.pm
+++ b/lib/Zonemaster/Backend/RPCAPI.pm
@@ -41,7 +41,7 @@ sub new {
             die "$@ \n" if $@;
             $self->{db} = "$params->{db}"->new();
         };
-        if $@ {
+        if ($@) {
             warn "Failed to initialize the [$params->{db}] database backend module: [$@] \n";
             die "Failed to initialize the [$params->{db}] database backend module \n";
         }
@@ -53,7 +53,7 @@ sub new {
             die "$@ \n" if $@;
             $self->{db} = $backend_module->new();
         };
-        if $@ {
+        if ($@) {
             warn "Failed to initialize the database backend module: [$@] \n";
             die "Failed to initialize the database backend module \n" if $@;
     }

From 87b9e7d1896b180ae7bb50f789d46fc42faa5694 Mon Sep 17 00:00:00 2001
From: Michal TOMA <toma@nic.fr>
Date: Mon, 28 Oct 2019 09:53:55 +0100
Subject: [PATCH 05/92] Added eval blocks to all API calls

---
 lib/Zonemaster/Backend/RPCAPI.pm | 514 ++++++++++++++++++-------------
 1 file changed, 302 insertions(+), 212 deletions(-)

diff --git a/lib/Zonemaster/Backend/RPCAPI.pm b/lib/Zonemaster/Backend/RPCAPI.pm
index 4aa31e962..ef664412c 100644
--- a/lib/Zonemaster/Backend/RPCAPI.pm
+++ b/lib/Zonemaster/Backend/RPCAPI.pm
@@ -56,19 +56,32 @@ sub new {
         if ($@) {
             warn "Failed to initialize the database backend module: [$@] \n";
             die "Failed to initialize the database backend module \n" if $@;
-    }
+        }
 
     return ( $self );
 }
 
+sub handle_exception {
+    my ( $method ) = @_;
+
+    warn "Unexpected error in the $method API call: [$@] \n";
+    die "Unexpected error in the $method API call \n" if $@;
+}
+
 $json_schemas{version_info} = joi->object->strict;
 sub version_info {
     my ( $self ) = @_;
 
     my %ver;
-    $ver{zonemaster_engine} = Zonemaster::Engine->VERSION;
-    $ver{zonemaster_backend} = Zonemaster::Backend->VERSION;
+    eval {
+        $ver{zonemaster_engine} = Zonemaster::Engine->VERSION;
+        $ver{zonemaster_backend} = Zonemaster::Backend->VERSION;
 
+    };
+    if ($@) {
+        handle_exception('version_info');
+    }
+    
     return \%ver;
 }
 
@@ -76,8 +89,15 @@ $json_schemas{profile_names} = joi->object->strict;
 sub profile_names {
     my ($self) = @_;
 
-    my @profiles = Zonemaster::Backend::Config->load_config()->ListPublicProfiles();
+    my @profiles;
+    eval {
+        @profiles = Zonemaster::Backend::Config->load_config()->ListPublicProfiles();
 
+    };
+    if ($@) {
+        handle_exception('profile_names');
+    }
+    
     return \@profiles;
 }
 
@@ -86,13 +106,19 @@ $json_schemas{get_host_by_name} = joi->object->strict->props(
 );
 sub get_host_by_name {
     my ( $self, $params ) = @_;
-    my $ns_name  = $params->{"hostname"};
+    my @adresses;
+    eval {
+        my $ns_name  = $params->{"hostname"};
 
-    my @adresses = map { {$ns_name => $_->short} } $recursor->get_addresses_for($ns_name);
-    @adresses = { $ns_name => '0.0.0.0' } if not @adresses;
+        @adresses = map { {$ns_name => $_->short} } $recursor->get_addresses_for($ns_name);
+        @adresses = { $ns_name => '0.0.0.0' } if not @adresses;
 
+    };
+    if ($@) {
+        handle_exception('get_host_by_name');
+    }
+    
     return \@adresses;
-
 }
 
 $json_schemas{get_data_from_parent_zone} = joi->object->strict->props(
@@ -102,39 +128,44 @@ sub get_data_from_parent_zone {
     my ( $self, $params ) = @_;
     my $domain = "";
 
-    if (ref \$params eq "SCALAR") {
-        $domain = $params;
-    } else {
-        $domain = $params->{"domain"};
-    }
-
     my %result;
+    eval {
+        if (ref \$params eq "SCALAR") {
+            $domain = $params;
+        } else {
+            $domain = $params->{"domain"};
+        }
 
-    my ( $dn, $dn_syntax ) = $self->_check_domain( $domain, 'Domain name' );
-    return $dn_syntax if ( $dn_syntax->{status} eq 'nok' );
+        my ( $dn, $dn_syntax ) = $self->_check_domain( $domain, 'Domain name' );
+        return $dn_syntax if ( $dn_syntax->{status} eq 'nok' );
 
-    my @ns_list;
-    my @ns_names;
+        my @ns_list;
+        my @ns_names;
 
-    my $zone = Zonemaster::Engine->zone( $domain );
-    push @ns_list, { ns => $_->name->string, ip => $_->address->short} for @{$zone->glue};
+        my $zone = Zonemaster::Engine->zone( $domain );
+        push @ns_list, { ns => $_->name->string, ip => $_->address->short} for @{$zone->glue};
 
-    my @ds_list;
+        my @ds_list;
 
-    $zone = Zonemaster::Engine->zone($domain);
-    my $ds_p = $zone->parent->query_one( $zone->name, 'DS', { dnssec => 1, cd => 1, recurse => 1 } );
-    if ($ds_p) {
-        my @ds = $ds_p->get_records( 'DS', 'answer' );
+        $zone = Zonemaster::Engine->zone($domain);
+        my $ds_p = $zone->parent->query_one( $zone->name, 'DS', { dnssec => 1, cd => 1, recurse => 1 } );
+        if ($ds_p) {
+            my @ds = $ds_p->get_records( 'DS', 'answer' );
 
-        foreach my $ds ( @ds ) {
-            next unless $ds->type eq 'DS';
-            push(@ds_list, { keytag => $ds->keytag, algorithm => $ds->algorithm, digtype => $ds->digtype, digest => $ds->hexdigest });
+            foreach my $ds ( @ds ) {
+                next unless $ds->type eq 'DS';
+                push(@ds_list, { keytag => $ds->keytag, algorithm => $ds->algorithm, digtype => $ds->digtype, digest => $ds->hexdigest });
+            }
         }
-    }
 
-    $result{ns_list} = \@ns_list;
-    $result{ds_list} = \@ds_list;
+        $result{ns_list} = \@ns_list;
+        $result{ds_list} = \@ds_list;
 
+    };
+    if ($@) {
+        handle_exception('get_data_from_parent_zone');
+    }
+    
     return \%result;
 }
 
@@ -142,49 +173,54 @@ sub get_data_from_parent_zone {
 sub _check_domain {
     my ( $self, $dn, $type ) = @_;
 
-    if ( !defined( $dn ) ) {
-        return ( $dn, { status => 'nok', message => encode_entities( "$type required" ) } );
-    }
+    eval {
+        if ( !defined( $dn ) ) {
+            return ( $dn, { status => 'nok', message => encode_entities( "$type required" ) } );
+        }
 
-    if ( $dn =~ m/[^[:ascii:]]+/ ) {
-        if ( Zonemaster::LDNS::has_idn() ) {
-            eval { $dn = Zonemaster::LDNS::to_idn( $dn ); };
-            if ( $@ ) {
+        if ( $dn =~ m/[^[:ascii:]]+/ ) {
+            if ( Zonemaster::LDNS::has_idn() ) {
+                eval { $dn = Zonemaster::LDNS::to_idn( $dn ); };
+                if ( $@ ) {
+                    return (
+                        $dn,
+                        {
+                            status  => 'nok',
+                            message => encode_entities( "The domain name is not a valid IDNA string and cannot be converted to an A-label" )
+                        }
+                    );
+                }
+            }
+            else {
                 return (
                     $dn,
                     {
-                        status  => 'nok',
-                        message => encode_entities( "The domain name is not a valid IDNA string and cannot be converted to an A-label" )
+                        status => 'nok',
+                        message =>
+                        encode_entities( "$type contains non-ascii characters and IDNA conversion is not installed" )
                     }
                 );
             }
         }
-        else {
+
+        if( $dn !~ m/^[\-a-zA-Z0-9\.\_]+$/ ) {
             return (
                 $dn,
                 {
-                    status => 'nok',
-                    message =>
-                      encode_entities( "$type contains non-ascii characters and IDNA conversion is not installed" )
+                status  => 'nok',
+                message => encode_entities( "The domain name contains unauthorized character(s)")
                 }
             );
         }
-    }
 
-    if( $dn !~ m/^[\-a-zA-Z0-9\.\_]+$/ ) {
-        return (
-            $dn,
-            {
-               status  => 'nok',
-               message => encode_entities( "The domain name contains unauthorized character(s)")
-            }
-        );
-    }
-
-    my @res;
-    @res = Zonemaster::Engine::Test::Basic->basic00($dn);
-    if (@res != 0) {
-        return ( $dn, { status => 'nok', message => encode_entities( "$type name or label outside allowed length" ) } );
+        my @res;
+        @res = Zonemaster::Engine::Test::Basic->basic00($dn);
+        if (@res != 0) {
+            return ( $dn, { status => 'nok', message => encode_entities( "$type name or label outside allowed length" ) } );
+        }
+    };
+    if ($@) {
+        handle_exception('_check_domain');
     }
 
     return ( $dn, { status => 'ok', message => 'Syntax ok' } );
@@ -193,95 +229,103 @@ sub _check_domain {
 sub validate_syntax {
     my ( $self, $syntax_input ) = @_;
 
-    my @allowed_params_keys = (
-        'domain',   'ipv4',      'ipv6', 'ds_info', 'nameservers', 'profile',
-        'client_id', 'client_version', 'config', 'priority', 'queue'
-    );
+    my $message;
+    eval {
+        my @allowed_params_keys = (
+            'domain',   'ipv4',      'ipv6', 'ds_info', 'nameservers', 'profile',
+            'client_id', 'client_version', 'config', 'priority', 'queue'
+        );
 
-    foreach my $k ( keys %$syntax_input ) {
-        return { status => 'nok', message => encode_entities( "Unknown option [$k] in parameters" ) }
-          unless ( grep { $_ eq $k } @allowed_params_keys );
-    }
+        foreach my $k ( keys %$syntax_input ) {
+            return { status => 'nok', message => encode_entities( "Unknown option [$k] in parameters" ) }
+            unless ( grep { $_ eq $k } @allowed_params_keys );
+        }
 
-    if ( ( defined $syntax_input->{nameservers} && @{ $syntax_input->{nameservers} } ) ) {
-        foreach my $ns_ip ( @{ $syntax_input->{nameservers} } ) {
-            foreach my $k ( keys %$ns_ip ) {
-                delete( $ns_ip->{$k} ) unless ( $k eq 'ns' || $k eq 'ip' );
+        if ( ( defined $syntax_input->{nameservers} && @{ $syntax_input->{nameservers} } ) ) {
+            foreach my $ns_ip ( @{ $syntax_input->{nameservers} } ) {
+                foreach my $k ( keys %$ns_ip ) {
+                    delete( $ns_ip->{$k} ) unless ( $k eq 'ns' || $k eq 'ip' );
+                }
             }
         }
-    }
 
-    if ( ( defined $syntax_input->{ds_info} && @{ $syntax_input->{ds_info} } ) ) {
-        foreach my $ds_digest ( @{ $syntax_input->{ds_info} } ) {
-            foreach my $k ( keys %$ds_digest ) {
-                delete( $ds_digest->{$k} ) unless ( $k eq 'algorithm' || $k eq 'digest' || $k eq 'digtype' || $k eq 'keytag' );
+        if ( ( defined $syntax_input->{ds_info} && @{ $syntax_input->{ds_info} } ) ) {
+            foreach my $ds_digest ( @{ $syntax_input->{ds_info} } ) {
+                foreach my $k ( keys %$ds_digest ) {
+                    delete( $ds_digest->{$k} ) unless ( $k eq 'algorithm' || $k eq 'digest' || $k eq 'digtype' || $k eq 'keytag' );
+                }
             }
         }
-    }
 
-    if ( defined $syntax_input->{ipv4} ) {
-        return { status => 'nok', message => encode_entities( "Invalid IPv4 transport option format" ) }
-          unless ( $syntax_input->{ipv4} eq JSON::PP::false
-            || $syntax_input->{ipv4} eq JSON::PP::true
-            || $syntax_input->{ipv4} eq '1'
-            || $syntax_input->{ipv4} eq '0' );
-    }
+        if ( defined $syntax_input->{ipv4} ) {
+            return { status => 'nok', message => encode_entities( "Invalid IPv4 transport option format" ) }
+            unless ( $syntax_input->{ipv4} eq JSON::PP::false
+                || $syntax_input->{ipv4} eq JSON::PP::true
+                || $syntax_input->{ipv4} eq '1'
+                || $syntax_input->{ipv4} eq '0' );
+        }
 
-    if ( defined $syntax_input->{ipv6} ) {
-        return { status => 'nok', message => encode_entities( "Invalid IPv6 transport option format" ) }
-          unless ( $syntax_input->{ipv6} eq JSON::PP::false
-            || $syntax_input->{ipv6} eq JSON::PP::true
-            || $syntax_input->{ipv6} eq '1'
-            || $syntax_input->{ipv6} eq '0' );
-    }
+        if ( defined $syntax_input->{ipv6} ) {
+            return { status => 'nok', message => encode_entities( "Invalid IPv6 transport option format" ) }
+            unless ( $syntax_input->{ipv6} eq JSON::PP::false
+                || $syntax_input->{ipv6} eq JSON::PP::true
+                || $syntax_input->{ipv6} eq '1'
+                || $syntax_input->{ipv6} eq '0' );
+        }
 
-    if ( defined $syntax_input->{profile} ) {
-        my @profiles = map lc, Zonemaster::Backend::Config->load_config()->ListPublicProfiles();
-        return { status => 'nok', message => encode_entities( "Invalid profile option format" ) }
-          unless ( grep { $_ eq lc $syntax_input->{profile} } @profiles );
-    }
+        if ( defined $syntax_input->{profile} ) {
+            my @profiles = map lc, Zonemaster::Backend::Config->load_config()->ListPublicProfiles();
+            return { status => 'nok', message => encode_entities( "Invalid profile option format" ) }
+            unless ( grep { $_ eq lc $syntax_input->{profile} } @profiles );
+        }
 
-    my ( $dn, $dn_syntax ) = $self->_check_domain( $syntax_input->{domain}, 'Domain name' );
+        my ( $dn, $dn_syntax ) = $self->_check_domain( $syntax_input->{domain}, 'Domain name' );
 
-    return $dn_syntax if ( $dn_syntax->{status} eq 'nok' );
+        return $dn_syntax if ( $dn_syntax->{status} eq 'nok' );
 
-    if ( defined $syntax_input->{nameservers} && @{ $syntax_input->{nameservers} } ) {
-        foreach my $ns_ip ( @{ $syntax_input->{nameservers} } ) {
-            my ( $ns, $ns_syntax ) = $self->_check_domain( $ns_ip->{ns}, "NS [$ns_ip->{ns}]" );
-            return $ns_syntax if ( $ns_syntax->{status} eq 'nok' );
-        }
+        if ( defined $syntax_input->{nameservers} && @{ $syntax_input->{nameservers} } ) {
+            foreach my $ns_ip ( @{ $syntax_input->{nameservers} } ) {
+                my ( $ns, $ns_syntax ) = $self->_check_domain( $ns_ip->{ns}, "NS [$ns_ip->{ns}]" );
+                return $ns_syntax if ( $ns_syntax->{status} eq 'nok' );
+            }
 
-        foreach my $ns_ip ( @{ $syntax_input->{nameservers} } ) {
-            return { status => 'nok', message => encode_entities( "Invalid IP address: [$ns_ip->{ip}]" ) }
-                unless( !$ns_ip->{ip} || $ns_ip->{ip} =~ /^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$/ || $ns_ip->{ip} =~ /^([0-9A-Fa-f]{1,4}:[0-9A-Fa-f:]{1,}(:[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})?)|([0-9A-Fa-f]{1,4}::[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})$/);
+            foreach my $ns_ip ( @{ $syntax_input->{nameservers} } ) {
+                return { status => 'nok', message => encode_entities( "Invalid IP address: [$ns_ip->{ip}]" ) }
+                    unless( !$ns_ip->{ip} || $ns_ip->{ip} =~ /^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$/ || $ns_ip->{ip} =~ /^([0-9A-Fa-f]{1,4}:[0-9A-Fa-f:]{1,}(:[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})?)|([0-9A-Fa-f]{1,4}::[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})$/);
 
-            return { status => 'nok', message => encode_entities( "Invalid IP address: [$ns_ip->{ip}]" ) }
-              unless ( !$ns_ip->{ip} || ip_is_ipv4( $ns_ip->{ip} ) || ip_is_ipv6( $ns_ip->{ip} ) );
-        }
+                return { status => 'nok', message => encode_entities( "Invalid IP address: [$ns_ip->{ip}]" ) }
+                unless ( !$ns_ip->{ip} || ip_is_ipv4( $ns_ip->{ip} ) || ip_is_ipv6( $ns_ip->{ip} ) );
+            }
 
-        foreach my $ds_digest ( @{ $syntax_input->{ds_info} } ) {
-            return {
-                status  => 'nok',
-                message => encode_entities( "Invalid algorithm type: [$ds_digest->{algorithm}]" )
-              }
-              if ( $ds_digest->{algorithm} =~ /\D/ );
-        }
+            foreach my $ds_digest ( @{ $syntax_input->{ds_info} } ) {
+                return {
+                    status  => 'nok',
+                    message => encode_entities( "Invalid algorithm type: [$ds_digest->{algorithm}]" )
+                }
+                if ( $ds_digest->{algorithm} =~ /\D/ );
+            }
 
-        foreach my $ds_digest ( @{ $syntax_input->{ds_info} } ) {
-            return {
-                status  => 'nok',
-                message => encode_entities( "Invalid digest format: [$ds_digest->{digest}]" )
+            foreach my $ds_digest ( @{ $syntax_input->{ds_info} } ) {
+                return {
+                    status  => 'nok',
+                    message => encode_entities( "Invalid digest format: [$ds_digest->{digest}]" )
+                }
+                if (
+                    ( length( $ds_digest->{digest} ) != 96 &&
+                        length( $ds_digest->{digest} ) != 64 &&
+                        length( $ds_digest->{digest} ) != 40 ) ||
+                        $ds_digest->{digest} =~ /[^A-Fa-f0-9]/
+                );
             }
-            if (
-                ( length( $ds_digest->{digest} ) != 96 &&
-                      length( $ds_digest->{digest} ) != 64 &&
-                      length( $ds_digest->{digest} ) != 40 ) ||
-                      $ds_digest->{digest} =~ /[^A-Fa-f0-9]/
-            );
         }
+        
+        $message = encode_entities( 'Syntax ok' );
+    };
+    if ($@) {
+        handle_exception('validate_syntax');
     }
 
-    return { status => 'ok', message => encode_entities( 'Syntax ok' ) };
+    return { status => 'ok', message =>  $message };
 }
 
 $json_schemas{start_domain_test} = joi->object->strict->props(
@@ -306,22 +350,27 @@ sub start_domain_test {
 
     my $result = 0;
 
-    $params->{domain} =~ s/^\.// unless ( !$params->{domain} || $params->{domain} eq '.' );
-    my $syntax_result = $self->validate_syntax( $params );
-    die "$syntax_result->{message} \n" unless ( $syntax_result && $syntax_result->{status} eq 'ok' );
+    eval {
+        $params->{domain} =~ s/^\.// unless ( !$params->{domain} || $params->{domain} eq '.' );
+        my $syntax_result = $self->validate_syntax( $params );
+        die "$syntax_result->{message} \n" unless ( $syntax_result && $syntax_result->{status} eq 'ok' );
 
-    die "No domain in parameters\n" unless ( $params->{domain} );
+        die "No domain in parameters\n" unless ( $params->{domain} );
 
-    if ($params->{config}) {
-        $params->{config} =~ s/[^\w_]//isg;
-        die "Unknown test configuration: [$params->{config}]\n" unless ( Zonemaster::Backend::Config->load_config()->GetCustomConfigParameter('ZONEMASTER', $params->{config}) );
-    }
+        if ($params->{config}) {
+            $params->{config} =~ s/[^\w_]//isg;
+            die "Unknown test configuration: [$params->{config}]\n" unless ( Zonemaster::Backend::Config->load_config()->GetCustomConfigParameter('ZONEMASTER', $params->{config}) );
+        }
 
-    $params->{priority}  //= 10;
-    $params->{queue}     //= 0;
-    my $minutes_between_tests_with_same_params = 10;
+        $params->{priority}  //= 10;
+        $params->{queue}     //= 0;
+        my $minutes_between_tests_with_same_params = 10;
 
-    $result = $self->{db}->create_new_test( $params->{domain}, $params, $minutes_between_tests_with_same_params );
+        $result = $self->{db}->create_new_test( $params->{domain}, $params, $minutes_between_tests_with_same_params );
+    };
+    if ($@) {
+        handle_exception('start_domain_test');
+    }
 
     return $result;
 }
@@ -331,16 +380,22 @@ $json_schemas{test_progress} = joi->object->strict->props(
 );
 sub test_progress {
     my ( $self, $params ) = @_;
-    my $test_id = "";
-    if (ref \$params eq "SCALAR") {
-        $test_id = $params;
-    } else {
-        $test_id = $params->{"test_id"};
-    }
-
+    
     my $result = 0;
+    eval {
+        my $test_id = "";
+        if (ref \$params eq "SCALAR") {
+            $test_id = $params;
+        } else {
+            $test_id = $params->{"test_id"};
+        }
+
 
-    $result = $self->{db}->test_progress( $test_id );
+        $result = $self->{db}->test_progress( $test_id );
+    };
+    if ($@) {
+        handle_exception('test_progress');
+    }
 
     return $result;
 }
@@ -354,7 +409,12 @@ sub get_test_params {
 
     my $result = 0;
 
-    $result = $self->{db}->get_test_params( $test_id );
+    eval {
+        $result = $self->{db}->get_test_params( $test_id );
+    };
+    if ($@) {
+        handle_exception('get_test_params');
+    }
 
     return $result;
 }
@@ -368,63 +428,68 @@ sub get_test_results {
 
     my $result;
 
-    my $translator;
-    $translator = Zonemaster::Backend::Translator->new;
-    my ( $browser_lang ) = ( $params->{language} =~ /^(\w{2})/ );
+    eval {
+        my $translator;
+        $translator = Zonemaster::Backend::Translator->new;
+        my ( $browser_lang ) = ( $params->{language} =~ /^(\w{2})/ );
 
-    eval { $translator->data } if $translator;    # Provoke lazy loading of translation data
+        eval { $translator->data } if $translator;    # Provoke lazy loading of translation data
 
-    my $test_info = $self->{db}->test_results( $params->{id} );
-    my @zm_results;
-    foreach my $test_res ( @{ $test_info->{results} } ) {
-        my $res;
-        if ( $test_res->{module} eq 'NAMESERVER' ) {
-            $res->{ns} = ( $test_res->{args}->{ns} ) ? ( $test_res->{args}->{ns} ) : ( 'All' );
-        }
-        elsif ($test_res->{module} eq 'SYSTEM'
-            && $test_res->{tag} eq 'POLICY_DISABLED'
-            && $test_res->{args}->{name} eq 'Example' )
-        {
-            next;
-        }
-
-        $res->{module} = $test_res->{module};
-        $res->{message} = $translator->translate_tag( $test_res, $browser_lang ) . "\n";
-        $res->{message} =~ s/,/, /isg;
-        $res->{message} =~ s/;/; /isg;
-        $res->{level} = $test_res->{level};
-
-        if ( $test_res->{module} eq 'SYSTEM' ) {
-            if ( $res->{message} =~ /policy\.json/ ) {
-                my ( $policy ) = ( $res->{message} =~ /\s(\/.*)$/ );
-                if ( $policy ) {
-                    my $policy_description = 'DEFAULT POLICY';
-                    $policy_description = 'SOME OTHER POLICY' if ( $policy =~ /some\/other\/policy\/path/ );
-                    $res->{message} =~ s/$policy/$policy_description/;
-                }
-                else {
-                    $res->{message} = 'UNKNOWN POLICY FORMAT';
-                }
+        my $test_info = $self->{db}->test_results( $params->{id} );
+        my @zm_results;
+        foreach my $test_res ( @{ $test_info->{results} } ) {
+            my $res;
+            if ( $test_res->{module} eq 'NAMESERVER' ) {
+                $res->{ns} = ( $test_res->{args}->{ns} ) ? ( $test_res->{args}->{ns} ) : ( 'All' );
             }
-            elsif ( $res->{message} =~ /config\.json/ ) {
-                my ( $config ) = ( $res->{message} =~ /\s(\/.*)$/ );
-                if ( $config ) {
-                    my $config_description = 'DEFAULT CONFIGURATION';
-                    $config_description = 'SOME OTHER CONFIGURATION' if ( $config =~ /some\/other\/configuration\/path/ );
-                    $res->{message} =~ s/$config/$config_description/;
+            elsif ($test_res->{module} eq 'SYSTEM'
+                && $test_res->{tag} eq 'POLICY_DISABLED'
+                && $test_res->{args}->{name} eq 'Example' )
+            {
+                next;
+            }
+
+            $res->{module} = $test_res->{module};
+            $res->{message} = $translator->translate_tag( $test_res, $browser_lang ) . "\n";
+            $res->{message} =~ s/,/, /isg;
+            $res->{message} =~ s/;/; /isg;
+            $res->{level} = $test_res->{level};
+
+            if ( $test_res->{module} eq 'SYSTEM' ) {
+                if ( $res->{message} =~ /policy\.json/ ) {
+                    my ( $policy ) = ( $res->{message} =~ /\s(\/.*)$/ );
+                    if ( $policy ) {
+                        my $policy_description = 'DEFAULT POLICY';
+                        $policy_description = 'SOME OTHER POLICY' if ( $policy =~ /some\/other\/policy\/path/ );
+                        $res->{message} =~ s/$policy/$policy_description/;
+                    }
+                    else {
+                        $res->{message} = 'UNKNOWN POLICY FORMAT';
+                    }
                 }
-                else {
-                    $res->{message} = 'UNKNOWN CONFIG FORMAT';
+                elsif ( $res->{message} =~ /config\.json/ ) {
+                    my ( $config ) = ( $res->{message} =~ /\s(\/.*)$/ );
+                    if ( $config ) {
+                        my $config_description = 'DEFAULT CONFIGURATION';
+                        $config_description = 'SOME OTHER CONFIGURATION' if ( $config =~ /some\/other\/configuration\/path/ );
+                        $res->{message} =~ s/$config/$config_description/;
+                    }
+                    else {
+                        $res->{message} = 'UNKNOWN CONFIG FORMAT';
+                    }
                 }
             }
+
+            push( @zm_results, $res );
         }
 
-        push( @zm_results, $res );
+        $result = $test_info;
+        $result->{results} = \@zm_results;
+    };
+    if ($@) {
+        handle_exception('get_test_results');
     }
 
-    $result = $test_info;
-    $result->{results} = \@zm_results;
-
     return $result;
 }
 
@@ -441,11 +506,16 @@ sub get_test_history {
 
     my $results;
 
-    $p->{offset} //= 0;
-    $p->{limit} //= 200;
-    $p->{filter} //= "all";
+    eval {
+        $p->{offset} //= 0;
+        $p->{limit} //= 200;
+        $p->{filter} //= "all";
 
-    $results = $self->{db}->get_test_history( $p );
+        $results = $self->{db}->get_test_history( $p );
+    };
+    if ($@) {
+        handle_exception('get_test_history');
+    }
 
     return $results;
 }
@@ -459,16 +529,21 @@ sub add_api_user {
 
     my $result = 0;
 
-    my $allow = 0;
-    if ( defined $remote_ip ) {
-        $allow = 1 if ( $remote_ip eq '::1' || $remote_ip eq '127.0.0.1' );
-    }
-    else {
-        $allow = 1;
-    }
+    eval {
+        my $allow = 0;
+        if ( defined $remote_ip ) {
+            $allow = 1 if ( $remote_ip eq '::1' || $remote_ip eq '127.0.0.1' );
+        }
+        else {
+            $allow = 1;
+        }
 
-    if ( $allow ) {
-        $result = 1 if ( $self->{db}->add_api_user( $p->{username}, $p->{api_key} ) eq '1' );
+        if ( $allow ) {
+            $result = 1 if ( $self->{db}->add_api_user( $p->{username}, $p->{api_key} ) eq '1' );
+        }
+    };
+    if ($@) {
+        handle_exception('add_api_user');
     }
 
     return $result;
@@ -503,7 +578,13 @@ sub add_batch_job {
     $params->{test_params}->{priority}  //= 5;
     $params->{test_params}->{queue}     //= 0;
 
-    my $results = $self->{db}->add_batch_job( $params );
+    my $results;
+    eval {
+        $results = $self->{db}->add_batch_job( $params );
+    };
+    if ($@) {
+        handle_exception('add_batch_job');
+    }
 
     return $results;
 }
@@ -514,9 +595,18 @@ $json_schemas{get_batch_job_result} = joi->object->strict->props(
 sub get_batch_job_result {
     my ( $self, $params ) = @_;
 
-    my $batch_id = $params->{"batch_id"};
-
-    return $self->{db}->get_batch_job_result($batch_id);
+    my $result;
+    
+    eval {
+        my $batch_id = $params->{"batch_id"};
+
+        $result = $self->{db}->get_batch_job_result($batch_id);
+    };
+    if ($@) {
+        handle_exception('get_batch_job_result');
+    }
+    
+    return $result;
 }
 
 my $rpc_request = joi->object->props(

From 620c5c5e6c68eb2367957601041d087ec5899c82 Mon Sep 17 00:00:00 2001
From: Michal TOMA <toma@nic.fr>
Date: Mon, 28 Oct 2019 10:16:10 +0100
Subject: [PATCH 06/92] Fixed missing exception string in method call

---
 lib/Zonemaster/Backend/RPCAPI.pm | 35 ++++++++++++++++----------------
 1 file changed, 18 insertions(+), 17 deletions(-)

diff --git a/lib/Zonemaster/Backend/RPCAPI.pm b/lib/Zonemaster/Backend/RPCAPI.pm
index ef664412c..b66b7fb28 100644
--- a/lib/Zonemaster/Backend/RPCAPI.pm
+++ b/lib/Zonemaster/Backend/RPCAPI.pm
@@ -57,15 +57,16 @@ sub new {
             warn "Failed to initialize the database backend module: [$@] \n";
             die "Failed to initialize the database backend module \n" if $@;
         }
+    }
 
     return ( $self );
 }
 
 sub handle_exception {
-    my ( $method ) = @_;
+    my ( $method, $exception ) = @_;
 
-    warn "Unexpected error in the $method API call: [$@] \n";
-    die "Unexpected error in the $method API call \n" if $@;
+    warn "Unexpected error in the $method API call: [$exception] \n";
+    die "Unexpected error in the $method API call \n";
 }
 
 $json_schemas{version_info} = joi->object->strict;
@@ -79,7 +80,7 @@ sub version_info {
 
     };
     if ($@) {
-        handle_exception('version_info');
+        handle_exception('version_info', $@);
     }
     
     return \%ver;
@@ -95,7 +96,7 @@ sub profile_names {
 
     };
     if ($@) {
-        handle_exception('profile_names');
+        handle_exception('profile_names', $@);
     }
     
     return \@profiles;
@@ -115,7 +116,7 @@ sub get_host_by_name {
 
     };
     if ($@) {
-        handle_exception('get_host_by_name');
+        handle_exception('get_host_by_name', $@);
     }
     
     return \@adresses;
@@ -163,7 +164,7 @@ sub get_data_from_parent_zone {
 
     };
     if ($@) {
-        handle_exception('get_data_from_parent_zone');
+        handle_exception('get_data_from_parent_zone', $@);
     }
     
     return \%result;
@@ -220,7 +221,7 @@ sub _check_domain {
         }
     };
     if ($@) {
-        handle_exception('_check_domain');
+        handle_exception('_check_domain', $@);
     }
 
     return ( $dn, { status => 'ok', message => 'Syntax ok' } );
@@ -322,7 +323,7 @@ sub validate_syntax {
         $message = encode_entities( 'Syntax ok' );
     };
     if ($@) {
-        handle_exception('validate_syntax');
+        handle_exception('validate_syntax', $@);
     }
 
     return { status => 'ok', message =>  $message };
@@ -369,7 +370,7 @@ sub start_domain_test {
         $result = $self->{db}->create_new_test( $params->{domain}, $params, $minutes_between_tests_with_same_params );
     };
     if ($@) {
-        handle_exception('start_domain_test');
+        handle_exception('start_domain_test', $@);
     }
 
     return $result;
@@ -394,7 +395,7 @@ sub test_progress {
         $result = $self->{db}->test_progress( $test_id );
     };
     if ($@) {
-        handle_exception('test_progress');
+        handle_exception('test_progress', $@);
     }
 
     return $result;
@@ -413,7 +414,7 @@ sub get_test_params {
         $result = $self->{db}->get_test_params( $test_id );
     };
     if ($@) {
-        handle_exception('get_test_params');
+        handle_exception('get_test_params', $@);
     }
 
     return $result;
@@ -487,7 +488,7 @@ sub get_test_results {
         $result->{results} = \@zm_results;
     };
     if ($@) {
-        handle_exception('get_test_results');
+        handle_exception('get_test_results', $@);
     }
 
     return $result;
@@ -514,7 +515,7 @@ sub get_test_history {
         $results = $self->{db}->get_test_history( $p );
     };
     if ($@) {
-        handle_exception('get_test_history');
+        handle_exception('get_test_history', $@);
     }
 
     return $results;
@@ -543,7 +544,7 @@ sub add_api_user {
         }
     };
     if ($@) {
-        handle_exception('add_api_user');
+        handle_exception('add_api_user', $@);
     }
 
     return $result;
@@ -583,7 +584,7 @@ sub add_batch_job {
         $results = $self->{db}->add_batch_job( $params );
     };
     if ($@) {
-        handle_exception('add_batch_job');
+        handle_exception('add_batch_job', $@);
     }
 
     return $results;
@@ -603,7 +604,7 @@ sub get_batch_job_result {
         $result = $self->{db}->get_batch_job_result($batch_id);
     };
     if ($@) {
-        handle_exception('get_batch_job_result');
+        handle_exception('get_batch_job_result', $@);
     }
     
     return $result;

From 84cca9c2f1b0af5d7a7d86c822ae773cb2b11271 Mon Sep 17 00:00:00 2001
From: Michal TOMA <toma@nic.fr>
Date: Mon, 16 Mar 2020 17:02:26 +0100
Subject: [PATCH 07/92] First version of the testing document

---
 docs/Garbage-Collection-Testing.md | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 docs/Garbage-Collection-Testing.md

diff --git a/docs/Garbage-Collection-Testing.md b/docs/Garbage-Collection-Testing.md
new file mode 100644
index 000000000..f51c061a5
--- /dev/null
+++ b/docs/Garbage-Collection-Testing.md
@@ -0,0 +1,4 @@
+# Testing instructions for the Garbage Collection feature of the Zonemaster Backend module
+
+## Introduction
+The purpose of this instruction is to serve as a notice for manual testing of the new garbage collection feature at release time.

From e98ec21cde310a188a369275b4cd61420211c5cf Mon Sep 17 00:00:00 2001
From: Michal TOMA <toma@nic.fr>
Date: Thu, 19 Mar 2020 13:32:39 +0100
Subject: [PATCH 08/92] Added the garbage collection testing documentation

---
 docs/Garbage-Collection-Testing.md | 118 +++++++++++++++++++++++++++++
 1 file changed, 118 insertions(+)

diff --git a/docs/Garbage-Collection-Testing.md b/docs/Garbage-Collection-Testing.md
index f51c061a5..e73269ec4 100644
--- a/docs/Garbage-Collection-Testing.md
+++ b/docs/Garbage-Collection-Testing.md
@@ -2,3 +2,121 @@
 
 ## Introduction
 The purpose of this instruction is to serve as a notice for manual testing of the new garbage collection feature at release time.
+
+## Testing the unfinished tests garbage collection feature
+
+1. Ensure that the database has the required aditionnal columns for this feature:
+```
+SELECT nb_retries FROM test_results LIMIT 0;
+```
+Should return:
+
+```
+ nb_retries 
+------------
+(0 rows)
+```
+
+2. Check that your backend_config.ini has the proper parameter set
+
+Either disabled:
+```
+#maximal_number_of_retries=3
+```
+or set to 0:
+```
+maximal_number_of_retries=0
+```
+
+3. Start a test and wait for it to be finished
+
+```
+SELECT hash_id, progress FROM test_results LIMIT 1;
+```
+Should return:
+
+```
+     hash_id      | progress 
+------------------+----------
+ 3f7a604683efaf93 |      100
+(1 row)
+```
+
+4. Sumulate a crashed test
+```
+UPDATE test_results SET progress = 50, test_start_time = '2020-01-01' WHERE hash_id = '3f7a604683efaf93';
+```
+
+5. Chaek that the backend finishes the test with a result stating it was unfinished
+
+```
+SELECT hash_id, progress FROM test_results WHERE hash_id = '3f7a604683efaf93';
+```
+Should return a finished result:
+```
+     hash_id      | progress 
+------------------+----------
+ 3f7a604683efaf93 |      100
+(1 row)
+```
+6. Ensure the test result contains the backend generated critical message message:
+```
+{"tag":"UNABLE_TO_FINISH_TEST","level":"CRITICAL","timestamp":"300","module":"BACKEND_TEST_AGENT"}
+```
+
+```
+SELECT hash_id, progress FROM test_results WHERE hash_id = '3f7a604683efaf93' AND results::text like '%UNABLE_TO_FINISH_TEST%';
+```
+Should return:
+```
+     hash_id      | progress 
+------------------+----------
+ 3f7a604683efaf93 |      100
+(1 row)
+
+```
+
+## Testing the unfinished tests garbage collection feature with a number of retries set to allow finishing tests without a critical error
+
+1. Set the maximal_number_of_retries parameter to a value greater than 0 in the backend_config.ini config file 
+```
+maximal_number_of_retries=1
+```
+
+2. Restart the test agent in order for the parameter to be taken into account
+```
+zonemaster_backend_testagent restart
+```
+
+3. Sumulate a crashed test
+```
+UPDATE test_results SET progress = 50, test_start_time = '2020-01-01' WHERE hash_id = '3f7a604683efaf93';
+```
+
+4. Check that the backend finishes the test WITHOUT a result stating it was unfinished
+
+```
+SELECT hash_id, progress FROM test_results WHERE hash_id = '3f7a604683efaf93';
+```
+Should return a finished result:
+```
+     hash_id      | progress 
+------------------+----------
+ 3f7a604683efaf93 |      100
+(1 row)
+```
+5. Ensure the test result does NOT contain the backend generated critical message message:
+```
+{"tag":"UNABLE_TO_FINISH_TEST","level":"CRITICAL","timestamp":"300","module":"BACKEND_TEST_AGENT"}
+```
+
+```
+SELECT hash_id, progress FROM test_results WHERE hash_id = '3f7a604683efaf93' AND results::text like '%UNABLE_TO_FINISH_TEST%';
+```
+Should return:
+```
+ hash_id | progress 
+---------+----------
+(0 rows)
+
+```

From 2cbaa94ad37f3f69ab6e97c50e11161f45e85f76 Mon Sep 17 00:00:00 2001
From: Michal TOMA <toma@nic.fr>
Date: Thu, 19 Mar 2020 13:35:06 +0100
Subject: [PATCH 09/92] Moved to a new subdirectory of the docs folder

---
 .../maintenance}/Garbage-Collection-Testing.md                    | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename docs/{ => internal-documentation/maintenance}/Garbage-Collection-Testing.md (100%)

diff --git a/docs/Garbage-Collection-Testing.md b/docs/internal-documentation/maintenance/Garbage-Collection-Testing.md
similarity index 100%
rename from docs/Garbage-Collection-Testing.md
rename to docs/internal-documentation/maintenance/Garbage-Collection-Testing.md

From 7cecc0f6bdc3ec9a3b137b661a4c87ca050ae315 Mon Sep 17 00:00:00 2001
From: Mats Dufberg <mats.dufberg@iis.se>
Date: Tue, 2 Jun 2020 15:48:02 +0200
Subject: [PATCH 10/92] Make the Zonemaster user and group get UID/GID meant
 for system user under FreeBSD.

---
 docs/Installation.md   | 3 ++-
 share/freebsd-pwd.conf | 7 +++++++
 2 files changed, 9 insertions(+), 1 deletion(-)
 create mode 100644 share/freebsd-pwd.conf

diff --git a/docs/Installation.md b/docs/Installation.md
index b8bd4a4d1..88b68a136 100644
--- a/docs/Installation.md
+++ b/docs/Installation.md
@@ -515,7 +515,8 @@ Unless they already exist, add `zonemaster` user and `zonemaster` group
 (the group is created automatically):
 
 ```sh
-pw useradd zonemaster -s /sbin/nologin -d /nonexistent -c "Zonemaster daemon user"
+cd `perl -MFile::ShareDir -le 'print File::ShareDir::dist_dir("Zonemaster-Backend")'`
+pw useradd zonemaster -C freebsd-pwd.conf -s /sbin/nologin -d /nonexistent -c "Zonemaster daemon user"
 ```
 
 Install files to their proper locations:
diff --git a/share/freebsd-pwd.conf b/share/freebsd-pwd.conf
new file mode 100644
index 000000000..894a3cb52
--- /dev/null
+++ b/share/freebsd-pwd.conf
@@ -0,0 +1,7 @@
+# Range of free system UIDs that can be used for Zonemaster user
+minuid = 736
+maxuid = 769
+
+# Range of free system GIDs that can be used for Zonemaster group
+mingid = 736
+maxgid = 769

From 59a267c49493e0b60f6bd68ca97a4559994d0d98 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Fri, 10 Jul 2020 22:41:04 +0200
Subject: [PATCH 11/92] Install zmb and zmtest under lib

---
 MANIFEST                 | 2 ++
 {script => share}/zmb    | 0
 {script => share}/zmtest | 0
 3 files changed, 2 insertions(+)
 rename {script => share}/zmb (100%)
 rename {script => share}/zmtest (100%)

diff --git a/MANIFEST b/MANIFEST
index 2c79f4f8c..3f2433b3b 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -71,6 +71,8 @@ share/zm-rpcapi.lsb
 share/zm-testagent.lsb
 share/zm_rpcapi-bsd
 share/zm_testagent-bsd
+share/zmb
+share/zmtest
 share/zonemaster_backend_testagent.conf
 t/test01.data
 t/test01.t
diff --git a/script/zmb b/share/zmb
similarity index 100%
rename from script/zmb
rename to share/zmb
diff --git a/script/zmtest b/share/zmtest
similarity index 100%
rename from script/zmtest
rename to share/zmtest

From 8ff26f304a3e1ae5fccaa92c8441361efef478df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Sat, 11 Jul 2020 01:02:47 +0200
Subject: [PATCH 12/92] Lint zmtest

---
 share/zmtest | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/share/zmtest b/share/zmtest
index e30a9d29f..1714db722 100755
--- a/share/zmtest
+++ b/share/zmtest
@@ -1,9 +1,9 @@
 #!/bin/sh
 
-bindir="`dirname $0`"
+bindir="$(dirname "$0")"
 
 ZMB="${bindir}/zmb"
-JQ="`which jq`"
+JQ="$(which jq)"
 
 error () {
     status="$1"
@@ -13,7 +13,7 @@ error () {
 }
 
 zmb () {
-    json="`"${ZMB}" "$@"`"
+    json="$("${ZMB}" "$@")"
     if echo "${json}" | "${JQ}" -e .error > /dev/null ; then
         error 1 "method $1: ${json}"
     else
@@ -27,12 +27,12 @@ zmb () {
 domain="$1" ; shift
 
 # Start test
-testid="`zmb start_domain_test --domain "${domain}" | "${JQ}" -r .result`" || exit 1
+testid="$(zmb start_domain_test --domain "${domain}" | "${JQ}" -r .result)" || exit 1
 
 # Wait for test to finish
 while true
 do
-    progress="`zmb test_progress --testid "${testid}" | "${JQ}" -r .result`" || exit 1
+    progress="$(zmb test_progress --testid "${testid}" | "${JQ}" -r .result)" || exit 1
     printf "\r${progress}%% done" >&2
     if [ "${progress}" -eq 100 ] ; then
         echo >&2
@@ -43,4 +43,4 @@ done
 
 # Get test results
 zmb get_test_results --testid "${testid}" --lang en
-echo "testid:" ${testid} >&2
+echo "testid: ${testid}" >&2

From b4838d420a2fd593b348fde03dd1235bb09f60f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Sat, 11 Jul 2020 00:50:27 +0200
Subject: [PATCH 13/92] Exit when somethings goes wrong

Also error messages are improved
---
 share/zmtest | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/share/zmtest b/share/zmtest
index 1714db722..269e6268c 100755
--- a/share/zmtest
+++ b/share/zmtest
@@ -13,12 +13,10 @@ error () {
 }
 
 zmb () {
-    json="$("${ZMB}" "$@")"
-    if echo "${json}" | "${JQ}" -e .error > /dev/null ; then
-        error 1 "method $1: ${json}"
-    else
-        echo "${json}"
-    fi
+    json="$("${ZMB}" "$@" 2>&1)" || error 1 "method $1: ${json}"
+    echo "${json}" | "${JQ}" . >/dev/null 2>&1 || error 1 "method $1 did not return valid JSON output"
+    error="$(echo "${json}" | "${JQ}" -e .error 2>/dev/null)" && error 1 "method $1: ${error}"
+    echo "${json}"
 }
 
 [ -n "${JQ}" ] || error 2 "Dependency not found: jq"
@@ -27,12 +25,14 @@ zmb () {
 domain="$1" ; shift
 
 # Start test
-testid="$(zmb start_domain_test --domain "${domain}" | "${JQ}" -r .result)" || exit 1
+output="$(zmb start_domain_test --domain "${domain}")" || exit $?
+testid="$(echo "${output}" | "${JQ}" -r .result)" || exit $?
 
 # Wait for test to finish
 while true
 do
-    progress="$(zmb test_progress --testid "${testid}" | "${JQ}" -r .result)" || exit 1
+    output="$(zmb test_progress --testid "${testid}")" || exit $?
+    progress="$(echo "${output}" | "${JQ}" -r .result)" || exit $?
     printf "\r${progress}%% done" >&2
     if [ "${progress}" -eq 100 ] ; then
         echo >&2

From ca354a13d06d43210b38201ebbf5289e58e3a58c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Fri, 10 Jul 2020 18:19:04 +0200
Subject: [PATCH 14/92] Make it possible to use a different backend

---
 share/zmb    | 18 +++++++++++-------
 share/zmtest | 31 +++++++++++++++++++++++++------
 2 files changed, 36 insertions(+), 13 deletions(-)

diff --git a/share/zmb b/share/zmb
index dda65c5b9..8ceb68a52 100755
--- a/share/zmb
+++ b/share/zmb
@@ -22,8 +22,9 @@ zmb COMMAND [OPTIONS]
 
 =head1 GLOBAL OPTIONS
 
- --help        Show usage
- --verbose     Show RPC query
+ --help         Show usage
+ --verbose      Show RPC query
+ --server URL   The server to connect to. Default is http://localhost:5000/.
 
 =cut
 
@@ -32,10 +33,12 @@ sub main {
 
     my $opt_help;
     my $opt_verbose;
+    my $opt_server = 'http://localhost:5000/';
     GetOptionsFromArray(
         \@argv,
-        'help'    => \$opt_help,
-        'verbose' => \$opt_verbose,
+        'help'     => \$opt_help,
+        'verbose'  => \$opt_verbose,
+        'server=s' => \$opt_server,
     ) or pod2usage( 2 );
     if ( !@argv ) {
         pod2usage( -verbose => 99, -sections => ['SYNOPSIS', 'GLOBAL OPTIONS'], -exitval => 'NOEXIT' );
@@ -52,7 +55,7 @@ sub main {
 
     if ( $json ) {
         say $json if $opt_verbose;
-        my $request  = to_request( $json );
+        my $request  = to_request( $opt_server, $json );
         my $response = submit( $request );
         say $response;
     }
@@ -398,9 +401,10 @@ sub to_jsonrpc {
 }
 
 sub to_request {
-    my $json = shift;
+    my $server = shift;
+    my $json   = shift;
 
-    my $req = HTTP::Request->new( POST => 'http://localhost:5000/' );
+    my $req = HTTP::Request->new( POST => $server );
     $req->content_type( 'application/json' );
     $req->content( $json );
 
diff --git a/share/zmtest b/share/zmtest
index 269e6268c..b7da8a89e 100755
--- a/share/zmtest
+++ b/share/zmtest
@@ -5,6 +5,17 @@ bindir="$(dirname "$0")"
 ZMB="${bindir}/zmb"
 JQ="$(which jq)"
 
+usage () {
+    status="$1"
+    message="$2"
+    echo "${message}" >&2
+    echo "Usage: zmtest [OPTIONS] DOMAIN" >&2
+    echo >&2
+    echo "Options:" >&2
+    echo "  -s URL --server=URL   Zonemaster Backend to query. Default is http://localhost:5000/" >&2
+    exit "${status}"
+}
+
 error () {
     status="$1"
     message="$2"
@@ -13,7 +24,8 @@ error () {
 }
 
 zmb () {
-    json="$("${ZMB}" "$@" 2>&1)" || error 1 "method $1: ${json}"
+    server_url="$1"; shift
+    json="$("${ZMB}" --server="${server_url}" "$@" 2>&1)" || error 1 "method $1: ${json}"
     echo "${json}" | "${JQ}" . >/dev/null 2>&1 || error 1 "method $1 did not return valid JSON output"
     error="$(echo "${json}" | "${JQ}" -e .error 2>/dev/null)" && error 1 "method $1: ${error}"
     echo "${json}"
@@ -21,17 +33,24 @@ zmb () {
 
 [ -n "${JQ}" ] || error 2 "Dependency not found: jq"
 
-[ $# -ge 1 ] || error 2 "No domain specified"
-domain="$1" ; shift
+domain=""
+server_url="http://localhost:5000/"
+while [ $# -gt 0 ] ; do
+    case "$1" in
+        -s|--server) server_url="$2"; shift 2;;
+        *) domain="$1" ; shift 1;;
+    esac
+done
+[ -n "${domain}" ] || usage 2 "No domain specified"
 
 # Start test
-output="$(zmb start_domain_test --domain "${domain}")" || exit $?
+output="$(zmb "${server_url}" start_domain_test --domain "${domain}")" || exit $?
 testid="$(echo "${output}" | "${JQ}" -r .result)" || exit $?
 
 # Wait for test to finish
 while true
 do
-    output="$(zmb test_progress --testid "${testid}")" || exit $?
+    output="$(zmb "${server_url}" test_progress --testid "${testid}")" || exit $?
     progress="$(echo "${output}" | "${JQ}" -r .result)" || exit $?
     printf "\r${progress}%% done" >&2
     if [ "${progress}" -eq 100 ] ; then
@@ -42,5 +61,5 @@ do
 done
 
 # Get test results
-zmb get_test_results --testid "${testid}" --lang en
+zmb "${server_url}" get_test_results --testid "${testid}" --lang en
 echo "testid: ${testid}" >&2

From 4d2239b20b17e53794441c2af7e46e83a2e2c747 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Fri, 10 Jul 2020 22:07:58 +0200
Subject: [PATCH 15/92] Make it possible to run tests without IPv4/IPv6

---
 share/zmb    | 14 ++++++++++++++
 share/zmtest |  8 +++++++-
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/share/zmb b/share/zmb
index 8ceb68a52..42360c7cb 100755
--- a/share/zmb
+++ b/share/zmb
@@ -120,6 +120,8 @@ Options:
     --client-id CLIENT_ID
     --client-version CLIENT_VERSION
     --ds-info DS_INFO
+    --ipv4 true|false|null
+    --ipv6 true|false|null
 
 DS_INFO is a comma separated list of key-value pairs. The expected pairs are:
 
@@ -138,6 +140,8 @@ sub cmd_start_domain_test {
     my $opt_client_id;
     my $opt_client_version;
     my @opt_ds_info;
+    my $opt_ipv4;
+    my $opt_ipv6;
     GetOptionsFromArray(
         \@opts,
         'domain|d=s'       => \$opt_domain,
@@ -145,6 +149,8 @@ sub cmd_start_domain_test {
         'client-id=s'      => \$opt_client_id,
         'client-version=s' => \$opt_client_version,
         'ds-info=s'        => \@opt_ds_info,
+        'ipv4=s'           => \$opt_ipv4,
+        'ipv6=s'           => \$opt_ipv6,
     ) or pod2usage( 2 );
 
     my %params = ( domain => $opt_domain, );
@@ -186,6 +192,14 @@ sub cmd_start_domain_test {
         $params{nameservers} = \@nameserver_objects;
     }
 
+    if ( $opt_ipv4 ) {
+        $params{ipv4} = json_tern( $opt_ipv4 );
+    }
+
+    if ( $opt_ipv6 ) {
+        $params{ipv6} = json_tern( $opt_ipv6 );
+    }
+
     return to_jsonrpc(
         id     => 1,
         method => 'start_domain_test',
diff --git a/share/zmtest b/share/zmtest
index b7da8a89e..e1c58c29f 100755
--- a/share/zmtest
+++ b/share/zmtest
@@ -13,6 +13,8 @@ usage () {
     echo >&2
     echo "Options:" >&2
     echo "  -s URL --server=URL   Zonemaster Backend to query. Default is http://localhost:5000/" >&2
+    echo "  --noipv4              Run the test without ipv4." >&2
+    echo "  --noipv6              Run the test without ipv6." >&2
     exit "${status}"
 }
 
@@ -35,16 +37,20 @@ zmb () {
 
 domain=""
 server_url="http://localhost:5000/"
+ipv4="true"
+ipv6="true"
 while [ $# -gt 0 ] ; do
     case "$1" in
         -s|--server) server_url="$2"; shift 2;;
+        --noipv4) ipv4="false"; shift 1;;
+        --noipv6) ipv6="false"; shift 1;;
         *) domain="$1" ; shift 1;;
     esac
 done
 [ -n "${domain}" ] || usage 2 "No domain specified"
 
 # Start test
-output="$(zmb "${server_url}" start_domain_test --domain "${domain}")" || exit $?
+output="$(zmb "${server_url}" start_domain_test --domain "${domain}" --ipv4 "${ipv4}" --ipv6 "${ipv6}")" || exit $?
 testid="$(echo "${output}" | "${JQ}" -r .result)" || exit $?
 
 # Wait for test to finish

From 7674f97bd3488e850c7855f4cf77382870308233 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Sat, 11 Jul 2020 07:20:09 +0200
Subject: [PATCH 16/92] Make zmtest pretty-print its output

---
 share/zmtest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/share/zmtest b/share/zmtest
index e1c58c29f..e0d7c7be0 100755
--- a/share/zmtest
+++ b/share/zmtest
@@ -28,7 +28,7 @@ error () {
 zmb () {
     server_url="$1"; shift
     json="$("${ZMB}" --server="${server_url}" "$@" 2>&1)" || error 1 "method $1: ${json}"
-    echo "${json}" | "${JQ}" . >/dev/null 2>&1 || error 1 "method $1 did not return valid JSON output"
+    json="$(echo "${json}" | "${JQ}" -S . 2>/dev/null)" || error 1 "method $1 did not return valid JSON output"
     error="$(echo "${json}" | "${JQ}" -e .error 2>/dev/null)" && error 1 "method $1: ${error}"
     echo "${json}"
 }

From 8eaac738c34c5956203638334878ee6bcc73c435 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Thu, 16 Jul 2020 13:59:52 +0200
Subject: [PATCH 17/92] Fix usage documentation

---
 share/zmb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/share/zmb b/share/zmb
index 42360c7cb..ca0702e45 100755
--- a/share/zmb
+++ b/share/zmb
@@ -18,7 +18,7 @@ Zmb is meant to be pronounced I<Zimba>.
 
 =head1 SYNOPSIS
 
-zmb COMMAND [OPTIONS]
+zmb [GLOBAL OPTIONS] COMMAND [OPTIONS]
 
 =head1 GLOBAL OPTIONS
 

From 2a0088b8562ec29ef1bb4b1cc1f98ac79cce1cc5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Thu, 16 Jul 2020 14:08:57 +0200
Subject: [PATCH 18/92] Fix usage documentation some more

---
 share/zmb | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/share/zmb b/share/zmb
index ca0702e45..9b9b3bc05 100755
--- a/share/zmb
+++ b/share/zmb
@@ -68,7 +68,7 @@ sub main {
 
 Show the full manual page.
 
-    zmb man
+ zmb [GLOBAL OPTIONS] man
 
 =cut
 
@@ -81,7 +81,7 @@ sub cmd_man {
 
 Call a non-existing RPC method.
 
-    zmb non_existing_method
+ zmb [GLOBAL OPTIONS] non_existing_method
 
 =cut
 
@@ -95,7 +95,7 @@ sub cmd_non_existing_method {
 
 =head2 version_info
 
-    zmb version_info
+ zmb [GLOBAL OPTIONS] version_info
 
 =cut
 
@@ -109,11 +109,9 @@ sub cmd_version_info {
 
 =head2 start_domain_test
 
-Call start_domain_test.
+ zmb [GLOBAL OPTIONS] start_domain_test [OPTIONS]
 
-    zmb start_domain_test [OPTIONS]
-
-Options:
+ Options:
 
     --domain DOMAIN_NAME
     --nameserver DOMAIN_NAME:IP_ADDRESS
@@ -123,7 +121,7 @@ Options:
     --ipv4 true|false|null
     --ipv6 true|false|null
 
-DS_INFO is a comma separated list of key-value pairs. The expected pairs are:
+ DS_INFO is a comma separated list of key-value pairs. The expected pairs are:
 
     keytag=UNSIGNED_INTEGER
     algorithm=UNSIGNED_INTEGER
@@ -210,7 +208,7 @@ sub cmd_start_domain_test {
 
 =head2 test_progress
 
- zmb test_progress [OPTIONS]
+ zmb [GLOBAL OPTIONS] test_progress [OPTIONS]
 
  Options:
    --testid TEST_ID
@@ -237,7 +235,7 @@ sub cmd_test_progress {
 
 =head2 get_test_results
 
- zmb get_test_results [OPTIONS]
+ zmb [GLOBAL OPTIONS] get_test_results [OPTIONS]
 
  Options:
    --testid TEST_ID
@@ -269,7 +267,7 @@ sub cmd_get_test_results {
 
 =head2 get_test_history
 
- zmb get_test_history [OPTIONS]
+ zmb [GLOBAL OPTIONS] get_test_history [OPTIONS]
 
  Options:
    --domain DOMAIN_NAME
@@ -322,7 +320,7 @@ sub cmd_get_test_history {
 
 =head2 add_api_user
 
- zmb add_api_user [OPTIONS]
+ zmb [GLOBAL OPTIONS] add_api_user [OPTIONS]
 
  Options:
    --username USERNAME

From 88f6bc97107765426519e39f999bfe4339861a62 Mon Sep 17 00:00:00 2001
From: Mats Dufberg <mats.dufberg@iis.se>
Date: Thu, 16 Jul 2020 15:44:37 +0200
Subject: [PATCH 19/92] Updated documentation

---
 docs/API.md           | 102 +++++++++++++++++++++++++++++++++++-------
 docs/Configuration.md |  92 ++++++++++++++++++++++++++++++++++---
 2 files changed, 172 insertions(+), 22 deletions(-)

diff --git a/docs/API.md b/docs/API.md
index 01b47b0df..b91d8d771 100644
--- a/docs/API.md
+++ b/docs/API.md
@@ -263,25 +263,37 @@ Default database timestamp format: "Y-M-D H:M:S.ms".
 Example: "2017-12-18 07:56:17.156939"
 
 
-### Translation language
+### Language tag
 
 Basic data type: string
 
-A string of alphanumeric, hyphens, underscores, full stops and at-signs (`@`),
-of at least 1 and at most 30 characters.
-I.e. a string matching `/^[a-zA-Z0-9-_.@]{1,30}$/`.
+A string of A-Z, a-z and underscores matching the regular expression
+`/^[a-z]{2}(_[A-Z]{2})?$/`.
 
-* Any string starting with `"fr"` is interpreted as French.
-* Any string starting with `"sv"` is interpreted as Swedish.
-* Any string starting with `"da"` is interpreted as Danish.
-* Any other string is interpreted as English.
+The `language tag` must match a `locale tag` in the configuration file.
+If the `language tag` is a two-character string, it only needs to match the
+first two characters of the `locale tag` from the configuration file, if
+that is unique (there is only one `locale tag` starting with the same two
+characters), else it is an error.
+
+Any other string is an error.
+
+The two first characters of the `language tag` are intended to be an
+[ISO 639-1] two-character language code and the optional two last characters
+are intended to be an [ISO 3166-1 alpha-2] two-character country code.
+
+A default installation will accept the following `language tags`:
+* `da` or `da_DK` for Danish language.
+* `en` or `en_US` for English language.
+* `fr` or `fr_FR` for French language.
+* `sv` or `sv_SE` for Swedish language.
 
 
 ### Unsigned integer
 
- Basic data type: number (integer)
+Basic data type: number (integer)
  
- An unsigned integer is either positive or zero.
+An unsigned integer is either positive or zero.
  
 
 ### Username
@@ -367,6 +379,52 @@ Example response:
 An array of *Profile names* in lower case. `"default"` is always included.
 
 
+## API method: `get_language_tags`
+
+Returns all valid [language tags][language tag] generated from the setting in
+the configuration file.
+
+Example request:
+```json
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "get_language_tags"
+}
+```
+
+Example response:
+```json
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "result": [
+    "en",
+    "en_US",
+    "fr",
+    "fr_FR",
+    "sv",
+    "sv_SE"
+  ]
+}
+```
+
+
+#### `"result"`
+
+An array of *language tags*. It is never empty.
+
+
+#### `"error"`
+
+>
+> TODO: List all possible error codes and describe what they mean enough for
+> clients to know how react to them. Or prevent RPCAPI from starting with
+> errors in the configuration file and make it not to reread the configuration
+> file while running.
+>
+
+
 ## API method: `get_host_by_name`
 
 Looks up the A and AAAA records for a hostname (*domain name*) on the public Internet.
@@ -418,6 +476,7 @@ value `0.0.0.0` if the lookup returned no A or AAAA records.
 > TODO: If the name resolves to two or more IPv4 address, how is that represented?
 >
 
+
 #### `"error"`
 
 >
@@ -573,13 +632,15 @@ An object with the following properties:
 > TODO: Clarify the purpose of each `"params"` property.
 >
 
+
 #### `"result"`
 
 A *test id*. 
 
 If the test has been run with the same domain name within an interval of 10 mins (hard coded), 
 then the new request does not trigger a new test, but returns with the results of the last test
- 
+
+
 #### `"error"`
 
 * If the given `profile` is not among the [available profiles], a user
@@ -637,7 +698,8 @@ A *progress percentage*.
 
 ## API method: `get_test_results`
 
-Return all *test result* objects of a *test*, with *messages* in the requested *translation language*.
+Return all *test result* objects of a *test*, with *messages* in the requested language as selected by the
+*language tag*.
 
 Example request:
 ```json
@@ -652,6 +714,9 @@ Example request:
 }
 ```
 
+The `id` parameter must match the `result` in the response to a `start_domain_test` call,
+and that test must have been completed.
+
 Example response:
 ```json
 {
@@ -710,7 +775,7 @@ Example response:
 An object with the following properties:
 
 * `"id"`: A *test id*, required.
-* `"language"`: A *translation language*, required.
+* `"language"`: A *language tag*, required.
 
 
 #### `"result"`
@@ -740,7 +805,6 @@ In the case of a test created with `add_batch_job`:
 >
 
 
-
 #### `"error"`
 
 >
@@ -877,10 +941,12 @@ An object with the following properties:
 * `"username"`: An *username*, required. The name of the user to add.
 * `"api_key"`: An *api key*, required. The API key for the user to add.
 
+
 #### `"result"`
 
 An integer. The value is equal to 1 if the registration is a success, or 0 if it failed.
 
+
 #### `"error"`
 >
 > TODO: List all possible error codes and describe what they mean enough for clients to know how react to them.
@@ -1088,6 +1154,7 @@ Example response:
 }
 ```
 
+
 #### `"params"`
 
 An object with the property:
@@ -1106,5 +1173,8 @@ The `"params"` object sent to `start_domain_test` or `add_batch_job` when the *t
 > TODO: List all possible error codes and describe what they mean enough for clients to know how react to them.
 >
 
-[Available profiles]: Configuration.md#profiles-section
-[Privilege levels]: #privilege-levels
+[Available profiles]:           Configuration.md#profiles-section
+[ISO 3166-1 alpha-2]:           https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2
+[ISO 639-1]:                    https://en.wikipedia.org/wiki/ISO_639-1
+[Privilege levels]:             #privilege-levels
+[Language tag]:                 #language-tag
diff --git a/docs/Configuration.md b/docs/Configuration.md
index 16353f263..468c89ae0 100644
--- a/docs/Configuration.md
+++ b/docs/Configuration.md
@@ -32,6 +32,83 @@ SQLite            | `SQLite`
 
 TBD
 
+## LANGUAGE section
+
+The LANGUAGE section has one key, `locale`.
+
+The value of the `locale` key is a space separated list of
+`locale tags` where each tag must match the regular expression
+`/^[a-z]{2}_[A-Z]{2}$/`.
+
+If the `locale` key is empty or absent, the `locale tag` value
+"en_US" is set by default.
+
+The two first characters of a `locale tag` are intended to be an
+[ISO 639-1] two-character language code and the two last characters
+are intended to be an [ISO 3166-1 alpha-2] two-character country code.
+A `locale tag` is a locale setting for the available translation
+of messages without ".UTF-8", which is implied.
+
+If a new `locale tag` is added to the configuration then the equivalent
+MO file should be added to Zonemaster-Engine at the correct place so
+that gettext can retrieve it, or else the added `locale tag` will not
+add any actual language support. See the
+[Zonemaster-Engine share directory] for the existing PO files that are
+converted to MO files. (Here we should have a link
+to documentation instead.)
+
+Removing a language from the configuration file just blocks that
+language from being allowed. If there are more than one `locale tag`
+(with different country codes) for the same language, then
+all those must be removed to block that language.
+
+English is the Zonemaster default language, but it can be blocked
+from being allowed by RPC-API by not including it in the
+configuration.
+
+In the RPCAPI, `language tag` is used ([Language tag]). The
+`language tags` are generated from the `locale tags`. Each
+`locale tag` will generate two `language tags`, a short tag
+equal to the first two letters (usually the same as a language
+code) and a long tag which is equal to the full `locale tag`.
+If "en_US" is the `locale tag` then "en" and "en_US" are the
+`language tags`.
+
+If there are two `locale tags` that would give the same short
+`language tag` then that is excluded. E.g. "en_US en_UK" will
+only give "en_US" and "en_UK" as `language tags`.
+
+The default installation and configuration supports the
+following languages.
+
+Language | Locale tag value   | Locale value used
+---------|--------------------|------------------
+Danish   | da_DK              | da_DK.UTF-8
+English  | en_US              | en_US.UTF-8
+French   | fr_FR              | fr_FR.UTF-8
+Swedish  | sv_SE              | sv_SE.UTF-8
+
+The following `language tags` are generated:
+* da
+* da_DK
+* en
+* en_US
+* fr
+* fr_FR
+* sv
+* sv_SE
+
+It is an error to repeat the same `locale tag`.
+
+Setting in the default configuration file:
+
+```
+locale = da_DK en_US fr_FR sv_SE
+```
+
+Each locale set in the configuration file, including the implied
+".UTF-8", must also be installed or activate on the system
+running the RPCAPI daemon for the translation to work correctly.
 
 ## LOG section
 
@@ -74,11 +151,14 @@ TBD
 
 --------
 
-[Zonemaster::Engine::Profile]: https://metacpan.org/pod/Zonemaster::Engine::Profile#PROFILE-PROPERTIES
-[Default JSON profile file]: https://github.com/zonemaster/zonemaster-engine/blob/master/share/profile.json
-[Profile JSON files]: https://github.com/zonemaster/zonemaster-engine/blob/master/docs/Profiles.md
-[Profile names]: API.md#profile-name
-[Profiles]: Architecture.md#profile
-
+[Default JSON profile file]:          https://github.com/zonemaster/zonemaster-engine/blob/master/share/profile.json
+[ISO 3166-1 alpha-2]:                 https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2
+[ISO 639-1]:                          https://en.wikipedia.org/wiki/ISO_639-1
+[Profile JSON files]:                 https://github.com/zonemaster/zonemaster-engine/blob/master/docs/Profiles.md
+[Profile names]:                      API.md#profile-name
+[Profiles]:                           Architecture.md#profile
+[Zonemaster-Engine share directory]:  https://github.com/zonemaster/zonemaster-engine/tree/master/share
+[Zonemaster::Engine::Profile]:        https://metacpan.org/pod/Zonemaster::Engine::Profile#PROFILE-PROPERTIES
+[Language tag]:                       API.md#language-tag
 
 

From 221af30251717424251627485ef1a0ebdd05da3b Mon Sep 17 00:00:00 2001
From: Mats Dufberg <mats.dufberg@iis.se>
Date: Thu, 16 Jul 2020 15:45:27 +0200
Subject: [PATCH 20/92] Updated implementation

---
 lib/Zonemaster/Backend/Config.pm      | 71 +++++++++++++++++++++++++++
 lib/Zonemaster/Backend/RPCAPI.pm      | 16 ++++--
 lib/Zonemaster/Backend/Translator.pm  | 21 ++++----
 lib/Zonemaster/Backend/Validator.pm   |  4 +-
 script/zonemaster_backend_rpcapi.psgi | 15 ++++--
 5 files changed, 107 insertions(+), 20 deletions(-)

diff --git a/lib/Zonemaster/Backend/Config.pm b/lib/Zonemaster/Backend/Config.pm
index e091d227c..0cd66063c 100644
--- a/lib/Zonemaster/Backend/Config.pm
+++ b/lib/Zonemaster/Backend/Config.pm
@@ -76,6 +76,77 @@ sub DB_name {
     return $self->{cfg}->val( 'DB', 'database_name' );
 }
 
+=head2 Language_Locale_hash
+
+Read LANGUAGE.locale from the configuration (.ini) file and returns
+the valid language tags for RPCAPI. The incoming language tag
+from RPCAPI is compared to those. The language tags are mapped to
+locale setting value.
+
+=head3 INPUT
+
+None
+
+=head3 RETURNS
+
+A hash of valid language tags as keys with set locale value as value.
+The hash is never empty.
+
+=cut
+
+sub Language_Locale_hash {
+    # There is one special value to capture ambiguous (and therefore
+    # not permitted) translation language tags.
+    my ($self) = @_;
+    my $data = $self->{cfg}->val( 'LANGUAGE', 'locale' );
+    $data = 'en_US' unless $data;
+    my @localetags = split (/\s+/,$data);
+    my %locale;
+    foreach my $la (@localetags) {
+        die "Incorrect locale tag in LANGUAGE.locale: $la\n" unless $la =~ /^[a-z]{2}_[A-Z]{2}$/;
+        die "Repeated locale tag in LANGUAGE.locale: $la\n" if $locale{$la};
+        (my $a) = split (/_/,$la); # $a is the language code only
+        my $lo = "$la.UTF-8";
+        # Set special value if the same language code is used more than once
+        # with different country codes.
+        if ( $locale{$a} and $locale{$a} ne $lo ) {
+            $locale{$a} = 'NOT-UNIQUE';
+        }
+        else {
+            $locale{$a} = $lo;
+        }
+        $locale{$la} = $lo;
+    }
+    return %locale;
+}
+
+=head2 ListLanguageTags
+
+Read indirectly LANGUAGE.locale from the configuration (.ini) file
+and returns a list of valid language tags for RPCAPI. The list can
+be retrieved via an RPCAPI method.
+
+=head3 INPUT
+
+None
+
+=head3 RETURNS
+
+An array of valid language tags. The array is never empty.
+
+=cut
+
+sub ListLanguageTags {
+    my ($self) = @_;
+    my %locale = &Language_Locale_hash($self);
+    my @langtags;
+    foreach my $key (keys %locale) {
+        push @langtags, $key unless $locale{$key} eq 'NOT-UNIQUE';
+    }
+    return @langtags;
+}
+
+
 sub DB_connection_string {
     my ($self) = @_;
 
diff --git a/lib/Zonemaster/Backend/RPCAPI.pm b/lib/Zonemaster/Backend/RPCAPI.pm
index 14e435cae..2cc22f379 100644
--- a/lib/Zonemaster/Backend/RPCAPI.pm
+++ b/lib/Zonemaster/Backend/RPCAPI.pm
@@ -78,6 +78,17 @@ sub profile_names {
     return \@profiles;
 }
 
+# Return the list of language tags supported by get_test_results(). The tags are
+# derived from the locale tags set in the configuration file.
+$json_schemas{get_language_tags} = joi->object->strict;
+sub get_language_tags {
+    my ($self) = @_;
+
+    my @lang = Zonemaster::Backend::Config->load_config()->ListLanguageTags();
+
+    return \@lang;
+}
+
 $json_schemas{get_host_by_name} = joi->object->strict->props(
     hostname   => $zm_validator->domain_name->required
 );
@@ -358,7 +369,7 @@ sub get_test_params {
 
 $json_schemas{get_test_results} = joi->object->strict->props(
     id => $zm_validator->test_id->required,
-    language => $zm_validator->translation_language->required
+    language => $zm_validator->language_tag->required
 );
 sub get_test_results {
     my ( $self, $params ) = @_;
@@ -367,7 +378,6 @@ sub get_test_results {
 
     my $translator;
     $translator = Zonemaster::Backend::Translator->new;
-    my ( $browser_lang ) = ( $params->{language} =~ /^(\w{2})/ );
 
     eval { $translator->data } if $translator;    # Provoke lazy loading of translation data
 
@@ -386,7 +396,7 @@ sub get_test_results {
         }
 
         $res->{module} = $test_res->{module};
-        $res->{message} = $translator->translate_tag( $test_res, $browser_lang ) . "\n";
+        $res->{message} = $translator->translate_tag( $test_res, $params->{language} ) . "\n";
         $res->{message} =~ s/,/, /isg;
         $res->{message} =~ s/;/; /isg;
         $res->{level} = $test_res->{level};
diff --git a/lib/Zonemaster/Backend/Translator.pm b/lib/Zonemaster/Backend/Translator.pm
index f250c5f9f..5584098fc 100644
--- a/lib/Zonemaster/Backend/Translator.pm
+++ b/lib/Zonemaster/Backend/Translator.pm
@@ -7,6 +7,7 @@ use 5.14.2;
 use Moose;
 use Encode;
 use POSIX qw[setlocale LC_MESSAGES LC_CTYPE];
+use Zonemaster::Backend::Config;
 
 # Zonemaster Modules
 require Zonemaster::Engine::Translator;
@@ -16,19 +17,19 @@ extends 'Zonemaster::Engine::Translator';
 
 sub translate_tag {
     my ( $self, $entry, $browser_lang ) = @_;
-
+    my %locale = Zonemaster::Backend::Config->load_config()->Language_Locale_hash();
     my $previous_locale = $self->locale;
-    if ( $browser_lang eq 'fr' ) {
-        $self->locale( "fr_FR.UTF-8" );
-    }
-    elsif ( $browser_lang eq 'sv' ) {
-        $self->locale( "sv_SE.UTF-8" );
-    }
-    elsif ( $browser_lang eq 'da' ) {
-        $self->locale( "da_DK.UTF-8" );
+
+    if ( $locale{$browser_lang} ) {
+        if ( $locale{$browser_lang} eq 'NOT-UNIQUE') {
+            die "Language string not unique: '$browser_lang'\n";
+        }
+        else {
+            $self->locale( $locale{$browser_lang} );
+        }
     }
     else {
-        $self->locale( "en_US.UTF-8" );
+        die "Undefined language string: '$browser_lang'\n";
     }
 
     # Make locale really be set. Fix that makes translation work on FreeBSD 12.1. Solution copied from
diff --git a/lib/Zonemaster/Backend/Validator.pm b/lib/Zonemaster/Backend/Validator.pm
index 7da59b8fe..157e26bd1 100644
--- a/lib/Zonemaster/Backend/Validator.pm
+++ b/lib/Zonemaster/Backend/Validator.pm
@@ -64,8 +64,8 @@ sub queue {
 sub test_id {
     return joi->string->regex('^[0-9]$|^[1-9][0-9]{1,8}$|^[0-9a-f]{16}$');
 }
-sub translation_language {
-    return joi->string->regex('^[a-zA-Z0-9-_.@]{1,30}$');
+sub language_tag {
+    return joi->string->regex('^[a-z]{2}(_[A-Z]{2})?$');
 }
 sub username {
     return joi->string->regex('^[a-zA-Z0-9-.@]{1,50}$');
diff --git a/script/zonemaster_backend_rpcapi.psgi b/script/zonemaster_backend_rpcapi.psgi
index 4b6dcb2cf..751ae8c3c 100644
--- a/script/zonemaster_backend_rpcapi.psgi
+++ b/script/zonemaster_backend_rpcapi.psgi
@@ -42,11 +42,16 @@ my $router = router {
 	};
 
 	connect "profile_names" => {
-        handler => "+Zonemaster::Backend::RPCAPI",
-        action => "profile_names"
-  };
-  
-  connect "get_host_by_name" => {
+                handler => "+Zonemaster::Backend::RPCAPI",
+                action => "profile_names"
+        };
+
+	connect "get_language_tags" => {
+                handler => "+Zonemaster::Backend::RPCAPI",
+                action => "get_language_tags"
+        };
+
+        connect "get_host_by_name" => {
 		handler => "+Zonemaster::Backend::RPCAPI",
 		action => "get_host_by_name"
 	};

From 96cc6ef16f72abece203772202a3b1896bfacb6e Mon Sep 17 00:00:00 2001
From: Mats Dufberg <mats.dufberg@iis.se>
Date: Thu, 16 Jul 2020 15:45:58 +0200
Subject: [PATCH 21/92] Updated default configuration

---
 share/backend_config.ini | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/share/backend_config.ini b/share/backend_config.ini
index b8a36fd88..f947146b4 100755
--- a/share/backend_config.ini
+++ b/share/backend_config.ini
@@ -1,3 +1,6 @@
+# For documentation of the backend_config.ini file see
+# https://github.com/zonemaster/zonemaster-backend/blob/master/docs/Configuration.md
+
 [DB]
 engine            = MySQL
 user              = zonemaster
@@ -23,6 +26,9 @@ number_of_processes_for_batch_testing    = 20
 #
 #maximal_number_of_retries=3
 
+[LANGUAGE]
+locale = da_DK en_US fr_FR sv_SE
+
 [PUBLIC PROFILES]
 #example_profile_1=/example/directory/test1_profile.json
 #default=/example/directory/default_profile.json

From e69b514ba16b25db9bd5dd21476e95affa8c74f2 Mon Sep 17 00:00:00 2001
From: Mats Dufberg <mats.dufberg@iis.se>
Date: Thu, 16 Jul 2020 15:54:38 +0200
Subject: [PATCH 22/92] Updated unit tests

---
 share/travis_mysql_backend_config.ini      |  4 ++++
 share/travis_postgresql_backend_config.ini |  4 ++++
 share/travis_sqlite_backend_config.ini     |  3 +++
 t/test01.t                                 |  9 ++++++++-
 t/test_DB_backend.pl                       | 12 +++++++++++-
 5 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/share/travis_mysql_backend_config.ini b/share/travis_mysql_backend_config.ini
index 2cf612a0e..7f60f2cf6 100644
--- a/share/travis_mysql_backend_config.ini
+++ b/share/travis_mysql_backend_config.ini
@@ -27,3 +27,7 @@ number_of_processes_for_batch_testing=20
 # upt to that value (the id value given here will be the highest posible value allowed as 
 # simple id, for everything above hash_id will be forced).
 force_hash_id_use_in_API_starting_from_id=1
+
+[LANGUAGE]
+locale = da_DK en_US fr_FR sv_SE
+
diff --git a/share/travis_postgresql_backend_config.ini b/share/travis_postgresql_backend_config.ini
index 5a14431a4..5c181dd6e 100644
--- a/share/travis_postgresql_backend_config.ini
+++ b/share/travis_postgresql_backend_config.ini
@@ -27,3 +27,7 @@ number_of_professes_for_batch_testing=20
 # upt to that value (the id value given here will be the highest posible value allowed as 
 # simple id, for everything above hash_id will be forced).
 force_hash_id_use_in_API_starting_from_id=1
+
+[LANGUAGE]
+locale = da_DK en_US fr_FR sv_SE
+
diff --git a/share/travis_sqlite_backend_config.ini b/share/travis_sqlite_backend_config.ini
index 44098cdc0..90d3fb2d8 100644
--- a/share/travis_sqlite_backend_config.ini
+++ b/share/travis_sqlite_backend_config.ini
@@ -21,3 +21,6 @@ number_of_processes_for_frontend_testing=20
 number_of_processes_for_batch_testing=20
 #seconds
 
+[LANGUAGE]
+locale = da_DK en_US fr_FR sv_SE
+
diff --git a/t/test01.t b/t/test01.t
index 388cd78a7..5e48113c8 100644
--- a/t/test01.t
+++ b/t/test01.t
@@ -3,6 +3,7 @@ use warnings;
 use 5.14.2;
 
 use Test::More;    # see done_testing()
+use Test::Exception;
 use Zonemaster::Engine;
 use JSON::PP;
 
@@ -82,13 +83,19 @@ sub run_zonemaster_test_with_backend_API {
 		last if ( $progress == 100 );
 	}
 	ok( $engine->test_progress( $test_id ) == 100 );
-	my $test_results = $engine->get_test_results( { id => $test_id, language => 'fr-FR' } );
+
+        my $test_results = $engine->get_test_results( { id => $test_id, language => 'fr_FR' } );
 	ok( defined $test_results->{id},                 'TEST1 $test_results->{id} defined' );
 	ok( defined $test_results->{params},             'TEST1 $test_results->{params} defined' );
 	ok( defined $test_results->{creation_time},      'TEST1 $test_results->{creation_time} defined' );
 	ok( defined $test_results->{results},            'TEST1 $test_results->{results} defined' );
 	ok( scalar( @{ $test_results->{results} } ) > 1, 'TEST1 got some results' );
 
+        dies_ok { $engine->get_test_results( { id => $test_id, language => 'fr-FR' } ); }
+        'API get_test_results -> [results] parameter not present (wrong language tag)'; # Should be underscore, not hyphen.
+
+        dies_ok { $engine->get_test_results( { id => $test_id, language => 'zz' } ); }
+        'API get_test_results -> [results] parameter not present (wrong language tag)'; # "zz" is not our configuration file.
 }
 
 run_zonemaster_test_with_backend_API(1);
diff --git a/t/test_DB_backend.pl b/t/test_DB_backend.pl
index 5b351d112..dfa110615 100644
--- a/t/test_DB_backend.pl
+++ b/t/test_DB_backend.pl
@@ -4,6 +4,7 @@
 
 use Test::More;    # see done_testing()
 use JSON::PP;
+use Test::Exception;
 
 my $db_backend = $ARGV[0];
 ok( $db_backend eq 'PostgreSQL' || $db_backend eq 'MySQL' , "Testing a supported database backend: $db_backend" );
@@ -59,12 +60,20 @@ sub run_zonemaster_test_with_backend_API {
         last if ( $progress == 100 );
     }
     ok( $engine->test_progress( $api_test_id ) == 100 , 'API test_progress -> Test finished' );
-    my $test_results = $engine->get_test_results( { id => $api_test_id, language => 'fr-FR' } );
+
+    my $test_results = $engine->get_test_results( { id => $api_test_id, language => 'fr_FR' } );
     ok( defined $test_results->{id} , 'API get_test_results -> [id] paramater present' );
     ok( defined $test_results->{params} , 'API get_test_results -> [params] paramater present' );
     ok( defined $test_results->{creation_time} , 'API get_test_results -> [creation_time] paramater present' );
     ok( defined $test_results->{results} , 'API get_test_results -> [results] paramater present' );
     ok( scalar( @{ $test_results->{results} } ) > 1 , 'API get_test_results -> [results] paramater contains data' );
+
+    dies_ok { $engine->get_test_results( { id => $api_test_id, language => 'fr-FR' } ); }
+    'API get_test_results -> [results] parameter not present (wrong language tag)';
+
+    dies_ok { $engine->get_test_results( { id => $api_test_id, language => 'zz' } ); }
+    'API get_test_results -> [results] parameter not present (wrong language tag)';
+
 }
 
 # add test user
@@ -99,3 +108,4 @@ sub run_zonemaster_test_with_backend_API {
 ok( length($test_history->[0]->{id}) == 16 || length($test_history->[1]->{id}) == 16 );
 
 done_testing();
+

From 3cba60e5ec79bc3c815b4c638c705c15af2b9659 Mon Sep 17 00:00:00 2001
From: Mats Dufberg <mats.dufberg@iis.se>
Date: Tue, 30 Jun 2020 14:22:54 +0200
Subject: [PATCH 23/92] Removed unused GEOLOCATION

---
 docs/Configuration.md    | 4 ----
 share/backend_config.ini | 6 ------
 2 files changed, 10 deletions(-)

diff --git a/docs/Configuration.md b/docs/Configuration.md
index 468c89ae0..4c8250816 100644
--- a/docs/Configuration.md
+++ b/docs/Configuration.md
@@ -28,10 +28,6 @@ PostgreSQL        | `PostgreSQL`
 SQLite            | `SQLite`
 
 
-## GEOLOCATION section
-
-TBD
-
 ## LANGUAGE section
 
 The LANGUAGE section has one key, `locale`.
diff --git a/share/backend_config.ini b/share/backend_config.ini
index f947146b4..9bd18d474 100755
--- a/share/backend_config.ini
+++ b/share/backend_config.ini
@@ -36,9 +36,3 @@ locale = da_DK en_US fr_FR sv_SE
 [PRIVATE PROFILES]
 #example_profile_2=/example/directory/test2_profile.json
 
-[GEOLOCATION]
-# Requires the Geo::IP Perl module to be installed
-#maxmind_isp_db_file  =
-
-# Requires the GeoIP2::Database::Reader Perl module to be installed
-#maxmind_city_db_file =

From 6cfc0c78a867a1310b5db4e2fa50a54996609fc7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Tue, 28 Jul 2020 18:13:33 +0200
Subject: [PATCH 24/92] Reorder existing start_domain_test params to match API
 docs

---
 share/zmb | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/share/zmb b/share/zmb
index 9b9b3bc05..5a292cda8 100755
--- a/share/zmb
+++ b/share/zmb
@@ -114,12 +114,12 @@ sub cmd_version_info {
  Options:
 
     --domain DOMAIN_NAME
+    --ipv4 true|false|null
+    --ipv6 true|false|null
     --nameserver DOMAIN_NAME:IP_ADDRESS
+    --ds-info DS_INFO
     --client-id CLIENT_ID
     --client-version CLIENT_VERSION
-    --ds-info DS_INFO
-    --ipv4 true|false|null
-    --ipv6 true|false|null
 
  DS_INFO is a comma separated list of key-value pairs. The expected pairs are:
 

From 27afb2e6b9af4d6a2bd60a77e3bb1d60d90d9666 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Fri, 31 Jul 2020 17:55:10 +0200
Subject: [PATCH 25/92] Fix usage documentation

---
 share/zmtest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/share/zmtest b/share/zmtest
index e0d7c7be0..8e861016b 100755
--- a/share/zmtest
+++ b/share/zmtest
@@ -12,7 +12,7 @@ usage () {
     echo "Usage: zmtest [OPTIONS] DOMAIN" >&2
     echo >&2
     echo "Options:" >&2
-    echo "  -s URL --server=URL   Zonemaster Backend to query. Default is http://localhost:5000/" >&2
+    echo "  -s URL --server URL   Zonemaster Backend to query. Default is http://localhost:5000/" >&2
     echo "  --noipv4              Run the test without ipv4." >&2
     echo "  --noipv6              Run the test without ipv6." >&2
     exit "${status}"

From 754fe69bf0ad6bfbdeebdbbb4ddfbb0ea87dfe74 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Fri, 31 Jul 2020 17:55:31 +0200
Subject: [PATCH 26/92] Add --lang option to zmtest

---
 share/zmtest | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/share/zmtest b/share/zmtest
index 8e861016b..78fbc61ad 100755
--- a/share/zmtest
+++ b/share/zmtest
@@ -15,6 +15,7 @@ usage () {
     echo "  -s URL --server URL   Zonemaster Backend to query. Default is http://localhost:5000/" >&2
     echo "  --noipv4              Run the test without ipv4." >&2
     echo "  --noipv6              Run the test without ipv6." >&2
+    echo "  --lang LANG           A language tag. Default is en." >&2
     exit "${status}"
 }
 
@@ -44,6 +45,7 @@ while [ $# -gt 0 ] ; do
         -s|--server) server_url="$2"; shift 2;;
         --noipv4) ipv4="false"; shift 1;;
         --noipv6) ipv6="false"; shift 1;;
+        --lang) lang="$1"; shift 1;;
         *) domain="$1" ; shift 1;;
     esac
 done
@@ -67,5 +69,5 @@ do
 done
 
 # Get test results
-zmb "${server_url}" get_test_results --testid "${testid}" --lang en
+zmb "${server_url}" get_test_results --testid "${testid}" --lang "${lang}"
 echo "testid: ${testid}" >&2

From ef0301038e681cebf6aaf84ba0dd1c6681036947 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Fri, 4 Sep 2020 20:31:54 +0200
Subject: [PATCH 27/92] Improved error handling

---
 share/zmtest | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/share/zmtest b/share/zmtest
index 78fbc61ad..93c8f8968 100755
--- a/share/zmtest
+++ b/share/zmtest
@@ -55,6 +55,10 @@ done
 output="$(zmb "${server_url}" start_domain_test --domain "${domain}" --ipv4 "${ipv4}" --ipv6 "${ipv6}")" || exit $?
 testid="$(echo "${output}" | "${JQ}" -r .result)" || exit $?
 
+if echo "${testid}" | grep -qE '[^0-9a-fA-F]' ; then
+    error 1 "start_domain_test did not return a testid: ${testid}"
+fi
+
 # Wait for test to finish
 while true
 do

From 0185c15eb6c55d42d8211809e7dee886688251c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Mon, 27 Apr 2020 18:08:38 +0200
Subject: [PATCH 28/92] Install binary package for Try::Tiny on Debian/Ubuntu

---
 .travis.yml          | 1 +
 Makefile.PL          | 2 +-
 docs/Installation.md | 4 ++--
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index f0eab3b87..fd9864df0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -67,6 +67,7 @@ addons:
     - librole-tiny-perl
     - librouter-simple-perl
     - libstring-shellquote-perl
+    - libtry-tiny-perl
     - starman
 
 before_install:
diff --git a/Makefile.PL b/Makefile.PL
index ed157e9de..a3e24b6b3 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -29,7 +29,7 @@ requires
   'Router::Simple::Declare'     => 0,
   'Starman'                     => 0,
   'String::ShellQuote'          => 0,
-  'Try::Tiny'                   => 0.30,
+  'Try::Tiny'                   => 0.24,
   'Zonemaster::Engine'          => 3.0,
   'Zonemaster::LDNS'            => 2.0,
   ;
diff --git a/docs/Installation.md b/docs/Installation.md
index 88b68a136..b5e16278e 100644
--- a/docs/Installation.md
+++ b/docs/Installation.md
@@ -327,13 +327,13 @@ sudo locale-gen
 Install dependencies available from binary packages:
 
 ```sh
-sudo apt install libclass-method-modifiers-perl libconfig-inifiles-perl libdbd-sqlite3-perl libdbi-perl libfile-sharedir-perl libfile-slurp-perl libhtml-parser-perl libio-captureoutput-perl libjson-pp-perl libjson-rpc-perl liblog-any-adapter-dispatch-perl liblog-any-perl liblog-dispatch-perl libmoose-perl libparallel-forkmanager-perl libplack-perl libplack-middleware-debug-perl librole-tiny-perl librouter-simple-perl libstring-shellquote-perl starman
+sudo apt install libclass-method-modifiers-perl libconfig-inifiles-perl libdbd-sqlite3-perl libdbi-perl libfile-sharedir-perl libfile-slurp-perl libhtml-parser-perl libio-captureoutput-perl libjson-pp-perl libjson-rpc-perl liblog-any-adapter-dispatch-perl liblog-any-perl liblog-dispatch-perl libmoose-perl libparallel-forkmanager-perl libplack-perl libplack-middleware-debug-perl librole-tiny-perl librouter-simple-perl libstring-shellquote-perl libtry-tiny-perl starman
 ```
 
 Install dependencies not available from binary packages:
 
 ```sh
-sudo cpanm Daemon::Control JSON::Validator Net::IP::XS Try::Tiny
+sudo cpanm Daemon::Control JSON::Validator Net::IP::XS
 ```
 
 Install Zonemaster::Backend:

From f0a62aa2db1bef45614d83020959ef99f8962eff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Fri, 4 Sep 2020 22:08:35 +0200
Subject: [PATCH 29/92] Fail earlier with better feedback

---
 t/test_DB_backend.pl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/t/test_DB_backend.pl b/t/test_DB_backend.pl
index dfa110615..104f9a524 100644
--- a/t/test_DB_backend.pl
+++ b/t/test_DB_backend.pl
@@ -6,8 +6,8 @@
 use JSON::PP;
 use Test::Exception;
 
-my $db_backend = $ARGV[0];
-ok( $db_backend eq 'PostgreSQL' || $db_backend eq 'MySQL' , "Testing a supported database backend: $db_backend" );
+my $db_backend = $ARGV[0] // BAIL_OUT( "No database backend specified" );
+( $db_backend eq 'PostgreSQL' || $db_backend eq 'MySQL' ) or BAIL_OUT( "Unsupported database backend: $db_backend" );
 
 my $frontend_params_1 = {
 	client_id      => "$db_backend Unit Test",         # free string

From 00614f4949d06631e81cde7412d8f788b683eb4b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Sun, 6 Sep 2020 21:16:42 +0200
Subject: [PATCH 30/92] Fix warnings

---
 lib/Zonemaster/Backend/RPCAPI.pm    | 6 +++++-
 lib/Zonemaster/Backend/Validator.pm | 6 +++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/lib/Zonemaster/Backend/RPCAPI.pm b/lib/Zonemaster/Backend/RPCAPI.pm
index 2cc22f379..ee8414b8d 100644
--- a/lib/Zonemaster/Backend/RPCAPI.pm
+++ b/lib/Zonemaster/Backend/RPCAPI.pm
@@ -13,7 +13,7 @@ use File::Slurp qw(append_file);
 use Zonemaster::LDNS;
 use Net::IP::XS qw(:PROC);
 use HTML::Entities;
-use JSON::Validator "joi";
+use JSON::Validator::Joi;
 
 # Zonemaster Modules
 use Zonemaster::Engine;
@@ -29,6 +29,10 @@ my $zm_validator = Zonemaster::Backend::Validator->new;
 my %json_schemas;
 my $recursor = Zonemaster::Engine::Recursor->new;
 
+sub joi {
+    return JSON::Validator::Joi->new;
+}
+
 sub new {
     my ( $type, $params ) = @_;
 
diff --git a/lib/Zonemaster/Backend/Validator.pm b/lib/Zonemaster/Backend/Validator.pm
index 157e26bd1..e51e32961 100644
--- a/lib/Zonemaster/Backend/Validator.pm
+++ b/lib/Zonemaster/Backend/Validator.pm
@@ -6,7 +6,11 @@ use strict;
 use warnings;
 use 5.14.2;
 
-use JSON::Validator "joi";
+use JSON::Validator::Joi;
+
+sub joi {
+    return JSON::Validator::Joi->new;
+}
 
 sub new {
     my ( $type ) = @_;

From 99f1170035ab2fb0ff1d6e9eb2e5c2a68947fdf4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Sun, 6 Sep 2020 21:31:36 +0200
Subject: [PATCH 31/92] Fix warning

---
 lib/Zonemaster/Backend/DB.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/Zonemaster/Backend/DB.pm b/lib/Zonemaster/Backend/DB.pm
index bacdbdd3b..ebbba82c1 100644
--- a/lib/Zonemaster/Backend/DB.pm
+++ b/lib/Zonemaster/Backend/DB.pm
@@ -128,7 +128,7 @@ sub process_unfinished_tests {
         }
         else {
             my $result;
-            if ($h->{results} =~ /^\[/) {
+            if ( defined $h->{results} && $h->{results} =~ /^\[/ ) {
                 $result = decode_json( $h->{results} );
             }
             else {

From 9b0e630b750590ba894e43d8bf0c7062d8df1d11 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Mon, 7 Sep 2020 15:23:51 +0200
Subject: [PATCH 32/92] Ignore low-severity messages when checking domain

---
 lib/Zonemaster/Backend/RPCAPI.pm | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/lib/Zonemaster/Backend/RPCAPI.pm b/lib/Zonemaster/Backend/RPCAPI.pm
index 2cc22f379..fba651e57 100644
--- a/lib/Zonemaster/Backend/RPCAPI.pm
+++ b/lib/Zonemaster/Backend/RPCAPI.pm
@@ -17,8 +17,9 @@ use JSON::Validator "joi";
 
 # Zonemaster Modules
 use Zonemaster::Engine;
-use Zonemaster::Engine::Nameserver;
 use Zonemaster::Engine::DNSName;
+use Zonemaster::Engine::Logger::Entry;
+use Zonemaster::Engine::Nameserver;
 use Zonemaster::Engine::Recursor;
 use Zonemaster::Backend;
 use Zonemaster::Backend::Config;
@@ -189,8 +190,10 @@ sub _check_domain {
             );
     }
 
+    my %levels = Zonemaster::Engine::Logger::Entry::levels();
     my @res;
     @res = Zonemaster::Engine::Test::Basic->basic00($dn);
+    @res = grep { $_->numeric_level >= $levels{ERROR} } @res;
     if (@res != 0) {
         return ( $dn, { status => 'nok', message => encode_entities( "$type name or label outside allowed length" ) } );
     }

From 7736e9f64172e9cb9feefd79f83b29999b7ed279 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Mon, 7 Sep 2020 19:29:28 +0200
Subject: [PATCH 33/92] Empty


From f567602afe276f14951142ea028115b07b0df953 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Mon, 7 Sep 2020 19:29:59 +0200
Subject: [PATCH 34/92] Empty


From a28acbfc460e4981effd05035df87be359a2a234 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Mon, 7 Sep 2020 19:36:30 +0200
Subject: [PATCH 35/92] Add note about valid values

---
 share/zmtest | 1 +
 1 file changed, 1 insertion(+)

diff --git a/share/zmtest b/share/zmtest
index 93c8f8968..d40ed0aca 100755
--- a/share/zmtest
+++ b/share/zmtest
@@ -16,6 +16,7 @@ usage () {
     echo "  --noipv4              Run the test without ipv4." >&2
     echo "  --noipv6              Run the test without ipv6." >&2
     echo "  --lang LANG           A language tag. Default is en." >&2
+    echo "                        Valid values are determined by backend_config.ini." >&2
     exit "${status}"
 }
 

From 3f964424268f68e7f1a286fc99a28253f2f89a9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Mon, 7 Sep 2020 19:39:41 +0200
Subject: [PATCH 36/92] Empty


From 53dcaac125c01c5b5dc5af2aecca76f59388657a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Mon, 7 Sep 2020 21:26:12 +0200
Subject: [PATCH 37/92] Install dependencies from EPEL instead of CPAN

---
 docs/Installation.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/Installation.md b/docs/Installation.md
index 88b68a136..09c4fcdd9 100644
--- a/docs/Installation.md
+++ b/docs/Installation.md
@@ -62,13 +62,13 @@ for Zonemaster::Backend, see the [declaration of prerequisites].
 Install dependencies available from binary packages:
 
 ```sh
-sudo yum install perl-Module-Install perl-IO-CaptureOutput perl-String-ShellQuote perl-Net-Server redhat-lsb-core
+sudo yum install perl-Config-IniFiles perl-JSON-RPC perl-Module-Install perl-IO-CaptureOutput perl-Parallel-ForkManager perl-Plack perl-Router-Simple perl-String-ShellQuote perl-Net-Server redhat-lsb-core
 ```
 
 Install dependencies not available from binary packages:
 
 ```sh
-sudo cpanm Class::Method::Modifiers Config::IniFiles Daemon::Control JSON::RPC::Dispatch Net::IP::XS Parallel::ForkManager Plack::Builder Plack::Middleware::Debug Role::Tiny Router::Simple::Declare Starman
+sudo cpanm Class::Method::Modifiers Daemon::Control Net::IP::XS Plack::Middleware::Debug Role::Tiny Starman
 ```
 
 Install Zonemaster::Backend:

From 45db1c8eeafa5f71de4858b41d30bb8f18629a66 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Mon, 7 Sep 2020 22:58:09 +0200
Subject: [PATCH 38/92] Install dependencies from EPEL/PowerTools instead of
 CPAN

---
 docs/Installation.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/Installation.md b/docs/Installation.md
index 09c4fcdd9..0592e1c60 100644
--- a/docs/Installation.md
+++ b/docs/Installation.md
@@ -62,13 +62,13 @@ for Zonemaster::Backend, see the [declaration of prerequisites].
 Install dependencies available from binary packages:
 
 ```sh
-sudo yum install perl-Config-IniFiles perl-JSON-RPC perl-Module-Install perl-IO-CaptureOutput perl-Parallel-ForkManager perl-Plack perl-Router-Simple perl-String-ShellQuote perl-Net-Server redhat-lsb-core
+sudo yum install perl-Class-Method-Modifiers perl-Config-IniFiles perl-JSON-RPC perl-Module-Install perl-IO-CaptureOutput perl-Parallel-ForkManager perl-Plack perl-Router-Simple perl-String-ShellQuote perl-Net-Server perl-Role-Tiny redhat-lsb-core
 ```
 
 Install dependencies not available from binary packages:
 
 ```sh
-sudo cpanm Class::Method::Modifiers Daemon::Control Net::IP::XS Plack::Middleware::Debug Role::Tiny Starman
+sudo cpanm Daemon::Control Net::IP::XS Plack::Middleware::Debug Starman
 ```
 
 Install Zonemaster::Backend:

From c1df3e2e84dd86dcef4369f1a57810bb3f085e0b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Mon, 7 Sep 2020 23:07:00 +0200
Subject: [PATCH 39/92] Remove unused dependency

---
 Makefile.PL          | 1 -
 docs/Installation.md | 4 ++--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/Makefile.PL b/Makefile.PL
index ed157e9de..2b20048eb 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -24,7 +24,6 @@ requires
   'Net::IP::XS'                 => 0.14,
   'Parallel::ForkManager'       => 1.12,
   'Plack::Builder'              => 0,
-  'Plack::Middleware::Debug'    => 0.14,
   'Role::Tiny'                  => 1.001003,
   'Router::Simple::Declare'     => 0,
   'Starman'                     => 0,
diff --git a/docs/Installation.md b/docs/Installation.md
index 0592e1c60..9954c4b9b 100644
--- a/docs/Installation.md
+++ b/docs/Installation.md
@@ -68,7 +68,7 @@ sudo yum install perl-Class-Method-Modifiers perl-Config-IniFiles perl-JSON-RPC
 Install dependencies not available from binary packages:
 
 ```sh
-sudo cpanm Daemon::Control Net::IP::XS Plack::Middleware::Debug Starman
+sudo cpanm Daemon::Control Net::IP::XS Starman
 ```
 
 Install Zonemaster::Backend:
@@ -487,7 +487,7 @@ su -l
 Install dependencies available from binary packages:
 
 ```sh
-pkg install p5-Class-Method-Modifiers p5-Config-IniFiles p5-Daemon-Control p5-DBI p5-File-ShareDir p5-File-Slurp p5-HTML-Parser p5-IO-CaptureOutput p5-JSON-PP p5-JSON-RPC p5-Moose p5-Parallel-ForkManager p5-Plack p5-Plack-Middleware-Debug p5-Role-Tiny p5-Router-Simple p5-Starman p5-String-ShellQuote net-mgmt/p5-Net-IP-XS databases/p5-DBD-SQLite devel/p5-Log-Dispatch devel/p5-Log-Any devel/p5-Log-Any-Adapter-Dispatch
+pkg install p5-Class-Method-Modifiers p5-Config-IniFiles p5-Daemon-Control p5-DBI p5-File-ShareDir p5-File-Slurp p5-HTML-Parser p5-IO-CaptureOutput p5-JSON-PP p5-JSON-RPC p5-Moose p5-Parallel-ForkManager p5-Plack p5-Role-Tiny p5-Router-Simple p5-Starman p5-String-ShellQuote net-mgmt/p5-Net-IP-XS databases/p5-DBD-SQLite devel/p5-Log-Dispatch devel/p5-Log-Any devel/p5-Log-Any-Adapter-Dispatch
 ```
 
 Optionally install Curl (only needed for the post-installation smoke test):

From 49983cd73f63268cef85551de02a2dc579566643 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Mon, 7 Sep 2020 23:21:19 +0200
Subject: [PATCH 40/92] Add quotes per request in review comment

---
 share/zmtest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/share/zmtest b/share/zmtest
index d40ed0aca..790effc58 100755
--- a/share/zmtest
+++ b/share/zmtest
@@ -15,7 +15,7 @@ usage () {
     echo "  -s URL --server URL   Zonemaster Backend to query. Default is http://localhost:5000/" >&2
     echo "  --noipv4              Run the test without ipv4." >&2
     echo "  --noipv6              Run the test without ipv6." >&2
-    echo "  --lang LANG           A language tag. Default is en." >&2
+    echo "  --lang LANG           A language tag. Default is \"en\"." >&2
     echo "                        Valid values are determined by backend_config.ini." >&2
     exit "${status}"
 }

From 73726900a0d5a7e0320b3dff68f9290d2af39a21 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Thu, 17 Sep 2020 16:29:19 +0200
Subject: [PATCH 41/92] Deploy without debug middleware

---
 script/zonemaster_backend_rpcapi.psgi | 1 -
 1 file changed, 1 deletion(-)

diff --git a/script/zonemaster_backend_rpcapi.psgi b/script/zonemaster_backend_rpcapi.psgi
index 751ae8c3c..669212a17 100644
--- a/script/zonemaster_backend_rpcapi.psgi
+++ b/script/zonemaster_backend_rpcapi.psgi
@@ -23,7 +23,6 @@ use Zonemaster::Backend::Config;
 local $| = 1;
 
 builder {
-    enable 'Debug';
     enable sub {
         my $app = shift;
 

From 639b48d449e2cb8ac20a34caed543bff85a841ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Fri, 18 Sep 2020 03:55:48 +0200
Subject: [PATCH 42/92] Remove direct dependency on Net::IP::XS

Net::IP::XS will still be used (via Zonemaster::Engine::Net::IP) if it's
installed.
---
 Makefile.PL                      | 1 -
 docs/Installation.md             | 6 +++---
 lib/Zonemaster/Backend/RPCAPI.pm | 8 +++++---
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/Makefile.PL b/Makefile.PL
index 9b3d2b360..ab39f36b2 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -21,7 +21,6 @@ requires
   'Log::Any::Adapter::Dispatch' => 0,
   'Log::Dispatch'               => 0,
   'Moose'                       => 2.04,
-  'Net::IP::XS'                 => 0.14,
   'Parallel::ForkManager'       => 1.12,
   'Plack::Builder'              => 0,
   'Role::Tiny'                  => 1.001003,
diff --git a/docs/Installation.md b/docs/Installation.md
index 0611a049b..ef55a4ca8 100644
--- a/docs/Installation.md
+++ b/docs/Installation.md
@@ -68,7 +68,7 @@ sudo yum install perl-Class-Method-Modifiers perl-Config-IniFiles perl-JSON-RPC
 Install dependencies not available from binary packages:
 
 ```sh
-sudo cpanm Daemon::Control Net::IP::XS Starman
+sudo cpanm Daemon::Control Starman
 ```
 
 Install Zonemaster::Backend:
@@ -333,7 +333,7 @@ sudo apt install libclass-method-modifiers-perl libconfig-inifiles-perl libdbd-s
 Install dependencies not available from binary packages:
 
 ```sh
-sudo cpanm Daemon::Control JSON::Validator Net::IP::XS
+sudo cpanm Daemon::Control JSON::Validator
 ```
 
 Install Zonemaster::Backend:
@@ -487,7 +487,7 @@ su -l
 Install dependencies available from binary packages:
 
 ```sh
-pkg install p5-Class-Method-Modifiers p5-Config-IniFiles p5-Daemon-Control p5-DBI p5-File-ShareDir p5-File-Slurp p5-HTML-Parser p5-IO-CaptureOutput p5-JSON-PP p5-JSON-RPC p5-Moose p5-Parallel-ForkManager p5-Plack p5-Role-Tiny p5-Router-Simple p5-Starman p5-String-ShellQuote net-mgmt/p5-Net-IP-XS databases/p5-DBD-SQLite devel/p5-Log-Dispatch devel/p5-Log-Any devel/p5-Log-Any-Adapter-Dispatch
+pkg install p5-Class-Method-Modifiers p5-Config-IniFiles p5-Daemon-Control p5-DBI p5-File-ShareDir p5-File-Slurp p5-HTML-Parser p5-IO-CaptureOutput p5-JSON-PP p5-JSON-RPC p5-Moose p5-Parallel-ForkManager p5-Plack p5-Role-Tiny p5-Router-Simple p5-Starman p5-String-ShellQuote databases/p5-DBD-SQLite devel/p5-Log-Dispatch devel/p5-Log-Any devel/p5-Log-Any-Adapter-Dispatch
 ```
 
 Optionally install Curl (only needed for the post-installation smoke test):
diff --git a/lib/Zonemaster/Backend/RPCAPI.pm b/lib/Zonemaster/Backend/RPCAPI.pm
index b7bf27ff1..5cc918e30 100644
--- a/lib/Zonemaster/Backend/RPCAPI.pm
+++ b/lib/Zonemaster/Backend/RPCAPI.pm
@@ -10,16 +10,16 @@ use DBI qw(:utils);
 use Digest::MD5 qw(md5_hex);
 use String::ShellQuote;
 use File::Slurp qw(append_file);
-use Zonemaster::LDNS;
-use Net::IP::XS qw(:PROC);
 use HTML::Entities;
 use JSON::Validator::Joi;
 
 # Zonemaster Modules
+use Zonemaster::LDNS;
 use Zonemaster::Engine;
 use Zonemaster::Engine::DNSName;
 use Zonemaster::Engine::Logger::Entry;
 use Zonemaster::Engine::Nameserver;
+use Zonemaster::Engine::Net::IP;
 use Zonemaster::Engine::Recursor;
 use Zonemaster::Backend;
 use Zonemaster::Backend::Config;
@@ -271,7 +271,9 @@ sub validate_syntax {
                 unless( !$ns_ip->{ip} || $ns_ip->{ip} =~ /^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$/ || $ns_ip->{ip} =~ /^([0-9A-Fa-f]{1,4}:[0-9A-Fa-f:]{1,}(:[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})?)|([0-9A-Fa-f]{1,4}::[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})$/);
 
             return { status => 'nok', message => encode_entities( "Invalid IP address: [$ns_ip->{ip}]" ) }
-              unless ( !$ns_ip->{ip} || ip_is_ipv4( $ns_ip->{ip} ) || ip_is_ipv6( $ns_ip->{ip} ) );
+              unless ( !$ns_ip->{ip}
+                || Zonemaster::Engine::Net::IP::ip_is_ipv4( $ns_ip->{ip} )
+                || Zonemaster::Engine::Net::IP::ip_is_ipv6( $ns_ip->{ip} ) );
         }
 
         foreach my $ds_digest ( @{ $syntax_input->{ds_info} } ) {

From 64f8eefd35b58dcd7668a4c3226799ae9096183f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Sun, 20 Sep 2020 09:28:05 +0200
Subject: [PATCH 43/92] Support logging to STDOUT

---
 script/zonemaster_backend_testagent | 73 +++++++++++++++++++++++------
 share/zm-rpcapi.lsb                 |  4 +-
 share/zm-testagent.lsb              |  6 +--
 3 files changed, 63 insertions(+), 20 deletions(-)

diff --git a/script/zonemaster_backend_testagent b/script/zonemaster_backend_testagent
index f7f8e9c2b..4f32bebb5 100755
--- a/script/zonemaster_backend_testagent
+++ b/script/zonemaster_backend_testagent
@@ -61,22 +61,51 @@ $loglevel = lc $loglevel;
 
 $loglevel =~ /^(?:trace|debug|info|inform|notice|warning|warn|error|err|critical|crit|fatal|alert|emergency)$/ or die "Error: Unrecognized --loglevel $loglevel\n";
 
-print STDERR "Logging to $logfile\n";
-
-{
-    my $dispatcher = Log::Dispatch->new(outputs => [
-        [
-            'File',
-            min_level => $loglevel,
-            filename => $logfile,
-            mode => '>>',
-            callbacks => sub {
-                my %args = @_;
-                $args{message} = sprintf "%s [%d] %s - %s\n", strftime("%FT%TZ", gmtime), $PID, uc $args{level}, $args{message};
-            },
+# Returns a Log::Dispatch object logging to STDOUT
+#
+# This procedure duplicates the STDOUT file descriptor to make sure that it keeps
+# logging to the same place even if STDOUT is later redirected.
+sub log_dispatcher_dup_stdout {
+    my $min_level = shift;
+
+    open( my $fd, '>&', \*STDOUT ) or die "Can't dup STDOUT: $!";
+    my $handle = IO::Handle->new_from_fd( $fd, "w" ) or die "Can't fdopen duplicated STDOUT: $!";
+    $handle->autoflush(1);
+
+    return Log::Dispatch->new(
+        outputs => [
+            [
+                'Handle',
+                handle    => $handle,
+                min_level => $min_level,
+                callbacks => sub {
+                    my %args = @_;
+                    $args{message} = sprintf "%s: %s\n", uc $args{level}, $args{message};
+                },
+            ],
         ]
-    ]);
-    Log::Any::Adapter->set( 'Dispatch', dispatcher => $dispatcher );
+    );
+}
+
+# Returns a Log::Dispatch object logging to a file
+sub log_dispatcher_file {
+    my $min_level = shift;
+    my $log_file  = shift;
+
+    return Log::Dispatch->new(
+        outputs => [
+            [
+                'File',
+                filename  => $log_file,
+                mode      => '>>',
+                min_level => $min_level,
+                callbacks => sub {
+                    my %args = @_;
+                    $args{message} = sprintf "%s [%d] %s - %s\n", strftime( "%FT%TZ", gmtime ), $PID, uc $args{level}, $args{message};
+                },
+            ],
+        ]
+    );
 }
 
 ###
@@ -136,6 +165,18 @@ sub main {
 # Make sure the environment is alright before forking
 my $initial_config;
 eval {
+    # Initialize logging
+    my $dispatcher;
+    if ( $logfile eq '-' ) {
+        $dispatcher = log_dispatcher_dup_stdout( $loglevel );
+    }
+    else {
+        $dispatcher = log_dispatcher_file( $loglevel, $logfile );
+        print STDERR "zonemaster-testagent logging to file $logfile\n";
+    }
+    Log::Any::Adapter->set( 'Dispatch', dispatcher => $dispatcher );
+
+    # Make sure we can load the configuration file
     $log->debug("Starting pre-flight check");
     $initial_config = Zonemaster::Backend::Config->load_config();
 
@@ -216,6 +257,8 @@ The location of the PID file to use.
 
 The location of the log file to use.
 
+When FILE is -, the log is written to standard output.
+
 =item B<--loglevel=LEVEL>
 
 The location of the log level to use.
diff --git a/share/zm-rpcapi.lsb b/share/zm-rpcapi.lsb
index 1ae491f2e..5527f4661 100644
--- a/share/zm-rpcapi.lsb
+++ b/share/zm-rpcapi.lsb
@@ -14,8 +14,8 @@
 ### END INIT INFO
 
 BINDIR=${ZM_BACKEND_BINDIR:-/usr/local/bin}
-LOGFILE=${ZM_BACKEND_LOGDIR:-/var/log/zonemaster}/zm-rpcapi.log
-PIDFILE=${ZM_BACKEND_PIDDIR:-/var/run/zonemaster}/zm-rpcapi.pid
+LOGFILE=${ZM_BACKEND_LOGFILE:-/var/log/zonemaster/zm-rpcapi.log}
+PIDFILE=${ZM_BACKEND_PIDFILE:-/var/run/zonemaster/zm-rpcapi.pid}
 LISTENIP=${ZM_BACKEND_LISTENIP:-127.0.0.1}
 LISTENPORT=${ZM_BACKEND_LISTENPORT:-5000}
 USER=${ZM_BACKEND_USER:-zonemaster}
diff --git a/share/zm-testagent.lsb b/share/zm-testagent.lsb
index 4f0e8d6df..6efd7841d 100644
--- a/share/zm-testagent.lsb
+++ b/share/zm-testagent.lsb
@@ -14,9 +14,9 @@
 ### END INIT INFO
 
 BINDIR=${ZM_BACKEND_BINDIR:-/usr/local/bin}
-LOGFILE=${ZM_BACKEND_LOGDIR:-/var/log/zonemaster}/zm-testagent.log
-OUTFILE=${ZM_BACKEND_LOGDIR:-/var/log/zonemaster}/zm-testagent.out
-PIDFILE=${ZM_BACKEND_PIDDIR:-/var/run/zonemaster}/zm-testagent.pid
+LOGFILE=${ZM_BACKEND_LOGFILE:-/var/log/zonemaster/zm-testagent.log}
+OUTFILE=${ZM_BACKEND_OUTFILE:-/var/log/zonemaster/zm-testagent.out}
+PIDFILE=${ZM_BACKEND_PIDFILE:-/var/run/zonemaster/zm-testagent.pid}
 USER=${ZM_BACKEND_USER:-zonemaster}
 GROUP=${ZM_BACKEND_GROUP:-zonemaster}
 

From fd0bcf3adce6c19601c8c3953ea91d2f3671443b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Mon, 21 Sep 2020 18:46:23 +0200
Subject: [PATCH 44/92] Re-enable Perl 5.16 in Travis

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index fd9864df0..f74f9081a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -15,7 +15,7 @@ perl:
     - "5.26"
     - "5.24"
     - "5.22"
-      # Perl 5.16 is temporarily excluded because Travis fails
+    - "5.16"
     
 addons:
   apt:

From a6c4f855285f07c576d5f956a5ca7fbba156d3d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Mon, 21 Sep 2020 18:46:38 +0200
Subject: [PATCH 45/92] Enable Perl 5.32 in Travis

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index f74f9081a..02c6fed4d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -10,6 +10,7 @@ env:
 language: perl
 
 perl:
+    - "5.32"
     - "5.30"
     - "5.28"
     - "5.26"

From 7625973bfd32135502e0ebdc30a358faaf2ec551 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Mon, 21 Sep 2020 18:59:06 +0200
Subject: [PATCH 46/92] Limit the number of test environments

---
 .travis.yml | 41 ++++++++++++++++++++++++-----------------
 1 file changed, 24 insertions(+), 17 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 02c6fed4d..507436983 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,22 +1,29 @@
-services: 
-  - mysql
-  - postgresql
-
-env:
-    - TARGET=MySQL ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_mysql_backend_config.ini
-    - TARGET=PostgreSQL ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_postgresql_backend_config.ini
-    - TARGET=SQLite ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_sqlite_backend_config.ini
-    
 language: perl
 
-perl:
-    - "5.32"
-    - "5.30"
-    - "5.28"
-    - "5.26"
-    - "5.24"
-    - "5.22"
-    - "5.16"
+jobs:
+  include:
+  # Cover MySQL and PostgreSQL with the latest supported Perl version
+  - perl: "5.32"
+    env: TARGET=MySQL ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_mysql_backend_config.ini
+    services: mysql
+  - perl: "5.32"
+    env: TARGET=PostgreSQL ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_postgresql_backend_config.ini
+    services: postgresql
+  # Cover all Perl versions with SQLite
+  - perl: "5.32"
+    env: TARGET=SQLite ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_sqlite_backend_config.ini
+  - perl: "5.30"
+    env: TARGET=SQLite ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_sqlite_backend_config.ini
+  - perl: "5.28"
+    env: TARGET=SQLite ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_sqlite_backend_config.ini
+  - perl: "5.26"
+    env: TARGET=SQLite ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_sqlite_backend_config.ini
+  - perl: "5.24"
+    env: TARGET=SQLite ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_sqlite_backend_config.ini
+  - perl: "5.22"
+    env: TARGET=SQLite ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_sqlite_backend_config.ini
+  - perl: "5.16"
+    env: TARGET=SQLite ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_sqlite_backend_config.ini
     
 addons:
   apt:

From 0b89af8df476de9da4f2720f9d036ea98fff79e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Tue, 22 Sep 2020 02:35:14 +0200
Subject: [PATCH 47/92] Fix broken zmtest --lang option

---
 share/zmtest | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/share/zmtest b/share/zmtest
index 790effc58..22b9c9d8e 100755
--- a/share/zmtest
+++ b/share/zmtest
@@ -41,12 +41,13 @@ domain=""
 server_url="http://localhost:5000/"
 ipv4="true"
 ipv6="true"
+lang="en"
 while [ $# -gt 0 ] ; do
     case "$1" in
         -s|--server) server_url="$2"; shift 2;;
         --noipv4) ipv4="false"; shift 1;;
         --noipv6) ipv6="false"; shift 1;;
-        --lang) lang="$1"; shift 1;;
+        --lang) lang="$2"; shift 2;;
         *) domain="$1" ; shift 1;;
     esac
 done

From 0e4aae092c1f68248401b4ec3b06ca1862ce3ea4 Mon Sep 17 00:00:00 2001
From: Mats Dufberg <mats.dufberg@iis.se>
Date: Wed, 23 Sep 2020 10:02:42 +0200
Subject: [PATCH 48/92] Removed unused scripts resolving issue #597.

---
 MANIFEST                                      |   3 -
 script/crontab_job_runner/execute_tests.pl    | 110 ------------------
 .../execute_zonemaster_P10.pl                 |  37 ------
 .../execute_zonemaster_P5.pl                  |  36 ------
 4 files changed, 186 deletions(-)
 delete mode 100644 script/crontab_job_runner/execute_tests.pl
 delete mode 100644 script/crontab_job_runner/execute_zonemaster_P10.pl
 delete mode 100644 script/crontab_job_runner/execute_zonemaster_P5.pl

diff --git a/MANIFEST b/MANIFEST
index 3f2433b3b..76af1411d 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -46,9 +46,6 @@ README.md
 script/add-test.pl
 script/create_db_mysql.pl
 script/create_db_postgresql_9.3.pl
-script/crontab_job_runner/execute_tests.pl
-script/crontab_job_runner/execute_zonemaster_P10.pl
-script/crontab_job_runner/execute_zonemaster_P5.pl
 script/zonemaster_backend_rpcapi.psgi
 script/zonemaster_backend_testagent
 share/backend_config.ini
diff --git a/script/crontab_job_runner/execute_tests.pl b/script/crontab_job_runner/execute_tests.pl
deleted file mode 100644
index 4aeb1c9db..000000000
--- a/script/crontab_job_runner/execute_tests.pl
+++ /dev/null
@@ -1,110 +0,0 @@
-use strict;
-use warnings;
-use utf8;
-use 5.10.1;
-
-use Data::Dumper;
-use DBI qw(:utils);
-use IO::CaptureOutput qw/capture_exec/;
-use POSIX;
-use Time::HiRes;
-use Proc::ProcessTable;
-
-local $| = 1;
-
-use Zonemaster::Backend::Config;
-
-use FindBin qw($RealScript $Script $RealBin $Bin);
-FindBin::again();
-##################################################################
-my $PROJECT_NAME = "zonemaster-backend";
-
-my $SCRITP_DIR = __FILE__;
-$SCRITP_DIR = $Bin unless ($SCRITP_DIR =~ /^\//);
-
-#warn "SCRITP_DIR:$SCRITP_DIR\n";
-#warn "SCRITP_DIR:$SCRITP_DIR\n";
-#warn "RealScript:$RealScript\n";
-#warn "Script:$Script\n";
-#warn "RealBin:$RealBin\n";
-#warn "Bin:$Bin\n";
-#warn "__PACKAGE__:".__PACKAGE__;
-#warn "__FILE__:".__FILE__;
-
-my ($PROD_DIR) = ($SCRITP_DIR =~ /(.*?\/)$PROJECT_NAME/);
-#warn "PROD_DIR:$PROD_DIR\n";
-
-my $PROJECT_BASE_DIR = $PROD_DIR.$PROJECT_NAME."/";
-#warn "PROJECT_BASE_DIR:$PROJECT_BASE_DIR\n";
-unshift(@INC, $PROJECT_BASE_DIR);
-###################################################################
-
-my $JOB_RUNNER_DIR = $PROD_DIR."zonemaster-backend/script/crontab_job_runner/";
-my $LOG_DIR = Zonemaster::Backend::Config->load_config()->LogDir();
-my $perl_command = Zonemaster::Backend::Config->load_config()->PerlInterpreter();
-my $polling_interval = Zonemaster::Backend::Config->load_config()->PollingInterval();
-my $zonemaster_timeout_interval = Zonemaster::Backend::Config->load_config()->MaxZonemasterExecutionTime();
-my $frontend_slots = Zonemaster::Backend::Config->load_config()->NumberOfProcessesForFrontendTesting();
-my $batch_slots = Zonemaster::Backend::Config->load_config()->NumberOfProcessesForBatchTesting();
-
-my $connection_string = Zonemaster::Backend::Config->load_config()->DB_connection_string();
-my $dbh = DBI->connect($connection_string, Zonemaster::Backend::Config->load_config()->DB_user(), Zonemaster::Backend::Config->load_config()->DB_password(), {RaiseError => 1, AutoCommit => 1});
-
-
-sub clean_hung_processes {
-	my $t = new Proc::ProcessTable;
-
-	foreach my $p (@{$t->table}) {
-		if (($p->cmndline =~ /execute_zonemaster_P10\.pl/ || $p->cmndline =~ /execute_zonemaster_P5\.pl/) && $p->cmndline !~ /sh -c/) {
-			if (time() - $p->start > $zonemaster_timeout_interval) {
-				say "Killing hung Zonemaster test process: [".$p->cmndline."]";
-				$p->kill(9);
-			}
-		}
-	}
-}
-
-sub can_start_new_worker {
-	my ($priority, $test_id) = @_;
-	my $result = 0;
-	
-	my @nb_instances = split(/\n+/, `ps -ef | grep "execute_zonemaster_P$priority.pl" | grep -v "sh -c" | grep -v grep | grep -v tail`);
-	my @same_test_id = split(/\n+/, `ps -ef | grep "execute_zonemaster_P$priority.pl $test_id " | grep -v "sh -c" | grep -v grep | grep -v tail`);
-	
-	my $max_slots = 0;
-	if ($priority == 5) {
-		$max_slots = $batch_slots;
-	}
-	elsif ($priority == 10) {
-		$max_slots = $frontend_slots;
-	}
-	
-	$result = 1 if (scalar @nb_instances < $max_slots && !@same_test_id);
-}
-
-sub process_jobs {
-	my ($priority, $start_time) = @_;
-
-	my $query = "SELECT id FROM test_results WHERE progress=0 AND priority=$priority ORDER BY id LIMIT 10";
-	my $sth1 = $dbh->prepare($query);
-	$sth1->execute;
-	while (my $h = $sth1->fetchrow_hashref) {
-		if (can_start_new_worker($priority, $h->{id})) {
-			my $command = "$perl_command $JOB_RUNNER_DIR/execute_zonemaster_P$priority.pl $h->{id} > $LOG_DIR/execute_zonemaster_P$priority"."_$h->{id}_$start_time.log 2>&1 &";
-			say $command;
-			system($command);
-		}
-	}
-	$sth1->finish();
-}
-
-my $start_time = time();
-do {
-	clean_hung_processes();
-	process_jobs(10, $start_time);
-	process_jobs(5, $start_time);
-	say '----------------------- '.strftime("%F %T", localtime()).' ------------------------';
-	Time::HiRes::sleep($polling_interval);
-} while (time() - $start_time < (15*60 - 15));
-
-say "WORKED FOR 15 minutes LEAVING";
diff --git a/script/crontab_job_runner/execute_zonemaster_P10.pl b/script/crontab_job_runner/execute_zonemaster_P10.pl
deleted file mode 100644
index 13d116aeb..000000000
--- a/script/crontab_job_runner/execute_zonemaster_P10.pl
+++ /dev/null
@@ -1,37 +0,0 @@
-use strict;
-use warnings;
-use utf8;
-use 5.10.1;
-
-use Data::Dumper;
-
-use FindBin qw($RealScript $Script $RealBin $Bin);
-FindBin::again();
-##################################################################
-my $PROJECT_NAME = "zonemaster-backend";
-
-my $SCRITP_DIR = __FILE__;
-$SCRITP_DIR = $Bin unless ($SCRITP_DIR =~ /^\//);
-
-#warn "SCRITP_DIR:$SCRITP_DIR\n";
-#warn "SCRITP_DIR:$SCRITP_DIR\n";
-#warn "RealScript:$RealScript\n";
-#warn "Script:$Script\n";
-#warn "RealBin:$RealBin\n";
-#warn "Bin:$Bin\n";
-#warn "__PACKAGE__:".__PACKAGE__;
-#warn "__FILE__:".__FILE__;
-
-my ($PROD_DIR) = ($SCRITP_DIR =~ /(.*?\/)$PROJECT_NAME/);
-#warn "PROD_DIR:$PROD_DIR\n";
-
-my $PROJECT_BASE_DIR = $PROD_DIR.$PROJECT_NAME."/";
-#warn "PROJECT_BASE_DIR:$PROJECT_BASE_DIR\n";
-unshift(@INC, $PROJECT_BASE_DIR);
-##################################################################
-
-unshift(@INC, $PROD_DIR."zonemaster-backend/lib/") unless $INC{$PROD_DIR."zonemaster-backend/lib/"};
-require Zonemaster::Backend::TestAgent;
-
-Zonemaster::Backend::TestAgent->new()->run($ARGV[0]);
-
diff --git a/script/crontab_job_runner/execute_zonemaster_P5.pl b/script/crontab_job_runner/execute_zonemaster_P5.pl
deleted file mode 100644
index 67d7f5c84..000000000
--- a/script/crontab_job_runner/execute_zonemaster_P5.pl
+++ /dev/null
@@ -1,36 +0,0 @@
-use strict;
-use warnings;
-use utf8;
-use 5.10.1;
-
-use Data::Dumper;
-
-use FindBin qw($RealScript $Script $RealBin $Bin);
-FindBin::again();
-##################################################################
-my $PROJECT_NAME = "zonemaster-backend";
-
-my $SCRITP_DIR = __FILE__;
-$SCRITP_DIR = $Bin unless ($SCRITP_DIR =~ /^\//);
-
-#warn "SCRITP_DIR:$SCRITP_DIR\n";
-#warn "SCRITP_DIR:$SCRITP_DIR\n";
-#warn "RealScript:$RealScript\n";
-#warn "Script:$Script\n";
-#warn "RealBin:$RealBin\n";
-#warn "Bin:$Bin\n";
-#warn "__PACKAGE__:".__PACKAGE__;
-#warn "__FILE__:".__FILE__;
-
-my ($PROD_DIR) = ($SCRITP_DIR =~ /(.*?\/)$PROJECT_NAME/);
-#warn "PROD_DIR:$PROD_DIR\n";
-
-my $PROJECT_BASE_DIR = $PROD_DIR.$PROJECT_NAME."/";
-#warn "PROJECT_BASE_DIR:$PROJECT_BASE_DIR\n";
-unshift(@INC, $PROJECT_BASE_DIR);
-##################################################################
-
-unshift(@INC, $PROD_DIR."zonemaster-backend/lib/") unless $INC{$PROD_DIR."zonemaster-backend/lib/"};
-require Zonemaster::Backend::TestAgent;
-
-Zonemaster::Backend::TestAgent->new()->run($ARGV[0]);

From e3df1055c911586fbd63f352713849143e0d6440 Mon Sep 17 00:00:00 2001
From: Mats Dufberg <mats.dufberg@iis.se>
Date: Wed, 23 Sep 2020 10:05:33 +0200
Subject: [PATCH 49/92] Remove unused dependency resolving issue #598.

---
 Makefile.PL          | 1 -
 docs/Installation.md | 6 +++---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/Makefile.PL b/Makefile.PL
index ab39f36b2..6c6f490ea 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -13,7 +13,6 @@ requires
   'File::ShareDir'              => 0,
   'File::Slurp'                 => 0,
   'HTML::Entities'              => 0,
-  'IO::CaptureOutput'           => 0,
   'JSON::PP'                    => 0,
   'JSON::RPC'                   => 1.01,
   'JSON::Validator'             => 3.12,
diff --git a/docs/Installation.md b/docs/Installation.md
index ef55a4ca8..1388b400a 100644
--- a/docs/Installation.md
+++ b/docs/Installation.md
@@ -62,7 +62,7 @@ for Zonemaster::Backend, see the [declaration of prerequisites].
 Install dependencies available from binary packages:
 
 ```sh
-sudo yum install perl-Class-Method-Modifiers perl-Config-IniFiles perl-JSON-RPC perl-Module-Install perl-IO-CaptureOutput perl-Parallel-ForkManager perl-Plack perl-Router-Simple perl-String-ShellQuote perl-Net-Server perl-Role-Tiny redhat-lsb-core
+sudo yum install perl-Class-Method-Modifiers perl-Config-IniFiles perl-JSON-RPC perl-Module-Install perl-Parallel-ForkManager perl-Plack perl-Router-Simple perl-String-ShellQuote perl-Net-Server perl-Role-Tiny redhat-lsb-core
 ```
 
 Install dependencies not available from binary packages:
@@ -327,7 +327,7 @@ sudo locale-gen
 Install dependencies available from binary packages:
 
 ```sh
-sudo apt install libclass-method-modifiers-perl libconfig-inifiles-perl libdbd-sqlite3-perl libdbi-perl libfile-sharedir-perl libfile-slurp-perl libhtml-parser-perl libio-captureoutput-perl libjson-pp-perl libjson-rpc-perl liblog-any-adapter-dispatch-perl liblog-any-perl liblog-dispatch-perl libmoose-perl libparallel-forkmanager-perl libplack-perl libplack-middleware-debug-perl librole-tiny-perl librouter-simple-perl libstring-shellquote-perl libtry-tiny-perl starman
+sudo apt install libclass-method-modifiers-perl libconfig-inifiles-perl libdbd-sqlite3-perl libdbi-perl libfile-sharedir-perl libfile-slurp-perl libhtml-parser-perl libjson-pp-perl libjson-rpc-perl liblog-any-adapter-dispatch-perl liblog-any-perl liblog-dispatch-perl libmoose-perl libparallel-forkmanager-perl libplack-perl libplack-middleware-debug-perl librole-tiny-perl librouter-simple-perl libstring-shellquote-perl libtry-tiny-perl starman
 ```
 
 Install dependencies not available from binary packages:
@@ -487,7 +487,7 @@ su -l
 Install dependencies available from binary packages:
 
 ```sh
-pkg install p5-Class-Method-Modifiers p5-Config-IniFiles p5-Daemon-Control p5-DBI p5-File-ShareDir p5-File-Slurp p5-HTML-Parser p5-IO-CaptureOutput p5-JSON-PP p5-JSON-RPC p5-Moose p5-Parallel-ForkManager p5-Plack p5-Role-Tiny p5-Router-Simple p5-Starman p5-String-ShellQuote databases/p5-DBD-SQLite devel/p5-Log-Dispatch devel/p5-Log-Any devel/p5-Log-Any-Adapter-Dispatch
+pkg install p5-Class-Method-Modifiers p5-Config-IniFiles p5-Daemon-Control p5-DBI p5-File-ShareDir p5-File-Slurp p5-HTML-Parser p5-JSON-PP p5-JSON-RPC p5-Moose p5-Parallel-ForkManager p5-Plack p5-Role-Tiny p5-Router-Simple p5-Starman p5-String-ShellQuote databases/p5-DBD-SQLite devel/p5-Log-Dispatch devel/p5-Log-Any devel/p5-Log-Any-Adapter-Dispatch
 ```
 
 Optionally install Curl (only needed for the post-installation smoke test):

From 99558d81acdbb586b508bc385290b0543087ca8a Mon Sep 17 00:00:00 2001
From: Mats Dufberg <mats.dufberg@iis.se>
Date: Wed, 23 Sep 2020 11:25:45 +0200
Subject: [PATCH 50/92] Added nb_NO or nb to documentation and configuration.

---
 docs/API.md                                | 1 +
 docs/Configuration.md                      | 5 ++++-
 docs/Installation.md                       | 5 +++--
 share/backend_config.ini                   | 2 +-
 share/travis_mysql_backend_config.ini      | 2 +-
 share/travis_postgresql_backend_config.ini | 2 +-
 share/travis_sqlite_backend_config.ini     | 2 +-
 7 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/docs/API.md b/docs/API.md
index b91d8d771..7922af5ec 100644
--- a/docs/API.md
+++ b/docs/API.md
@@ -286,6 +286,7 @@ A default installation will accept the following `language tags`:
 * `da` or `da_DK` for Danish language.
 * `en` or `en_US` for English language.
 * `fr` or `fr_FR` for French language.
+* `nb` or `nb_NO` for Norwegian language.
 * `sv` or `sv_SE` for Swedish language.
 
 
diff --git a/docs/Configuration.md b/docs/Configuration.md
index 4c8250816..8dd86fb83 100644
--- a/docs/Configuration.md
+++ b/docs/Configuration.md
@@ -82,6 +82,7 @@ Language | Locale tag value   | Locale value used
 Danish   | da_DK              | da_DK.UTF-8
 English  | en_US              | en_US.UTF-8
 French   | fr_FR              | fr_FR.UTF-8
+Norwegian| nb_NO              | nb_NO.UTF-8
 Swedish  | sv_SE              | sv_SE.UTF-8
 
 The following `language tags` are generated:
@@ -91,6 +92,8 @@ The following `language tags` are generated:
 * en_US
 * fr
 * fr_FR
+* nb
+* nb_NO
 * sv
 * sv_SE
 
@@ -99,7 +102,7 @@ It is an error to repeat the same `locale tag`.
 Setting in the default configuration file:
 
 ```
-locale = da_DK en_US fr_FR sv_SE
+locale = da_DK en_US fr_FR nb_NO sv_SE
 ```
 
 Each locale set in the configuration file, including the implied
diff --git a/docs/Installation.md b/docs/Installation.md
index ef55a4ca8..5d3877566 100644
--- a/docs/Installation.md
+++ b/docs/Installation.md
@@ -317,10 +317,11 @@ sudo apt install curl
 Install required locales:
 
 ```sh
+locale -a | grep da_DK.utf8 || echo da_DK.UTF-8 UTF-8 | sudo tee -a /etc/locale.gen
 locale -a | grep en_US.utf8 || echo en_US.UTF-8 UTF-8 | sudo tee -a /etc/locale.gen
-locale -a | grep sv_SE.utf8 || echo sv_SE.UTF-8 UTF-8 | sudo tee -a /etc/locale.gen
 locale -a | grep fr_FR.utf8 || echo fr_FR.UTF-8 UTF-8 | sudo tee -a /etc/locale.gen
-locale -a | grep da_DK.utf8 || echo da_DK.UTF-8 UTF-8 | sudo tee -a /etc/locale.gen
+locale -a | grep nb_NO.utf8 || echo nb_NO.UTF-8 UTF-8 | sudo tee -a /etc/locale.gen
+locale -a | grep sv_SE.utf8 || echo sv_SE.UTF-8 UTF-8 | sudo tee -a /etc/locale.gen
 sudo locale-gen
 ```
 
diff --git a/share/backend_config.ini b/share/backend_config.ini
index 9bd18d474..05be3f0df 100755
--- a/share/backend_config.ini
+++ b/share/backend_config.ini
@@ -27,7 +27,7 @@ number_of_processes_for_batch_testing    = 20
 #maximal_number_of_retries=3
 
 [LANGUAGE]
-locale = da_DK en_US fr_FR sv_SE
+locale = da_DK en_US fr_FR nb_NO sv_SE
 
 [PUBLIC PROFILES]
 #example_profile_1=/example/directory/test1_profile.json
diff --git a/share/travis_mysql_backend_config.ini b/share/travis_mysql_backend_config.ini
index 7f60f2cf6..3ae50b45f 100644
--- a/share/travis_mysql_backend_config.ini
+++ b/share/travis_mysql_backend_config.ini
@@ -29,5 +29,5 @@ number_of_processes_for_batch_testing=20
 force_hash_id_use_in_API_starting_from_id=1
 
 [LANGUAGE]
-locale = da_DK en_US fr_FR sv_SE
+locale = da_DK en_US fr_FR nb_NO sv_SE
 
diff --git a/share/travis_postgresql_backend_config.ini b/share/travis_postgresql_backend_config.ini
index 5c181dd6e..51c62eb63 100644
--- a/share/travis_postgresql_backend_config.ini
+++ b/share/travis_postgresql_backend_config.ini
@@ -29,5 +29,5 @@ number_of_professes_for_batch_testing=20
 force_hash_id_use_in_API_starting_from_id=1
 
 [LANGUAGE]
-locale = da_DK en_US fr_FR sv_SE
+locale = da_DK en_US fr_FR nb_NO sv_SE
 
diff --git a/share/travis_sqlite_backend_config.ini b/share/travis_sqlite_backend_config.ini
index 90d3fb2d8..832fd0f8c 100644
--- a/share/travis_sqlite_backend_config.ini
+++ b/share/travis_sqlite_backend_config.ini
@@ -22,5 +22,5 @@ number_of_processes_for_batch_testing=20
 #seconds
 
 [LANGUAGE]
-locale = da_DK en_US fr_FR sv_SE
+locale = da_DK en_US fr_FR nb_NO sv_SE
 

From dccb19e46c66edb10938bb23e5926059fc945966 Mon Sep 17 00:00:00 2001
From: Alexandre Pion <pion@afnic.fr>
Date: Wed, 30 Sep 2020 17:14:59 +0200
Subject: [PATCH 51/92] Fix API documentation for Client id and Client version

---
 docs/API.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/API.md b/docs/API.md
index 7922af5ec..050fe77a4 100644
--- a/docs/API.md
+++ b/docs/API.md
@@ -98,7 +98,7 @@ The unique id of a *batch*.
 Basic data type: string
 
 A string of alphanumerics, hyphens, underscores, pluses (`+`), tildes (`~`),
-full stops (`.`), colons (`:`) and spaces (` `), of at least 1 and at most 512
+full stops (`.`), colons (`:`) and spaces (` `), of at least 1 and at most 50
 characters.
 I.e. a string matching `/^[a-zA-Z0-9-+~_.: ]{1,50}$/`.
 
@@ -111,7 +111,7 @@ Used for monitoring which client (GUI) uses the API.
 Basic data type: string
 
 A string of alphanumerics, hyphens, pluses, tildes, underscores, full stops,
-colons and spaces, of at least 1 and at most 512 characters.
+colons and spaces, of at least 1 and at most 50 characters.
 I.e. a string matching `/^[a-zA-Z0-9-+~_.: ]{1,50}$/`.
 
 Represents the version of the client.

From 6afdcbedf1e3f221ee8b437798c489f9481d5ed6 Mon Sep 17 00:00:00 2001
From: Alexandre Pion <pion@afnic.fr>
Date: Wed, 30 Sep 2020 17:51:27 +0200
Subject: [PATCH 52/92] Fix API documentation for Username

---
 docs/API.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/API.md b/docs/API.md
index 050fe77a4..4485099bd 100644
--- a/docs/API.md
+++ b/docs/API.md
@@ -303,7 +303,7 @@ Basic data type: string
 
 A string of alphanumerics, dashes, full stops and at-signs, of at least 1 and at
 most 50 characters.
-I.e. a string matching `/^[a-zA-Z0-9]{1,50}$/`.
+I.e. a string matching `/^[a-zA-Z0-9-.@]{1,50}$/`.
 
 Represents the name of an authenticated account (see *[Privilege levels]*)
 

From e7c92815d1061c5291e3ecba3314a2dfe38ac0c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Wed, 30 Sep 2020 17:15:37 +0200
Subject: [PATCH 53/92] Attach adapter to Log::Any in RPCAPI

---
 script/zonemaster_backend_rpcapi.psgi | 28 +++++++++++++++++++++++----
 1 file changed, 24 insertions(+), 4 deletions(-)

diff --git a/script/zonemaster_backend_rpcapi.psgi b/script/zonemaster_backend_rpcapi.psgi
index 669212a17..942740a38 100644
--- a/script/zonemaster_backend_rpcapi.psgi
+++ b/script/zonemaster_backend_rpcapi.psgi
@@ -6,14 +6,17 @@ our $VERSION = '1.1.0';
 
 use 5.14.2;
 
-use JSON::RPC::Dispatch;
-use Router::Simple::Declare;
+use English qw( $PID );
 use JSON::PP;
+use JSON::RPC::Dispatch;
+use Log::Any qw( $log );
+use Log::Any::Adapter;
+use Log::Dispatch;
 use POSIX;
-use Try::Tiny;
-
 use Plack::Builder;
 use Plack::Response;
+use Router::Simple::Declare;
+use Try::Tiny;
 
 BEGIN { $ENV{PERL_JSON_BACKEND} = 'JSON::PP' };
 
@@ -103,6 +106,23 @@ my $router = router {
 	};
 };
 
+Log::Any::Adapter->set(
+    'Dispatch',
+    dispatcher => Log::Dispatch->new(
+        outputs => [
+            [
+                'Screen',
+                min_level => 'warning',
+                stderr    => 1,
+                callbacks => sub {
+                    my %args = @_;
+                    $args{message} = sprintf "%s [%d] %s - %s\n", strftime( "%FT%TZ", gmtime ), $PID, uc $args{level}, $args{message};
+                },
+            ],
+        ]
+    ),
+);
+
 my $dispatch = JSON::RPC::Dispatch->new(
 	router => $router,
 );

From 5f3e59c43682930a58430da179c5850f42d47119 Mon Sep 17 00:00:00 2001
From: Alexandre Pion <pion@afnic.fr>
Date: Wed, 30 Sep 2020 18:14:49 +0200
Subject: [PATCH 54/92] Fix typo

---
 docs/API.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/API.md b/docs/API.md
index 4485099bd..fb06cc043 100644
--- a/docs/API.md
+++ b/docs/API.md
@@ -573,7 +573,7 @@ If an identical *test* was already enqueued and hasn't been started or was enque
 no new *test* is enqueued.
 Instead the id for the already enqueued or run test is returned.
 
-*Tests* enqueud using this method are assigned a *priority* of 10.
+*Tests* enqueued using this method are assigned a *priority* of 10.
 
 Example request:
 ```json

From ef097c1f0d2fc26284b7e38053100b321acb744e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Thu, 1 Oct 2020 01:41:51 +0200
Subject: [PATCH 55/92] Introduce ZM_BACKEND_RPCAPI_LOGLEVEL

---
 script/zonemaster_backend_rpcapi.psgi | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/script/zonemaster_backend_rpcapi.psgi b/script/zonemaster_backend_rpcapi.psgi
index 942740a38..19b12d909 100644
--- a/script/zonemaster_backend_rpcapi.psgi
+++ b/script/zonemaster_backend_rpcapi.psgi
@@ -11,6 +11,7 @@ use JSON::PP;
 use JSON::RPC::Dispatch;
 use Log::Any qw( $log );
 use Log::Any::Adapter;
+use Log::Any::Adapter::Util qw( logging_methods logging_aliases );
 use Log::Dispatch;
 use POSIX;
 use Plack::Builder;
@@ -106,13 +107,24 @@ my $router = router {
 	};
 };
 
+# Returns a Log::Any-compatible log level string, or throws an exception.
+sub get_loglevel {
+    my $value = $ENV{ZM_BACKEND_RPCAPI_LOGLEVEL} || 'warning';
+    for my $method ( logging_methods(), logging_aliases() ) {
+        if ( $value eq $method ) {
+            return $method;
+        }
+    }
+    die "Error: Unrecognized ZM_BACKEND_RPCAPI_LOGLEVEL $value\n";
+}
+
 Log::Any::Adapter->set(
     'Dispatch',
     dispatcher => Log::Dispatch->new(
         outputs => [
             [
                 'Screen',
-                min_level => 'warning',
+                min_level => get_loglevel(),
                 stderr    => 1,
                 callbacks => sub {
                     my %args = @_;

From 88744d76b295edee62a910ea961945a6aaf7feca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Thu, 1 Oct 2020 02:59:41 +0200
Subject: [PATCH 56/92] Fix warning

---
 lib/Zonemaster/Backend/Translator.pm | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/lib/Zonemaster/Backend/Translator.pm b/lib/Zonemaster/Backend/Translator.pm
index 5584098fc..10361af26 100644
--- a/lib/Zonemaster/Backend/Translator.pm
+++ b/lib/Zonemaster/Backend/Translator.pm
@@ -16,7 +16,7 @@ require Zonemaster::Engine::Logger::Entry;
 extends 'Zonemaster::Engine::Translator';
 
 sub translate_tag {
-    my ( $self, $entry, $browser_lang ) = @_;
+    my ( $self, $hashref, $browser_lang ) = @_;
     my %locale = Zonemaster::Backend::Config->load_config()->Language_Locale_hash();
     my $previous_locale = $self->locale;
 
@@ -45,17 +45,13 @@ sub translate_tag {
         $ENV{LC_ALL} || $ENV{LC_CTYPE};
     }
 
-    my $string = $self->data->{ $entry->{module} }{ $entry->{tag} };
-
-    if ( not $string ) {
-        return $entry->{string};
-    }
-
-    my $blessed_entry = bless($entry, 'Zonemaster::Engine::Logger::Entry');
-    my $octets = Zonemaster::Engine::Translator::translate_tag( $self, $blessed_entry );
+    my $entry = Zonemaster::Engine::Logger::Entry->new( %{ $hashref } );
+    my $octets = Zonemaster::Engine::Translator::translate_tag( $self, $entry );
     $self->locale( $previous_locale );
-    my $str = decode_utf8( $octets );
-    return $str;
+
+    return ( defined $octets )
+      ? decode_utf8( $octets )
+      : $entry->string;
 }
 
 1;

From 43abc2b2c9b23e176539823baea9dc3e4089215e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Thu, 1 Oct 2020 06:58:17 +0200
Subject: [PATCH 57/92] Remove dead code

---
 lib/Zonemaster/Backend/Translator.pm | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/lib/Zonemaster/Backend/Translator.pm b/lib/Zonemaster/Backend/Translator.pm
index 10361af26..8b2f8f298 100644
--- a/lib/Zonemaster/Backend/Translator.pm
+++ b/lib/Zonemaster/Backend/Translator.pm
@@ -49,9 +49,7 @@ sub translate_tag {
     my $octets = Zonemaster::Engine::Translator::translate_tag( $self, $entry );
     $self->locale( $previous_locale );
 
-    return ( defined $octets )
-      ? decode_utf8( $octets )
-      : $entry->string;
+    return decode_utf8( $octets );
 }
 
 1;

From 2699267525cbe8f6143e6b54036e9d77d39aa8c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Thu, 1 Oct 2020 08:21:01 +0200
Subject: [PATCH 58/92] We don't use Z::E::Translator::data anymore

---
 lib/Zonemaster/Backend/RPCAPI.pm | 2 --
 1 file changed, 2 deletions(-)

diff --git a/lib/Zonemaster/Backend/RPCAPI.pm b/lib/Zonemaster/Backend/RPCAPI.pm
index 5cc918e30..1ecd2045e 100644
--- a/lib/Zonemaster/Backend/RPCAPI.pm
+++ b/lib/Zonemaster/Backend/RPCAPI.pm
@@ -388,8 +388,6 @@ sub get_test_results {
     my $translator;
     $translator = Zonemaster::Backend::Translator->new;
 
-    eval { $translator->data } if $translator;    # Provoke lazy loading of translation data
-
     my $test_info = $self->{db}->test_results( $params->{id} );
     my @zm_results;
     foreach my $test_res ( @{ $test_info->{results} } ) {

From 6bac05962424020236c7c84010c53f8a3431d4a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Thu, 1 Oct 2020 07:35:58 +0200
Subject: [PATCH 59/92] Make get_test_results validate its own input

---
 lib/Zonemaster/Backend/RPCAPI.pm     | 12 ++++++++++++
 lib/Zonemaster/Backend/Translator.pm | 14 +-------------
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/lib/Zonemaster/Backend/RPCAPI.pm b/lib/Zonemaster/Backend/RPCAPI.pm
index 1ecd2045e..f91ce7d8d 100644
--- a/lib/Zonemaster/Backend/RPCAPI.pm
+++ b/lib/Zonemaster/Backend/RPCAPI.pm
@@ -388,6 +388,18 @@ sub get_test_results {
     my $translator;
     $translator = Zonemaster::Backend::Translator->new;
 
+    my %locale = Zonemaster::Backend::Config->load_config()->Language_Locale_hash();
+    if ( $locale{$params->{language}} ) {
+        if ( $locale{$params->{language}} eq 'NOT-UNIQUE') {
+            die "Language string not unique: '$params->{language}'\n";
+        }
+    }
+    else {
+        die "Undefined language string: '$params->{language}'\n";
+    }
+
+    eval { $translator->data } if $translator;    # Provoke lazy loading of translation data
+
     my $test_info = $self->{db}->test_results( $params->{id} );
     my @zm_results;
     foreach my $test_res ( @{ $test_info->{results} } ) {
diff --git a/lib/Zonemaster/Backend/Translator.pm b/lib/Zonemaster/Backend/Translator.pm
index 8b2f8f298..2ddf26f81 100644
--- a/lib/Zonemaster/Backend/Translator.pm
+++ b/lib/Zonemaster/Backend/Translator.pm
@@ -17,20 +17,8 @@ extends 'Zonemaster::Engine::Translator';
 
 sub translate_tag {
     my ( $self, $hashref, $browser_lang ) = @_;
-    my %locale = Zonemaster::Backend::Config->load_config()->Language_Locale_hash();
     my $previous_locale = $self->locale;
-
-    if ( $locale{$browser_lang} ) {
-        if ( $locale{$browser_lang} eq 'NOT-UNIQUE') {
-            die "Language string not unique: '$browser_lang'\n";
-        }
-        else {
-            $self->locale( $locale{$browser_lang} );
-        }
-    }
-    else {
-        die "Undefined language string: '$browser_lang'\n";
-    }
+    $self->locale( $locale{$browser_lang} );
 
     # Make locale really be set. Fix that makes translation work on FreeBSD 12.1. Solution copied from
     # CLI.pm in the Zonemaster-CLI repository.

From e8045c0f9f12093c27fa283c2802c502cc2cc54e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Thu, 1 Oct 2020 08:20:09 +0200
Subject: [PATCH 60/92] Set locale per request, not per message

---
 lib/Zonemaster/Backend/RPCAPI.pm     | 5 +++++
 lib/Zonemaster/Backend/Translator.pm | 3 ---
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/lib/Zonemaster/Backend/RPCAPI.pm b/lib/Zonemaster/Backend/RPCAPI.pm
index f91ce7d8d..d72356275 100644
--- a/lib/Zonemaster/Backend/RPCAPI.pm
+++ b/lib/Zonemaster/Backend/RPCAPI.pm
@@ -398,6 +398,9 @@ sub get_test_results {
         die "Undefined language string: '$params->{language}'\n";
     }
 
+    my $previous_locale = $translator->locale;
+    $translator->locale( $locale{$params->{language}} );
+
     eval { $translator->data } if $translator;    # Provoke lazy loading of translation data
 
     my $test_info = $self->{db}->test_results( $params->{id} );
@@ -448,6 +451,8 @@ sub get_test_results {
         push( @zm_results, $res );
     }
 
+    $translator->locale( $previous_locale );
+
     $result = $test_info;
     $result->{results} = \@zm_results;
 
diff --git a/lib/Zonemaster/Backend/Translator.pm b/lib/Zonemaster/Backend/Translator.pm
index 2ddf26f81..212cb64f1 100644
--- a/lib/Zonemaster/Backend/Translator.pm
+++ b/lib/Zonemaster/Backend/Translator.pm
@@ -17,8 +17,6 @@ extends 'Zonemaster::Engine::Translator';
 
 sub translate_tag {
     my ( $self, $hashref, $browser_lang ) = @_;
-    my $previous_locale = $self->locale;
-    $self->locale( $locale{$browser_lang} );
 
     # Make locale really be set. Fix that makes translation work on FreeBSD 12.1. Solution copied from
     # CLI.pm in the Zonemaster-CLI repository.
@@ -35,7 +33,6 @@ sub translate_tag {
 
     my $entry = Zonemaster::Engine::Logger::Entry->new( %{ $hashref } );
     my $octets = Zonemaster::Engine::Translator::translate_tag( $self, $entry );
-    $self->locale( $previous_locale );
 
     return decode_utf8( $octets );
 }

From 0d8d067e6f4a183058ea3387a68bf914f7239924 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Thu, 1 Oct 2020 11:33:22 +0200
Subject: [PATCH 61/92] Add some context to a workaround

---
 lib/Zonemaster/Backend/Translator.pm | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/Zonemaster/Backend/Translator.pm b/lib/Zonemaster/Backend/Translator.pm
index 212cb64f1..32b974a71 100644
--- a/lib/Zonemaster/Backend/Translator.pm
+++ b/lib/Zonemaster/Backend/Translator.pm
@@ -18,6 +18,7 @@ extends 'Zonemaster::Engine::Translator';
 sub translate_tag {
     my ( $self, $hashref, $browser_lang ) = @_;
 
+    # Workaround for broken Zonemaster::Engine::translate_tag in Zonemaster-Engine 3.1.2.
     # Make locale really be set. Fix that makes translation work on FreeBSD 12.1. Solution copied from
     # CLI.pm in the Zonemaster-CLI repository.
     undef $ENV{LANGUAGE};

From 65f5257b4d726fbb7be35c1212f13d3bb6f54b1b Mon Sep 17 00:00:00 2001
From: Michal TOMA <toma@afnic.fr>
Date: Fri, 9 Oct 2020 11:58:55 +0200
Subject: [PATCH 62/92] Several fxes to address remarks in the pull request

---
 .../maintenance/Garbage-Collection-Testing.md | 200 +++++++++---------
 1 file changed, 103 insertions(+), 97 deletions(-)

diff --git a/docs/internal-documentation/maintenance/Garbage-Collection-Testing.md b/docs/internal-documentation/maintenance/Garbage-Collection-Testing.md
index e73269ec4..c3853ed48 100644
--- a/docs/internal-documentation/maintenance/Garbage-Collection-Testing.md
+++ b/docs/internal-documentation/maintenance/Garbage-Collection-Testing.md
@@ -5,118 +5,124 @@ The purpose of this instruction is to serve as a notice for manual testing of th
 
 ## Testing the unfinished tests garbage collection feature
 
-1. Ensure that the database has the required aditionnal columns for this feature:
-```
-SELECT nb_retries FROM test_results LIMIT 0;
-```
-Should return:
-
-```
- nb_retries 
-------------
-(0 rows)
-```
-
-2. Check that your backend_config.ini has the proper parameter set
-
-Either disabled:
-```
-#maximal_number_of_retries=3
-```
-or set to 0:
-```
-maximal_number_of_retries=0
-```
+1. Ensure that the database has the required additionnal columns for this feature:
+     ```
+     SELECT nb_retries FROM test_results LIMIT 0;
+     ```
+     Should return:
+
+     ```
+     nb_retries 
+     ------------
+     (0 rows)
+     ```
+     _Remark: for MySQL use `SHOW COLUMNS FROM test_results` and ensure the `nb_retries` column is present in the list._
+
+2. Check that your `/etc/zonemaster/bacend_config.ini` (or, in FreeBSD, `/usr/local/etc/zonemaster/bacend_config.ini`) has the proper parameter set
+
+     Either disabled:
+     ```
+     #maximal_number_of_retries=3
+     ```
+     or set to 0:
+     ```
+     maximal_number_of_retries=0
+     ```
 
 3. Start a test and wait for it to be finished
 
-```
-SELECT hash_id, progress FROM test_results LIMIT 1;
-```
-Should return:
+     ```
+     SELECT hash_id, progress FROM test_results LIMIT 1;
+     ```
+     Should return:
 
-```
-     hash_id      | progress 
-------------------+----------
- 3f7a604683efaf93 |      100
-(1 row)
-```
+     ```
+          hash_id      | progress 
+     ------------------+----------
+     3f7a604683efaf93 |      100
+     (1 row)
+     ```
 
-4. Sumulate a crashed test
-```
-UPDATE test_results SET progress = 50, test_start_time = '2020-01-01' WHERE hash_id = '3f7a604683efaf93';
-```
+4. Simulate a crashed test
+     ```
+     UPDATE test_results SET progress = 50, test_start_time = '2020-01-01' WHERE hash_id = '3f7a604683efaf93';
+     ```
 
 5. Chaek that the backend finishes the test with a result stating it was unfinished
 
-```
-SELECT hash_id, progress FROM test_results WHERE hash_id = '3f7a604683efaf93';
-```
-Should return a finished result:
-```
-     hash_id      | progress 
-------------------+----------
- 3f7a604683efaf93 |      100
-(1 row)
-```
-6. Ensure the test result contains the backend generated critical message message:
-```
-{"tag":"UNABLE_TO_FINISH_TEST","level":"CRITICAL","timestamp":"300","module":"BACKEND_TEST_AGENT"}
-```
-
-```
-SELECT hash_id, progress FROM test_results WHERE hash_id = '3f7a604683efaf93' AND results::text like '%UNABLE_TO_FINISH_TEST%';
-```
-Should return:
-```
-     hash_id      | progress 
-------------------+----------
- 3f7a604683efaf93 |      100
-(1 row)
-
-```
+     ```
+     SELECT hash_id, progress FROM test_results WHERE hash_id = '3f7a604683efaf93';
+     ```
+     Should return a finished result:
+     ```
+          hash_id      | progress 
+     ------------------+----------
+     3f7a604683efaf93 |      100
+     (1 row)
+     ```
+
+6. Ensure the test result contains the backend generated critical message:
+     ```
+     {"tag":"UNABLE_TO_FINISH_TEST","level":"CRITICAL","timestamp":"300","module":"BACKEND_TEST_AGENT"}
+     ```
+
+     ```
+     SELECT hash_id, progress FROM test_results WHERE hash_id = '3f7a604683efaf93' AND results::text like '%UNABLE_TO_FINISH_TEST%';
+     ```
+     _Remark: for MySQL queries remove the `::text` from all queries_
+
+     Should return:
+     ```
+          hash_id      | progress 
+     ------------------+----------
+     3f7a604683efaf93 |      100
+     (1 row)
+
+     ```
 
 ## Testing the unfinished tests garbage collection feature with a number of retries set to allow finishing tests without a critical error
 
 1. Set the maximal_number_of_retries parameter to a value greater than 0 in the backend_config.ini config file 
-```
-maximal_number_of_retries=1
-```
+     ```
+     maximal_number_of_retries=1
+     ```
 
 2. Restart the test agent in order for the parameter to be taken into account
-```
-zonemaster_backend_testagent restart
-```
+     ```
+     zonemaster_backend_testagent restart
+     ```
 
-3. Sumulate a crashed test
-```
-UPDATE test_results SET progress = 50, test_start_time = '2020-01-01' WHERE hash_id = '3f7a604683efaf93';
-```
+3. Simulate a crashed test
+     ```
+     UPDATE test_results SET progress = 50, test_start_time = '2020-01-01' WHERE hash_id = '3f7a604683efaf93';
+     ```
 
 4. Check that the backend finishes the test WITHOUT a result stating it was unfinished
 
-```
-SELECT hash_id, progress FROM test_results WHERE hash_id = '3f7a604683efaf93';
-```
-Should return a finished result:
-```
-     hash_id      | progress 
-------------------+----------
- 3f7a604683efaf93 |      100
-(1 row)
-```
-5. Ensure the test result does NOT contain the backend generated critical message message:
-```
-{"tag":"UNABLE_TO_FINISH_TEST","level":"CRITICAL","timestamp":"300","module":"BACKEND_TEST_AGENT"}
-```
-
-```
-SELECT hash_id, progress FROM test_results WHERE hash_id = '3f7a604683efaf93' AND results::text like '%UNABLE_TO_FINISH_TEST%';
-```
-Should return:
-```
- hash_id | progress 
----------+----------
-(0 rows)
-
-```
+     ```
+     SELECT hash_id, progress FROM test_results WHERE hash_id = '3f7a604683efaf93';
+     ```
+     Should return a finished result:
+     ```
+          hash_id      | progress 
+     ------------------+----------
+     3f7a604683efaf93 |      100
+     (1 row)
+     ```
+
+5. Ensure the test result does NOT contain the backend generated critical message:
+     ```
+     {"tag":"UNABLE_TO_FINISH_TEST","level":"CRITICAL","timestamp":"300","module":"BACKEND_TEST_AGENT"}
+     ```
+
+     ```
+     SELECT hash_id, progress FROM test_results WHERE hash_id = '3f7a604683efaf93' AND results::text like '%UNABLE_TO_FINISH_TEST%';
+     ```
+     _Remark: for MySQL queries remove the `::text` from all queries_
+     Should return:
+     ```
+     hash_id | progress 
+     ---------+----------
+     (0 rows)
+
+     ```

From 416b48ba2e628bad0e713cc3fbd35cf266f62379 Mon Sep 17 00:00:00 2001
From: Michal TOMA <toma@afnic.fr>
Date: Fri, 9 Oct 2020 12:19:28 +0200
Subject: [PATCH 63/92] Fixed code according to remarks in pull request

---
 lib/Zonemaster/Backend/RPCAPI.pm | 48 ++++++++++++++++----------------
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/lib/Zonemaster/Backend/RPCAPI.pm b/lib/Zonemaster/Backend/RPCAPI.pm
index b66b7fb28..b1ff374a8 100644
--- a/lib/Zonemaster/Backend/RPCAPI.pm
+++ b/lib/Zonemaster/Backend/RPCAPI.pm
@@ -42,8 +42,8 @@ sub new {
             $self->{db} = "$params->{db}"->new();
         };
         if ($@) {
-            warn "Failed to initialize the [$params->{db}] database backend module: [$@] \n";
-            die "Failed to initialize the [$params->{db}] database backend module \n";
+            warn "Internal error #001: Failed to initialize the [$params->{db}] database backend module: [$@] \n";
+            die "Internal error #001 \n";
         }
     }
     else {
@@ -54,8 +54,8 @@ sub new {
             $self->{db} = $backend_module->new();
         };
         if ($@) {
-            warn "Failed to initialize the database backend module: [$@] \n";
-            die "Failed to initialize the database backend module \n" if $@;
+            warn "Internal error #002: Failed to initialize the database backend module: [$@] \n";
+            die "Internal error #002 \n" if $@;
         }
     }
 
@@ -63,10 +63,10 @@ sub new {
 }
 
 sub handle_exception {
-    my ( $method, $exception ) = @_;
+    my ( $method, $exception, $exception_id ) = @_;
 
-    warn "Unexpected error in the $method API call: [$exception] \n";
-    die "Unexpected error in the $method API call \n";
+    warn "Internal error $exception_id: Unexpected error in the $method API call: [$exception] \n";
+    die "Internal error $exception_id \n";
 }
 
 $json_schemas{version_info} = joi->object->strict;
@@ -80,7 +80,7 @@ sub version_info {
 
     };
     if ($@) {
-        handle_exception('version_info', $@);
+        handle_exception('version_info', $@, '#003');
     }
     
     return \%ver;
@@ -96,7 +96,7 @@ sub profile_names {
 
     };
     if ($@) {
-        handle_exception('profile_names', $@);
+        handle_exception('profile_names', $@, '#004');
     }
     
     return \@profiles;
@@ -116,7 +116,7 @@ sub get_host_by_name {
 
     };
     if ($@) {
-        handle_exception('get_host_by_name', $@);
+        handle_exception('get_host_by_name', $@, '#005');
     }
     
     return \@adresses;
@@ -164,7 +164,7 @@ sub get_data_from_parent_zone {
 
     };
     if ($@) {
-        handle_exception('get_data_from_parent_zone', $@);
+        handle_exception('get_data_from_parent_zone', $@, '#006');
     }
     
     return \%result;
@@ -198,7 +198,7 @@ sub _check_domain {
                     {
                         status => 'nok',
                         message =>
-                        encode_entities( "$type contains non-ascii characters and IDNA conversion is not installed" )
+                        encode_entities( "$type contains non-ascii characters and IDNA is not installed" )
                     }
                 );
             }
@@ -209,7 +209,7 @@ sub _check_domain {
                 $dn,
                 {
                 status  => 'nok',
-                message => encode_entities( "The domain name contains unauthorized character(s)")
+                message => encode_entities( "The domain name character(s) not supported")
                 }
             );
         }
@@ -217,11 +217,11 @@ sub _check_domain {
         my @res;
         @res = Zonemaster::Engine::Test::Basic->basic00($dn);
         if (@res != 0) {
-            return ( $dn, { status => 'nok', message => encode_entities( "$type name or label outside allowed length" ) } );
+            return ( $dn, { status => 'nok', message => encode_entities( "$type name or label is too long" ) } );
         }
     };
     if ($@) {
-        handle_exception('_check_domain', $@);
+        handle_exception('_check_domain', $@, '#007');
     }
 
     return ( $dn, { status => 'ok', message => 'Syntax ok' } );
@@ -323,7 +323,7 @@ sub validate_syntax {
         $message = encode_entities( 'Syntax ok' );
     };
     if ($@) {
-        handle_exception('validate_syntax', $@);
+        handle_exception('validate_syntax', $@, '#008');
     }
 
     return { status => 'ok', message =>  $message };
@@ -370,7 +370,7 @@ sub start_domain_test {
         $result = $self->{db}->create_new_test( $params->{domain}, $params, $minutes_between_tests_with_same_params );
     };
     if ($@) {
-        handle_exception('start_domain_test', $@);
+        handle_exception('start_domain_test', $@, '#009');
     }
 
     return $result;
@@ -395,7 +395,7 @@ sub test_progress {
         $result = $self->{db}->test_progress( $test_id );
     };
     if ($@) {
-        handle_exception('test_progress', $@);
+        handle_exception('test_progress', $@, '#010');
     }
 
     return $result;
@@ -414,7 +414,7 @@ sub get_test_params {
         $result = $self->{db}->get_test_params( $test_id );
     };
     if ($@) {
-        handle_exception('get_test_params', $@);
+        handle_exception('get_test_params', $@, '#011');
     }
 
     return $result;
@@ -488,7 +488,7 @@ sub get_test_results {
         $result->{results} = \@zm_results;
     };
     if ($@) {
-        handle_exception('get_test_results', $@);
+        handle_exception('get_test_results', $@, '#012');
     }
 
     return $result;
@@ -515,7 +515,7 @@ sub get_test_history {
         $results = $self->{db}->get_test_history( $p );
     };
     if ($@) {
-        handle_exception('get_test_history', $@);
+        handle_exception('get_test_history', $@, '#013');
     }
 
     return $results;
@@ -544,7 +544,7 @@ sub add_api_user {
         }
     };
     if ($@) {
-        handle_exception('add_api_user', $@);
+        handle_exception('add_api_user', $@, '#014');
     }
 
     return $result;
@@ -584,7 +584,7 @@ sub add_batch_job {
         $results = $self->{db}->add_batch_job( $params );
     };
     if ($@) {
-        handle_exception('add_batch_job', $@);
+        handle_exception('add_batch_job', $@, '#015');
     }
 
     return $results;
@@ -604,7 +604,7 @@ sub get_batch_job_result {
         $result = $self->{db}->get_batch_job_result($batch_id);
     };
     if ($@) {
-        handle_exception('get_batch_job_result', $@);
+        handle_exception('get_batch_job_result', $@, '#016');
     }
     
     return $result;

From 36cfdddee11e4306c72a54fb3a29fe31b74f9bee Mon Sep 17 00:00:00 2001
From: Michal TOMA <toma@afnic.fr>
Date: Fri, 9 Oct 2020 12:51:36 +0200
Subject: [PATCH 64/92] Fixed merge mistakes

---
 lib/Zonemaster/Backend/RPCAPI.pm | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/lib/Zonemaster/Backend/RPCAPI.pm b/lib/Zonemaster/Backend/RPCAPI.pm
index 37cfb0a71..28e8c4c21 100644
--- a/lib/Zonemaster/Backend/RPCAPI.pm
+++ b/lib/Zonemaster/Backend/RPCAPI.pm
@@ -232,6 +232,11 @@ sub _check_domain {
             );
         }
 
+    };
+    if ($@) {
+        handle_exception('_check_domain', $@, '#007');
+    }
+    
     my %levels = Zonemaster::Engine::Logger::Entry::levels();
     my @res;
     @res = Zonemaster::Engine::Test::Basic->basic00($dn);
@@ -310,11 +315,11 @@ sub validate_syntax {
                 return { status => 'nok', message => encode_entities( "Invalid IP address: [$ns_ip->{ip}]" ) }
                     unless( !$ns_ip->{ip} || $ns_ip->{ip} =~ /^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$/ || $ns_ip->{ip} =~ /^([0-9A-Fa-f]{1,4}:[0-9A-Fa-f:]{1,}(:[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})?)|([0-9A-Fa-f]{1,4}::[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})$/);
 
-            return { status => 'nok', message => encode_entities( "Invalid IP address: [$ns_ip->{ip}]" ) }
-              unless ( !$ns_ip->{ip}
-                || Zonemaster::Engine::Net::IP::ip_is_ipv4( $ns_ip->{ip} )
-                || Zonemaster::Engine::Net::IP::ip_is_ipv6( $ns_ip->{ip} ) );
-        }
+                return { status => 'nok', message => encode_entities( "Invalid IP address: [$ns_ip->{ip}]" ) }
+                unless ( !$ns_ip->{ip}
+                    || Zonemaster::Engine::Net::IP::ip_is_ipv4( $ns_ip->{ip} )
+                    || Zonemaster::Engine::Net::IP::ip_is_ipv6( $ns_ip->{ip} ) );
+            }
 
             foreach my $ds_digest ( @{ $syntax_input->{ds_info} } ) {
                 return {
@@ -449,6 +454,7 @@ sub get_test_results {
 
     my $translator;
     $translator = Zonemaster::Backend::Translator->new;
+    my ( $browser_lang ) = ( $params->{language} =~ /^(\w{2})/ );
 
     my %locale = Zonemaster::Backend::Config->load_config()->Language_Locale_hash();
     if ( $locale{$params->{language}} ) {

From e07118f7c3bae244d68b84232ddb78f544cafea8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Mon, 12 Oct 2020 07:23:27 +0200
Subject: [PATCH 65/92] Update manifest

---
 MANIFEST | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/MANIFEST b/MANIFEST
index 76af1411d..9ec479bca 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -8,10 +8,10 @@ CodeSnippets/sample_api_call__validate_syntax.pl
 CodeSnippets/sample_api_call__version_info.pl
 docs/API.md
 docs/Architecture.md
+docs/Configuration.md
 docs/files-description.md
 docs/GettingStarted.md
 docs/Installation.md
-docs/Configuration.md
 docs/TypographicConventions.md
 docs/upgrade_db_zonemaster_backend_ver_1.0.3.md
 docs/upgrade_db_zonemaster_backend_ver_1.1.0.md
@@ -51,6 +51,7 @@ script/zonemaster_backend_testagent
 share/backend_config.ini
 share/cleanup-mysql.sql
 share/cleanup-postgres.sql
+share/freebsd-pwd.conf
 share/initial-mysql.sql
 share/initial-postgres.sql
 share/patch_db_README.txt
@@ -76,5 +77,3 @@ t/test01.t
 t/test_DB_backend.pl
 t/test_runner.pl
 t/test_validate_syntax.t
-
-

From bf612b07a9188d48bae6be77eb7544110d5cdd2c Mon Sep 17 00:00:00 2001
From: Michal TOMA <toma@afnic.fr>
Date: Mon, 12 Oct 2020 16:53:13 +0200
Subject: [PATCH 66/92] Fixes required to make all unit tests pass

---
 lib/Zonemaster/Backend/RPCAPI.pm | 64 ++++++++++++--------------------
 1 file changed, 24 insertions(+), 40 deletions(-)

diff --git a/lib/Zonemaster/Backend/RPCAPI.pm b/lib/Zonemaster/Backend/RPCAPI.pm
index 28e8c4c21..60fa02fd3 100644
--- a/lib/Zonemaster/Backend/RPCAPI.pm
+++ b/lib/Zonemaster/Backend/RPCAPI.pm
@@ -72,6 +72,8 @@ sub new {
 sub handle_exception {
     my ( $method, $exception, $exception_id ) = @_;
 
+    $exception =~ s/\n/ /g;
+    chomp($exception);
     warn "Internal error $exception_id: Unexpected error in the $method API call: [$exception] \n";
     die "Internal error $exception_id \n";
 }
@@ -251,8 +253,7 @@ sub _check_domain {
 sub validate_syntax {
     my ( $self, $syntax_input ) = @_;
 
-    my $message;
-    eval {
+    my $result = eval {
         my @allowed_params_keys = (
             'domain',   'ipv4',      'ipv6', 'ds_info', 'nameservers', 'profile',
             'client_id', 'client_version', 'config', 'priority', 'queue'
@@ -342,14 +343,16 @@ sub validate_syntax {
                 );
             }
         }
-        
-        $message = encode_entities( 'Syntax ok' );
     };
     if ($@) {
         handle_exception('validate_syntax', $@, '#008');
     }
-
-    return { status => 'ok', message =>  $message };
+    elsif ($result) {
+        return $result;
+    }
+    else {
+        return { status => 'ok', message =>  encode_entities( 'Syntax ok' ) };
+    }
 }
 
 $json_schemas{start_domain_test} = joi->object->strict->props(
@@ -451,10 +454,8 @@ sub get_test_results {
     my ( $self, $params ) = @_;
 
     my $result;
-
     my $translator;
     $translator = Zonemaster::Backend::Translator->new;
-    my ( $browser_lang ) = ( $params->{language} =~ /^(\w{2})/ );
 
     my %locale = Zonemaster::Backend::Config->load_config()->Language_Locale_hash();
     if ( $locale{$params->{language}} ) {
@@ -469,43 +470,26 @@ sub get_test_results {
     my $previous_locale = $translator->locale;
     $translator->locale( $locale{$params->{language}} );
 
-        eval { $translator->data } if $translator;    # Provoke lazy loading of translation data
+    eval { $translator->data } if $translator;    # Provoke lazy loading of translation data
 
-    my $test_info = $self->{db}->test_results( $params->{id} );
+    my $test_info;
     my @zm_results;
-    foreach my $test_res ( @{ $test_info->{results} } ) {
-        my $res;
-        if ( $test_res->{module} eq 'NAMESERVER' ) {
-            $res->{ns} = ( $test_res->{args}->{ns} ) ? ( $test_res->{args}->{ns} ) : ( 'All' );
-        }
-        elsif ($test_res->{module} eq 'SYSTEM'
-            && $test_res->{tag} eq 'POLICY_DISABLED'
-            && $test_res->{args}->{name} eq 'Example' )
-        {
-            next;
-        }
-
-        $res->{module} = $test_res->{module};
-        $res->{message} = $translator->translate_tag( $test_res, $params->{language} ) . "\n";
-        $res->{message} =~ s/,/, /isg;
-        $res->{message} =~ s/;/; /isg;
-        $res->{level} = $test_res->{level};
-
-        if ( $test_res->{module} eq 'SYSTEM' ) {
-            if ( $res->{message} =~ /policy\.json/ ) {
-                my ( $policy ) = ( $res->{message} =~ /\s(\/.*)$/ );
-                if ( $policy ) {
-                    my $policy_description = 'DEFAULT POLICY';
-                    $policy_description = 'SOME OTHER POLICY' if ( $policy =~ /some\/other\/policy\/path/ );
-                    $res->{message} =~ s/$policy/$policy_description/;
-                }
-                else {
-                    $res->{message} = 'UNKNOWN POLICY FORMAT';
-                }
+    eval{
+        $test_info = $self->{db}->test_results( $params->{id} );
+        foreach my $test_res ( @{ $test_info->{results} } ) {
+            my $res;
+            if ( $test_res->{module} eq 'NAMESERVER' ) {
+                $res->{ns} = ( $test_res->{args}->{ns} ) ? ( $test_res->{args}->{ns} ) : ( 'All' );
+            }
+            elsif ($test_res->{module} eq 'SYSTEM'
+                && $test_res->{tag} eq 'POLICY_DISABLED'
+                && $test_res->{args}->{name} eq 'Example' )
+            {
+                next;
             }
 
             $res->{module} = $test_res->{module};
-            $res->{message} = $translator->translate_tag( $test_res, $browser_lang ) . "\n";
+            $res->{message} = $translator->translate_tag( $test_res, $params->{language} ) . "\n";
             $res->{message} =~ s/,/, /isg;
             $res->{message} =~ s/;/; /isg;
             $res->{level} = $test_res->{level};

From 94c325a9b9304695c0873df0fa9be23682cf82e4 Mon Sep 17 00:00:00 2001
From: Michal TOMA <toma@afnic.fr>
Date: Mon, 12 Oct 2020 17:09:50 +0200
Subject: [PATCH 67/92] Fixes according to remarks in the pull request

---
 .../maintenance/Garbage-Collection-Testing.md                | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/internal-documentation/maintenance/Garbage-Collection-Testing.md b/docs/internal-documentation/maintenance/Garbage-Collection-Testing.md
index c3853ed48..392477eb3 100644
--- a/docs/internal-documentation/maintenance/Garbage-Collection-Testing.md
+++ b/docs/internal-documentation/maintenance/Garbage-Collection-Testing.md
@@ -18,7 +18,7 @@ The purpose of this instruction is to serve as a notice for manual testing of th
      ```
      _Remark: for MySQL use `SHOW COLUMNS FROM test_results` and ensure the `nb_retries` column is present in the list._
 
-2. Check that your `/etc/zonemaster/bacend_config.ini` (or, in FreeBSD, `/usr/local/etc/zonemaster/bacend_config.ini`) has the proper parameter set
+2. Check that your `/etc/zonemaster/backend_config.ini` (or, in FreeBSD, `/usr/local/etc/zonemaster/backend_config.ini`) has the proper parameter set
 
      Either disabled:
      ```
@@ -48,7 +48,7 @@ The purpose of this instruction is to serve as a notice for manual testing of th
      UPDATE test_results SET progress = 50, test_start_time = '2020-01-01' WHERE hash_id = '3f7a604683efaf93';
      ```
 
-5. Chaek that the backend finishes the test with a result stating it was unfinished
+5. Check that the backend finishes the test with a result stating it was unfinished
 
      ```
      SELECT hash_id, progress FROM test_results WHERE hash_id = '3f7a604683efaf93';
@@ -91,6 +91,7 @@ The purpose of this instruction is to serve as a notice for manual testing of th
      ```
      zonemaster_backend_testagent restart
      ```
+     _Remark: Update accordingly to the OS where the tests are done (ex use init script for FreeBSD, etc.)_
 
 3. Simulate a crashed test
      ```

From 1c2d4ad4e6c5deba2a82626d3e7fb08621f1efd9 Mon Sep 17 00:00:00 2001
From: Mats Dufberg <mats.dufberg@iis.se>
Date: Mon, 12 Oct 2020 18:18:23 +0200
Subject: [PATCH 68/92] Resolve issue #653

---
 share/zm-rpcapi.lsb | 1 +
 share/zm_rpcapi-bsd | 1 +
 2 files changed, 2 insertions(+)

diff --git a/share/zm-rpcapi.lsb b/share/zm-rpcapi.lsb
index 5527f4661..61ffa06a9 100644
--- a/share/zm-rpcapi.lsb
+++ b/share/zm-rpcapi.lsb
@@ -22,6 +22,7 @@ USER=${ZM_BACKEND_USER:-zonemaster}
 GROUP=${ZM_BACKEND_GROUP:-zonemaster}
 
 STARMAN=`PATH="$PATH:/usr/local/bin" /usr/bin/which starman`
+export ZM_BACKEND_RPCAPI_LOGLEVEL='warning' # Lowest level that will be logged
 
 . /lib/lsb/init-functions
 
diff --git a/share/zm_rpcapi-bsd b/share/zm_rpcapi-bsd
index 4f0c0fbe3..d89cb2600 100755
--- a/share/zm_rpcapi-bsd
+++ b/share/zm_rpcapi-bsd
@@ -18,6 +18,7 @@ load_rc_config $name
 : ${zm_rpcapi_listen="127.0.0.1:5000"}
 
 export ZONEMASTER_BACKEND_CONFIG_FILE="/usr/local/etc/zonemaster/backend_config.ini"
+export ZM_BACKEND_RPCAPI_LOGLEVEL='warning' # Lowest level that will be logged
 
 command="/usr/local/bin/starman"
 command_args="--daemonize --user=${zm_rpcapi_user} --group=${zm_rpcapi_group} --pid=${zm_rpcapi_pidfile} --error-log=${zm_rpcapi_logfile} --listen=${zm_rpcapi_listen} --app /usr/local/bin/zonemaster_backend_rpcapi.psgi"

From dcf35a6e18c9c1a50b2a4505580f8d647cb87216 Mon Sep 17 00:00:00 2001
From: Michal TOMA <toma@afnic.fr>
Date: Mon, 12 Oct 2020 18:20:54 +0200
Subject: [PATCH 69/92] Fixed several problems related to returns inside eval
 blocks and pr remarks

---
 lib/Zonemaster/Backend/RPCAPI.pm | 53 +++++++++++++++++---------------
 1 file changed, 28 insertions(+), 25 deletions(-)

diff --git a/lib/Zonemaster/Backend/RPCAPI.pm b/lib/Zonemaster/Backend/RPCAPI.pm
index 60fa02fd3..d80f7d95a 100644
--- a/lib/Zonemaster/Backend/RPCAPI.pm
+++ b/lib/Zonemaster/Backend/RPCAPI.pm
@@ -49,8 +49,7 @@ sub new {
             $self->{db} = "$params->{db}"->new( { config => $config } );
         };
         if ($@) {
-            warn "Internal error #001: Failed to initialize the [$params->{db}] database backend module: [$@] \n";
-            die "Internal error #001 \n";
+            handle_exception('new', "Failed to initialize the [$params->{db}] database backend module: [$@]", '001');
         }
     }
     else {
@@ -61,8 +60,7 @@ sub new {
             $self->{db} = $backend_module->new( { config => $config } );
         };
         if ($@) {
-            warn "Internal error #002: Failed to initialize the database backend module: [$@] \n";
-            die "Internal error #002 \n" if $@;
+            handle_exception('new', "Failed to initialize the [$params->{db}] database backend module: [$@]", '002');
         }
     }
 
@@ -73,7 +71,7 @@ sub handle_exception {
     my ( $method, $exception, $exception_id ) = @_;
 
     $exception =~ s/\n/ /g;
-    chomp($exception);
+    $exception =~ s/^\s+|\s+$//g;
     warn "Internal error $exception_id: Unexpected error in the $method API call: [$exception] \n";
     die "Internal error $exception_id \n";
 }
@@ -89,7 +87,7 @@ sub version_info {
 
     };
     if ($@) {
-        handle_exception('version_info', $@, '#003');
+        handle_exception('version_info', $@, '003');
     }
     
     return \%ver;
@@ -105,7 +103,7 @@ sub profile_names {
 
     };
     if ($@) {
-        handle_exception('profile_names', $@, '#004');
+        handle_exception('profile_names', $@, '004');
     }
     
     return \@profiles;
@@ -136,7 +134,7 @@ sub get_host_by_name {
 
     };
     if ($@) {
-        handle_exception('get_host_by_name', $@, '#005');
+        handle_exception('get_host_by_name', $@, '005');
     }
     
     return \@adresses;
@@ -149,8 +147,8 @@ sub get_data_from_parent_zone {
     my ( $self, $params ) = @_;
     my $domain = "";
 
-    my %result;
-    eval {
+    my $result = eval {
+		my %result;
         if (ref \$params eq "SCALAR") {
             $domain = $params;
         } else {
@@ -181,20 +179,21 @@ sub get_data_from_parent_zone {
 
         $result{ns_list} = \@ns_list;
         $result{ds_list} = \@ds_list;
-
+		return \%result;
     };
     if ($@) {
-        handle_exception('get_data_from_parent_zone', $@, '#006');
+        handle_exception('get_data_from_parent_zone', $@, '006');
+    }
+    elsif ($result) {
+		return $result;
     }
-    
-    return \%result;
 }
 
 
 sub _check_domain {
     my ( $self, $dn, $type ) = @_;
 
-    eval {
+    my $result = eval {
         if ( !defined( $dn ) ) {
             return ( $dn, { status => 'nok', message => encode_entities( "$type required" ) } );
         }
@@ -236,7 +235,10 @@ sub _check_domain {
 
     };
     if ($@) {
-        handle_exception('_check_domain', $@, '#007');
+        handle_exception('_check_domain', $@, '007');
+    }
+    elsif ($result) {
+		return $result;
     }
     
     my %levels = Zonemaster::Engine::Logger::Entry::levels();
@@ -313,6 +315,7 @@ sub validate_syntax {
             }
 
             foreach my $ns_ip ( @{ $syntax_input->{nameservers} } ) {
+                # Although counterintuitive both tests are necessary as Zonemaster::Engine::Net::IP accepts incomplete IP adresses (network adresses) as valid IP adresses
                 return { status => 'nok', message => encode_entities( "Invalid IP address: [$ns_ip->{ip}]" ) }
                     unless( !$ns_ip->{ip} || $ns_ip->{ip} =~ /^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$/ || $ns_ip->{ip} =~ /^([0-9A-Fa-f]{1,4}:[0-9A-Fa-f:]{1,}(:[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})?)|([0-9A-Fa-f]{1,4}::[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})$/);
 
@@ -345,7 +348,7 @@ sub validate_syntax {
         }
     };
     if ($@) {
-        handle_exception('validate_syntax', $@, '#008');
+        handle_exception('validate_syntax', $@, '008');
     }
     elsif ($result) {
         return $result;
@@ -396,7 +399,7 @@ sub start_domain_test {
         $result = $self->{db}->create_new_test( $params->{domain}, $params, $minutes_between_tests_with_same_params );
     };
     if ($@) {
-        handle_exception('start_domain_test', $@, '#009');
+        handle_exception('start_domain_test', $@, '009');
     }
 
     return $result;
@@ -421,7 +424,7 @@ sub test_progress {
         $result = $self->{db}->test_progress( $test_id );
     };
     if ($@) {
-        handle_exception('test_progress', $@, '#010');
+        handle_exception('test_progress', $@, '010');
     }
 
     return $result;
@@ -440,7 +443,7 @@ sub get_test_params {
         $result = $self->{db}->get_test_params( $test_id );
     };
     if ($@) {
-        handle_exception('get_test_params', $@, '#011');
+        handle_exception('get_test_params', $@, '011');
     }
 
     return $result;
@@ -526,7 +529,7 @@ sub get_test_results {
         $result->{results} = \@zm_results;
     };
     if ($@) {
-        handle_exception('get_test_results', $@, '#012');
+        handle_exception('get_test_results', $@, '012');
     }
 
     $translator->locale( $previous_locale );
@@ -558,7 +561,7 @@ sub get_test_history {
         $results = $self->{db}->get_test_history( $p );
     };
     if ($@) {
-        handle_exception('get_test_history', $@, '#013');
+        handle_exception('get_test_history', $@, '013');
     }
 
     return $results;
@@ -587,7 +590,7 @@ sub add_api_user {
         }
     };
     if ($@) {
-        handle_exception('add_api_user', $@, '#014');
+        handle_exception('add_api_user', $@, '014');
     }
 
     return $result;
@@ -627,7 +630,7 @@ sub add_batch_job {
         $results = $self->{db}->add_batch_job( $params );
     };
     if ($@) {
-        handle_exception('add_batch_job', $@, '#015');
+        handle_exception('add_batch_job', $@, '015');
     }
 
     return $results;
@@ -647,7 +650,7 @@ sub get_batch_job_result {
         $result = $self->{db}->get_batch_job_result($batch_id);
     };
     if ($@) {
-        handle_exception('get_batch_job_result', $@, '#016');
+        handle_exception('get_batch_job_result', $@, '016');
     }
     
     return $result;

From ce4d36e52382d52ec2a51178efc5de86cd3d849e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Tue, 13 Oct 2020 01:29:08 +0200
Subject: [PATCH 70/92] Remove copyright statement

---
 docs/Installation.md | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/docs/Installation.md b/docs/Installation.md
index 7cc269bab..9c54a81db 100644
--- a/docs/Installation.md
+++ b/docs/Installation.md
@@ -736,10 +736,3 @@ database to use it with the new version of Zonemaster-Backend. Please see the
 [Zonemaster::Engine installation]: https://github.com/zonemaster/zonemaster-engine/blob/master/docs/Installation.md
 [Zonemaster::Engine]: https://github.com/zonemaster/zonemaster-engine/blob/master/README.md
 [Zonemaster::LDNS]: https://github.com/zonemaster/zonemaster-ldns/blob/master/README.md
-
-Copyright (c) 2013 - 2017, IIS (The Internet Foundation in Sweden) \
-Copyright (c) 2013 - 2017, AFNIC \
-Creative Commons Attribution 4.0 International License
-
-You should have received a copy of the license along with this
-work.  If not, see <https://creativecommons.org/licenses/by/4.0/>.

From 572200c5fe6219cf9521d877a76f4d52ce9b2861 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Tue, 13 Oct 2020 01:29:39 +0200
Subject: [PATCH 71/92] Let zmtest print testid as soon as possible

---
 share/zmtest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/share/zmtest b/share/zmtest
index 22b9c9d8e..1a2e05a41 100755
--- a/share/zmtest
+++ b/share/zmtest
@@ -56,6 +56,7 @@ done
 # Start test
 output="$(zmb "${server_url}" start_domain_test --domain "${domain}" --ipv4 "${ipv4}" --ipv6 "${ipv6}")" || exit $?
 testid="$(echo "${output}" | "${JQ}" -r .result)" || exit $?
+echo "testid: ${testid}" >&2
 
 if echo "${testid}" | grep -qE '[^0-9a-fA-F]' ; then
     error 1 "start_domain_test did not return a testid: ${testid}"
@@ -76,4 +77,3 @@ done
 
 # Get test results
 zmb "${server_url}" get_test_results --testid "${testid}" --lang "${lang}"
-echo "testid: ${testid}" >&2

From 5125f16de8405784af34346f9c5fa16fdec678ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Tue, 13 Oct 2020 02:22:22 +0200
Subject: [PATCH 72/92] Don't let echo interpret backslashes in JSON

---
 share/zmtest | 37 +++++++++++++++++++------------------
 1 file changed, 19 insertions(+), 18 deletions(-)

diff --git a/share/zmtest b/share/zmtest
index 1a2e05a41..0eac5522a 100755
--- a/share/zmtest
+++ b/share/zmtest
@@ -4,35 +4,36 @@ bindir="$(dirname "$0")"
 
 ZMB="${bindir}/zmb"
 JQ="$(which jq)"
+ECHO=/bin/echo
 
 usage () {
     status="$1"
     message="$2"
-    echo "${message}" >&2
-    echo "Usage: zmtest [OPTIONS] DOMAIN" >&2
-    echo >&2
-    echo "Options:" >&2
-    echo "  -s URL --server URL   Zonemaster Backend to query. Default is http://localhost:5000/" >&2
-    echo "  --noipv4              Run the test without ipv4." >&2
-    echo "  --noipv6              Run the test without ipv6." >&2
-    echo "  --lang LANG           A language tag. Default is \"en\"." >&2
-    echo "                        Valid values are determined by backend_config.ini." >&2
+    "${ECHO}" "${message}" >&2
+    "${ECHO}" "Usage: zmtest [OPTIONS] DOMAIN" >&2
+    "${ECHO}" >&2
+    "${ECHO}" "Options:" >&2
+    "${ECHO}" "  -s URL --server URL   Zonemaster Backend to query. Default is http://localhost:5000/" >&2
+    "${ECHO}" "  --noipv4              Run the test without ipv4." >&2
+    "${ECHO}" "  --noipv6              Run the test without ipv6." >&2
+    "${ECHO}" "  --lang LANG           A language tag. Default is \"en\"." >&2
+    "${ECHO}" "                        Valid values are determined by backend_config.ini." >&2
     exit "${status}"
 }
 
 error () {
     status="$1"
     message="$2"
-    echo "error: ${message}" >&2
+    "${ECHO}" "error: ${message}" >&2
     exit "${status}"
 }
 
 zmb () {
     server_url="$1"; shift
     json="$("${ZMB}" --server="${server_url}" "$@" 2>&1)" || error 1 "method $1: ${json}"
-    json="$(echo "${json}" | "${JQ}" -S . 2>/dev/null)" || error 1 "method $1 did not return valid JSON output"
-    error="$(echo "${json}" | "${JQ}" -e .error 2>/dev/null)" && error 1 "method $1: ${error}"
-    echo "${json}"
+    json="$("${ECHO}" -E "${json}" | "${JQ}" -S . 2>/dev/null)" || error 1 "method $1 did not return valid JSON output"
+    error="$("${ECHO}" -E "${json}" | "${JQ}" -e .error 2>/dev/null)" && error 1 "method $1: ${error}"
+    "${ECHO}" -E "${json}"
 }
 
 [ -n "${JQ}" ] || error 2 "Dependency not found: jq"
@@ -55,10 +56,10 @@ done
 
 # Start test
 output="$(zmb "${server_url}" start_domain_test --domain "${domain}" --ipv4 "${ipv4}" --ipv6 "${ipv6}")" || exit $?
-testid="$(echo "${output}" | "${JQ}" -r .result)" || exit $?
-echo "testid: ${testid}" >&2
+testid="$("${ECHO}" -E "${output}" | "${JQ}" -r .result)" || exit $?
+"${ECHO}" -E "testid: ${testid}" >&2
 
-if echo "${testid}" | grep -qE '[^0-9a-fA-F]' ; then
+if "${ECHO}" -E "${testid}" | grep -qE '[^0-9a-fA-F]' ; then
     error 1 "start_domain_test did not return a testid: ${testid}"
 fi
 
@@ -66,10 +67,10 @@ fi
 while true
 do
     output="$(zmb "${server_url}" test_progress --testid "${testid}")" || exit $?
-    progress="$(echo "${output}" | "${JQ}" -r .result)" || exit $?
+    progress="$("${ECHO}" -E "${output}" | "${JQ}" -r .result)" || exit $?
     printf "\r${progress}%% done" >&2
     if [ "${progress}" -eq 100 ] ; then
-        echo >&2
+        "${ECHO}" >&2
         break
     fi
     sleep 1

From d5164d30cc6ac62d8eaaea8c3a11d37cbab3b4bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Tue, 13 Oct 2020 02:23:38 +0200
Subject: [PATCH 73/92] Perform full test in smoke test

---
 docs/Installation.md | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/docs/Installation.md b/docs/Installation.md
index 9c54a81db..7f2d848e5 100644
--- a/docs/Installation.md
+++ b/docs/Installation.md
@@ -52,6 +52,12 @@ for Zonemaster::Backend, see the [declaration of prerequisites].
 > **Note:** In addition to the normal dependencies, the post-installation
 > smoke test instruction assumes that you have curl installed.
 
+Optionally install jq (only needed for the post-installation smoke test):
+
+```sh
+sudo yum install jq
+```
+
 ## 3. Installation on CentOS
 
 ### 3.1 Install Zonemaster::Backend and related dependencies (CentOS)
@@ -308,10 +314,10 @@ See the [post-installation] section for post-installation matters.
 > **Note:** Zonemaster::LDNS and Zonemaster::Engine are not listed here as they
 > are dealt with in the [prerequisites](#prerequisites) section.
 
-Optionally install Curl (only needed for the post-installation smoke test):
+Optionally install Curl and jq (only needed for the post-installation smoke test):
 
 ```sh
-sudo apt install curl
+sudo apt install curl jq
 ```
 
 Install required locales:
@@ -491,10 +497,10 @@ Install dependencies available from binary packages:
 pkg install p5-Class-Method-Modifiers p5-Config-IniFiles p5-Daemon-Control p5-DBI p5-File-ShareDir p5-File-Slurp p5-HTML-Parser p5-JSON-PP p5-JSON-RPC p5-Moose p5-Parallel-ForkManager p5-Plack p5-Role-Tiny p5-Router-Simple p5-Starman p5-String-ShellQuote databases/p5-DBD-SQLite devel/p5-Log-Dispatch devel/p5-Log-Any devel/p5-Log-Any-Adapter-Dispatch
 ```
 
-Optionally install Curl (only needed for the post-installation smoke test):
+Optionally install Curl and jq (only needed for the post-installation smoke test):
 
 ```sh
-pkg install curl
+pkg install curl jq
 ```
 
 Install dependencies not available from binary packages:
@@ -677,7 +683,8 @@ you should be able to use the
 API on localhost port 5000 as below. The command requires that `curl` is installed.
 
 ```sh
-curl -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","method":"version_info","id":"1"}' http://localhost:5000/ && echo
+cd `perl -MFile::ShareDir -le 'print File::ShareDir::dist_dir("Zonemaster-Backend")'`
+./zmtest zonemaster.net
 ```
 
 The command is expected to give an immediate JSON response similiar to:

From c8d117421df9ab1a4d3dbebd384841567406e1fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Tue, 13 Oct 2020 02:55:44 +0200
Subject: [PATCH 74/92] Update description of the expected output from zmtest

---
 docs/Installation.md | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/docs/Installation.md b/docs/Installation.md
index 7f2d848e5..481d997f1 100644
--- a/docs/Installation.md
+++ b/docs/Installation.md
@@ -687,11 +687,9 @@ cd `perl -MFile::ShareDir -le 'print File::ShareDir::dist_dir("Zonemaster-Backen
 ./zmtest zonemaster.net
 ```
 
-The command is expected to give an immediate JSON response similiar to:
-
-```json
-{"id":"1","jsonrpc":"2.0","result":{"zonemaster_backend":"2.0.2","zonemaster_engine":"v2.0.6"}}
-```
+The command is expected to immediately print out a testid,
+followed by a percentage ticking up from 0% to 100%.
+Once the number reaches 100% a JSON object is printed and zmtest terminates.
 
 
 ### 7.2. What to do next?

From 98a52082efd451b1e166335e122e5750d47aa418 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?=
 <mattias-p@users.noreply.github.com>
Date: Tue, 20 Oct 2020 09:22:48 +0200
Subject: [PATCH 75/92] Remember jq in documentation

Co-authored-by: pnax <70589067+pnax@users.noreply.github.com>
---
 docs/Installation.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/Installation.md b/docs/Installation.md
index 481d997f1..67437a2a3 100644
--- a/docs/Installation.md
+++ b/docs/Installation.md
@@ -680,7 +680,8 @@ Use the procedure for installation on [Debian](#2-installation-on-debian).
 
 If you have followed the installation instructions for Zonemaster::Backend above,
 you should be able to use the
-API on localhost port 5000 as below. The command requires that `curl` is installed.
+API on localhost port 5000 as below. The command requires that `curl` and `jq` are installed.
+
 
 ```sh
 cd `perl -MFile::ShareDir -le 'print File::ShareDir::dist_dir("Zonemaster-Backend")'`

From 99d8a28ff5774dd8fc3e5325090fb02f832808e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Tue, 13 Oct 2020 21:04:56 +0200
Subject: [PATCH 76/92] Use printf instead of echo -E

FreeBSD compatibility
---
 share/zmtest | 37 ++++++++++++++++++-------------------
 1 file changed, 18 insertions(+), 19 deletions(-)

diff --git a/share/zmtest b/share/zmtest
index 0eac5522a..38e9b28c4 100755
--- a/share/zmtest
+++ b/share/zmtest
@@ -4,36 +4,35 @@ bindir="$(dirname "$0")"
 
 ZMB="${bindir}/zmb"
 JQ="$(which jq)"
-ECHO=/bin/echo
 
 usage () {
     status="$1"
     message="$2"
-    "${ECHO}" "${message}" >&2
-    "${ECHO}" "Usage: zmtest [OPTIONS] DOMAIN" >&2
-    "${ECHO}" >&2
-    "${ECHO}" "Options:" >&2
-    "${ECHO}" "  -s URL --server URL   Zonemaster Backend to query. Default is http://localhost:5000/" >&2
-    "${ECHO}" "  --noipv4              Run the test without ipv4." >&2
-    "${ECHO}" "  --noipv6              Run the test without ipv6." >&2
-    "${ECHO}" "  --lang LANG           A language tag. Default is \"en\"." >&2
-    "${ECHO}" "                        Valid values are determined by backend_config.ini." >&2
+    printf "%s" "${message}" >&2
+    echo "Usage: zmtest [OPTIONS] DOMAIN" >&2
+    echo >&2
+    echo "Options:" >&2
+    echo "  -s URL --server URL   Zonemaster Backend to query. Default is http://localhost:5000/" >&2
+    echo "  --noipv4              Run the test without ipv4." >&2
+    echo "  --noipv6              Run the test without ipv6." >&2
+    echo "  --lang LANG           A language tag. Default is \"en\"." >&2
+    echo "                        Valid values are determined by backend_config.ini." >&2
     exit "${status}"
 }
 
 error () {
     status="$1"
     message="$2"
-    "${ECHO}" "error: ${message}" >&2
+    printf "error: %s\n" "${message}" >&2
     exit "${status}"
 }
 
 zmb () {
     server_url="$1"; shift
     json="$("${ZMB}" --server="${server_url}" "$@" 2>&1)" || error 1 "method $1: ${json}"
-    json="$("${ECHO}" -E "${json}" | "${JQ}" -S . 2>/dev/null)" || error 1 "method $1 did not return valid JSON output"
-    error="$("${ECHO}" -E "${json}" | "${JQ}" -e .error 2>/dev/null)" && error 1 "method $1: ${error}"
-    "${ECHO}" -E "${json}"
+    json="$(printf "%s" "${json}" | "${JQ}" -S . 2>/dev/null)" || error 1 "method $1 did not return valid JSON output"
+    error="$(printf "%s" "${json}" | "${JQ}" -e .error 2>/dev/null)" && error 1 "method $1: ${error}"
+    printf "%s" "${json}"
 }
 
 [ -n "${JQ}" ] || error 2 "Dependency not found: jq"
@@ -56,10 +55,10 @@ done
 
 # Start test
 output="$(zmb "${server_url}" start_domain_test --domain "${domain}" --ipv4 "${ipv4}" --ipv6 "${ipv6}")" || exit $?
-testid="$("${ECHO}" -E "${output}" | "${JQ}" -r .result)" || exit $?
-"${ECHO}" -E "testid: ${testid}" >&2
+testid="$(printf "%s" "${output}" | "${JQ}" -r .result)" || exit $?
+printf "testid: %s\n" "${testid}" >&2
 
-if "${ECHO}" -E "${testid}" | grep -qE '[^0-9a-fA-F]' ; then
+if echo "${testid}" | grep -qE '[^0-9a-fA-F]' ; then
     error 1 "start_domain_test did not return a testid: ${testid}"
 fi
 
@@ -67,10 +66,10 @@ fi
 while true
 do
     output="$(zmb "${server_url}" test_progress --testid "${testid}")" || exit $?
-    progress="$("${ECHO}" -E "${output}" | "${JQ}" -r .result)" || exit $?
+    progress="$(printf "%s" "${output}" | "${JQ}" -r .result)" || exit $?
     printf "\r${progress}%% done" >&2
     if [ "${progress}" -eq 100 ] ; then
-        "${ECHO}" >&2
+        echo >&2
         break
     fi
     sleep 1

From f4a2b26c3784ea52f1611ba3aad3eb9c67d26976 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Tue, 13 Oct 2020 21:14:03 +0200
Subject: [PATCH 77/92] Put smoke test command on a single line

---
 docs/Installation.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/Installation.md b/docs/Installation.md
index 67437a2a3..aa04d5b95 100644
--- a/docs/Installation.md
+++ b/docs/Installation.md
@@ -684,8 +684,7 @@ API on localhost port 5000 as below. The command requires that `curl` and `jq` a
 
 
 ```sh
-cd `perl -MFile::ShareDir -le 'print File::ShareDir::dist_dir("Zonemaster-Backend")'`
-./zmtest zonemaster.net
+`perl -MFile::ShareDir -le 'print File::ShareDir::dist_dir("Zonemaster-Backend")'`/zmtest zonemaster.net
 ```
 
 The command is expected to immediately print out a testid,

From bb8718aad33e005734351df5d75e3afd764c6c90 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Wed, 14 Oct 2020 02:49:05 +0200
Subject: [PATCH 78/92] Move jq installation tothe right section

---
 docs/Installation.md | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/docs/Installation.md b/docs/Installation.md
index aa04d5b95..11eb6f9ac 100644
--- a/docs/Installation.md
+++ b/docs/Installation.md
@@ -52,12 +52,6 @@ for Zonemaster::Backend, see the [declaration of prerequisites].
 > **Note:** In addition to the normal dependencies, the post-installation
 > smoke test instruction assumes that you have curl installed.
 
-Optionally install jq (only needed for the post-installation smoke test):
-
-```sh
-sudo yum install jq
-```
-
 ## 3. Installation on CentOS
 
 ### 3.1 Install Zonemaster::Backend and related dependencies (CentOS)
@@ -65,6 +59,12 @@ sudo yum install jq
 > **Note:** Zonemaster::LDNS and Zonemaster::Engine are not listed here as they
 > are dealt with in the [prerequisites](#prerequisites) section.
 
+Optionally install jq (only needed for the post-installation smoke test):
+
+```sh
+sudo yum install jq
+```
+
 Install dependencies available from binary packages:
 
 ```sh

From f233acb9d89fa0c52d6526dd994ff94c9a7ee43e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Wed, 14 Oct 2020 02:57:56 +0200
Subject: [PATCH 79/92] Remove note

---
 docs/Installation.md | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/docs/Installation.md b/docs/Installation.md
index 11eb6f9ac..757b26769 100644
--- a/docs/Installation.md
+++ b/docs/Installation.md
@@ -49,9 +49,6 @@ Prerequisite for FreeBSD is that the package system is upadated and activated
 For details on supported versions of Perl, database engine and operating system
 for Zonemaster::Backend, see the [declaration of prerequisites].
 
-> **Note:** In addition to the normal dependencies, the post-installation
-> smoke test instruction assumes that you have curl installed.
-
 ## 3. Installation on CentOS
 
 ### 3.1 Install Zonemaster::Backend and related dependencies (CentOS)

From 87e36bfe148b68cd98d4bd4a18de4dfc71566238 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Wed, 21 Oct 2020 09:57:50 +0200
Subject: [PATCH 80/92] Empty


From 566b18c706b6b42aa4dd593d6cefa44edfc1513d Mon Sep 17 00:00:00 2001
From: Alexandre Pion <pion@afnic.fr>
Date: Mon, 19 Oct 2020 16:13:05 +0200
Subject: [PATCH 81/92] Fix typo

---
 docs/Installation.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/Installation.md b/docs/Installation.md
index 757b26769..104529652 100644
--- a/docs/Installation.md
+++ b/docs/Installation.md
@@ -43,7 +43,7 @@ Zonemaster::Engine installation].
 > install from your operating system distribution (rather than CPAN).
 > We recommend following the Zonemaster::Engine installation instruction.
 
-Prerequisite for FreeBSD is that the package system is upadated and activated
+Prerequisite for FreeBSD is that the package system is updated and activated
 (see the FreeBSD section of [Zonemaster::Engine installation]).
 
 For details on supported versions of Perl, database engine and operating system

From 634aedb057b7802615977096cfc5bf1f3a29abce Mon Sep 17 00:00:00 2001
From: Alexandre Pion <pion@afnic.fr>
Date: Mon, 19 Oct 2020 17:37:40 +0200
Subject: [PATCH 82/92] Fix link

---
 docs/Installation.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/Installation.md b/docs/Installation.md
index 104529652..b75e02065 100644
--- a/docs/Installation.md
+++ b/docs/Installation.md
@@ -668,7 +668,7 @@ See the [post-installation] section for post-installation matters.
 
 ## 6. Installation on Ubuntu
 
-Use the procedure for installation on [Debian](#2-installation-on-debian).
+Use the procedure for installation on [Debian](#4-installation-on-debian).
 
 
 ## 7. Post-installation

From 63c26fcb35feca5ef22c8458e1822bea48db5ccd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20P=C3=A4iv=C3=A4rinta?= <mattias.paivarinta@iis.se>
Date: Thu, 29 Oct 2020 15:17:16 +0100
Subject: [PATCH 83/92] Improved diagnostics from zmtest when zmb gives
 non-JSON output

---
 share/zmtest | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/share/zmtest b/share/zmtest
index 38e9b28c4..680ce4869 100755
--- a/share/zmtest
+++ b/share/zmtest
@@ -29,8 +29,8 @@ error () {
 
 zmb () {
     server_url="$1"; shift
-    json="$("${ZMB}" --server="${server_url}" "$@" 2>&1)" || error 1 "method $1: ${json}"
-    json="$(printf "%s" "${json}" | "${JQ}" -S . 2>/dev/null)" || error 1 "method $1 did not return valid JSON output"
+    output="$("${ZMB}" --server="${server_url}" "$@" 2>&1)" || error 1 "method $1: ${output}"
+    json="$(printf "%s" "${output}" | "${JQ}" -S . 2>/dev/null)" || error 1 "method $1 did not return valid JSON output: ${output}"
     error="$(printf "%s" "${json}" | "${JQ}" -e .error 2>/dev/null)" && error 1 "method $1: ${error}"
     printf "%s" "${json}"
 }

From aa0f886412f5f1b0d57209dd2083478f4be2d56e Mon Sep 17 00:00:00 2001
From: Sandoche Balakrichenan <sandoche.balakrichenan@afnic.fr>
Date: Tue, 3 Nov 2020 00:15:22 +0100
Subject: [PATCH 84/92] Remove MySQL and add MariaDB installation instructions

---
 docs/Installation.md | 35 +++++++++++++++++++++++++++--------
 1 file changed, 27 insertions(+), 8 deletions(-)

diff --git a/docs/Installation.md b/docs/Installation.md
index 6c6cda62e..f1a208951 100644
--- a/docs/Installation.md
+++ b/docs/Installation.md
@@ -64,6 +64,11 @@ Install dependencies available from binary packages:
 ```sh
 sudo yum install perl-Module-Install perl-IO-CaptureOutput perl-String-ShellQuote
 ```
+*  Only for CentOS 7 *
+
+```sh
+sudo yum install perl-Plack-Test
+```
 
 Install dependencies not available from binary packages:
 
@@ -101,7 +106,7 @@ Check the [declaration of prerequisites] to make sure your preferred combination
 of operating system version and database engine version is supported.
 
 
-#### 3.2.1 Instructions for MySQL (CentOS)
+#### 3.2.1 Instructions for MariaDB (CentOS)
 
 Configure Zonemaster::Backend to use the correct database engine:
 
@@ -111,18 +116,33 @@ sudo sed -i '/\bengine\b/ s/=.*/= MySQL/' /etc/zonemaster/backend_config.ini
 
 > **Note:** See the [backend configuration] documentation for details.
 
-Install, configure and start database engine (and Perl bindings):
+Install the database eengine:
+
+```sh
+sudo yum install mariadb-server
+```
+
+Start the database:
 
 ```sh
-sudo rpm -ivh http://repo.mysql.com/mysql-community-release-el7-5.noarch.rpm
-sudo yum install mysql-server perl-DBD-mysql
-sudo systemctl start mysqld
+sudo systemctl start mariadb
 ```
 
-Verify that MySQL has started:
+Verify that MariaDB has started:
+
+```sh
+sudo systemctl status mariadb
+```
+
+Ensure that MariaDB starts at boot:
+
+```sh
+sudo systemctl enable mariadb
+```
 
+Set the root password in case if it is not done:
 ```sh
-service mysqld status
+sudo mysql_secure_installation
 ```
 
 Initialize the database:
@@ -223,7 +243,6 @@ Check that the service has started:
 ```sh
 sudo /etc/init.d/zm-backend.sh status
 ```
-*Does not return any status as of now*
 
 
 ### 3.4 Post-installation (CentOS)

From 1a5cfb21de230073d20fd31a29acc89dee906d43 Mon Sep 17 00:00:00 2001
From: Mats Dufberg <mats.dufberg@iis.se>
Date: Thu, 5 Nov 2020 16:48:55 +0100
Subject: [PATCH 85/92] Some updates primarily for FreeBSD

---
 docs/Installation.md | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/docs/Installation.md b/docs/Installation.md
index 0b9c86b10..5fdc5ed09 100644
--- a/docs/Installation.md
+++ b/docs/Installation.md
@@ -510,8 +510,10 @@ su -l
 Install dependencies available from binary packages:
 
 ```sh
-pkg install p5-Class-Method-Modifiers p5-Config-IniFiles p5-Daemon-Control p5-DBI p5-File-ShareDir p5-File-Slurp p5-HTML-Parser p5-JSON-PP p5-JSON-RPC p5-Moose p5-Parallel-ForkManager p5-Plack p5-Role-Tiny p5-Router-Simple p5-Starman p5-String-ShellQuote databases/p5-DBD-SQLite devel/p5-Log-Dispatch devel/p5-Log-Any devel/p5-Log-Any-Adapter-Dispatch
+pkg install p5-Class-Method-Modifiers p5-Config-IniFiles p5-Daemon-Control p5-DBI p5-File-ShareDir p5-File-Slurp p5-HTML-Parser p5-JSON-PP p5-JSON-RPC p5-Moose p5-Parallel-ForkManager p5-Plack p5-Role-Tiny p5-Router-Simple p5-Starman p5-String-ShellQuote p5-DBD-SQLite p5-Log-Dispatch p5-Log-Any p5-Log-Any-Adapter-Dispatch p5-JSON-Validator p5-YAML-LibYAML
 ```
+<!-- JSON::Validator requires YAML::PP, but p5-JSON-Validator currently lacks a dependency on p5-YAML-LibYAML -->
+
 
 Optionally install Curl and jq (only needed for the post-installation smoke test):
 
@@ -519,12 +521,6 @@ Optionally install Curl and jq (only needed for the post-installation smoke test
 pkg install curl jq
 ```
 
-Install dependencies not available from binary packages:
-
-```sh
-cpanm JSON::Validator
-```
-
 Install Zonemaster::Backend:
 
 ```sh
@@ -635,7 +631,7 @@ sed -i '' '/[[:<:]]engine[[:>:]]/ s/=.*/= PostgreSQL/' /usr/local/etc/zonemaster
 Install, configure and start database engine (and Perl bindings):
 
 ```sh
-pkg install databases/postgresql11-server databases/p5-DBD-Pg
+pkg install databases/postgresql12-server databases/p5-DBD-Pg
 sysrc postgresql_enable="YES"
 service postgresql initdb
 service postgresql start
@@ -744,6 +740,9 @@ Zonemaster-Backend and keep the database, then you might have to upgrade the
 database to use it with the new version of Zonemaster-Backend. Please see the
 [upgrade][README.md-upgrade] information.
 
+If you keep the database but upgrade the database engine, read the instructions
+for the database engine if any steps have to be take first.
+
 -------
 
 [Backend configuration]: Configuration.md

From 56021d0a1183b91050c60eac98bf9849b11f796f Mon Sep 17 00:00:00 2001
From: Mats Dufberg <mats.dufberg@iis.se>
Date: Fri, 6 Nov 2020 09:48:28 +0100
Subject: [PATCH 86/92] Reverted out-of-scope text about database engines

---
 docs/Installation.md | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/docs/Installation.md b/docs/Installation.md
index 5fdc5ed09..07481a565 100644
--- a/docs/Installation.md
+++ b/docs/Installation.md
@@ -740,9 +740,6 @@ Zonemaster-Backend and keep the database, then you might have to upgrade the
 database to use it with the new version of Zonemaster-Backend. Please see the
 [upgrade][README.md-upgrade] information.
 
-If you keep the database but upgrade the database engine, read the instructions
-for the database engine if any steps have to be take first.
-
 -------
 
 [Backend configuration]: Configuration.md

From 69870e3bb3e17eb35ec3ff2cac9f12fbac069c0f Mon Sep 17 00:00:00 2001
From: Mats Dufberg <mats.dufberg@iis.se>
Date: Fri, 6 Nov 2020 09:50:41 +0100
Subject: [PATCH 87/92] Use package name instead of origin

---
 docs/Installation.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/Installation.md b/docs/Installation.md
index 07481a565..4f8513a29 100644
--- a/docs/Installation.md
+++ b/docs/Installation.md
@@ -631,7 +631,7 @@ sed -i '' '/[[:<:]]engine[[:>:]]/ s/=.*/= PostgreSQL/' /usr/local/etc/zonemaster
 Install, configure and start database engine (and Perl bindings):
 
 ```sh
-pkg install databases/postgresql12-server databases/p5-DBD-Pg
+pkg install postgresql12-server p5-DBD-Pg
 sysrc postgresql_enable="YES"
 service postgresql initdb
 service postgresql start

From 3872a2358519df55d127332697ccb25f573247e2 Mon Sep 17 00:00:00 2001
From: Mats Dufberg <mats.dufberg@iis.se>
Date: Sat, 7 Nov 2020 16:04:57 +0100
Subject: [PATCH 88/92] Updated with changes since last release.

---
 Changes | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/Changes b/Changes
index eabcf387e..7f6e8adc3 100644
--- a/Changes
+++ b/Changes
@@ -1,5 +1,34 @@
 Release history for Zonemaster component Zonemaster-Backend
 
+v6.0.0 2020-11-06 (public release version)
+
+ [Breaking changes]
+ - Updated language tag format in the RPCAPI (#629)
+   - Restricted language tag.
+   - Language tag is set in backend.ini.
+   - New languages can be added without code change.
+   - RPCAPI can report supported language tags.
+
+ [Features]
+ - New tools for command line test (#662, #658, #652, #632,
+   #628, #626, #536, #534)
+ - Improved log handling in RPCAPI (#653, #656, #650, #840)
+ - Improved log handling in testagent (#644, #612)
+
+ [Fixes]
+ - Updated installation instructions (#665, #663, #660, #658,
+   #633, #638, #449, #620)
+ - Corrected MANIFEST (#657)
+ - Improved error handling handling RPCAPI daemon (#545, #213)
+ - Garbage collection testing documentation (#567, #578)
+ - Corrected API documentation (#647, #648)
+ - Updates and corrections in Translator (#655, #649, #631)
+ - Add Norwegian in documentation and configuration (#643)
+ - Clean-up (#642, #598, #597, #639, #638, #641, #193)
+ - Fix warning in test agent (#635, #630, #625, #607)
+ - Fix warning in RPCAPI (#636, #624, #634)
+
+
 v5.0.2 2020-05-22
 
  [Fixes]

From 90813247ff3de5b7d3c1832dd096488e467ca66e Mon Sep 17 00:00:00 2001
From: Mats Dufberg <mats.dufberg@iis.se>
Date: Sat, 7 Nov 2020 16:07:09 +0100
Subject: [PATCH 89/92] Release version

---
 lib/Zonemaster/Backend.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/Zonemaster/Backend.pm b/lib/Zonemaster/Backend.pm
index 6bb172972..40f65299b 100644
--- a/lib/Zonemaster/Backend.pm
+++ b/lib/Zonemaster/Backend.pm
@@ -1,6 +1,6 @@
 package Zonemaster::Backend;
 
-our $VERSION = '5.0.2';
+our $VERSION = '6.0.0';
 
 use strict;
 use warnings;

From 8ec9401cbb852fe47fd749b70f380d1fbcd8ebbe Mon Sep 17 00:00:00 2001
From: Mats Dufberg <mats.dufberg@iis.se>
Date: Sat, 7 Nov 2020 16:08:26 +0100
Subject: [PATCH 90/92] Updated dependencies

---
 Makefile.PL | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile.PL b/Makefile.PL
index 6c6f490ea..056daaabc 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -27,8 +27,8 @@ requires
   'Starman'                     => 0,
   'String::ShellQuote'          => 0,
   'Try::Tiny'                   => 0.24,
-  'Zonemaster::Engine'          => 3.0,
-  'Zonemaster::LDNS'            => 2.0,
+  'Zonemaster::Engine'          => 4.0,
+  'Zonemaster::LDNS'            => 2.1,
   ;
 
 test_requires 'DBD::SQLite';

From 4fd6f37ae24e94abbad80c2e7edf9eb8d5a7612b Mon Sep 17 00:00:00 2001
From: Mats Dufberg <mats.dufberg@iis.se>
Date: Mon, 9 Nov 2020 16:21:09 +0100
Subject: [PATCH 91/92] Updated due to incorrect version number specification.

---
 Changes                   | 5 +++++
 Makefile.PL               | 5 ++++-
 lib/Zonemaster/Backend.pm | 2 +-
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/Changes b/Changes
index 7f6e8adc3..689fac966 100644
--- a/Changes
+++ b/Changes
@@ -1,5 +1,10 @@
 Release history for Zonemaster component Zonemaster-Backend
 
+v6.0.1 2020-11-09 (public release version)
+ [Fixes]
+ - Fixed a version specification error in Makefile.PL.
+
+
 v6.0.0 2020-11-06 (public release version)
 
  [Breaking changes]
diff --git a/Makefile.PL b/Makefile.PL
index 056daaabc..b376c0aa2 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -5,6 +5,9 @@ all_from 'lib/Zonemaster/Backend.pm';
 repository 'https://github.com/zonemaster/zonemaster-backend';
 bugtracker 'https://github.com/zonemaster/zonemaster-backend/issues';
 
+# "2.1.0" could be declared as "2.001" but not as "2.1"
+# (see Zonemaster::LDNS below)
+
 requires
   'Class::Method::Modifiers'    => 1.09,
   'Config::IniFiles'            => 0,
@@ -28,7 +31,7 @@ requires
   'String::ShellQuote'          => 0,
   'Try::Tiny'                   => 0.24,
   'Zonemaster::Engine'          => 4.0,
-  'Zonemaster::LDNS'            => 2.1,
+  'Zonemaster::LDNS'            => 2.001,
   ;
 
 test_requires 'DBD::SQLite';
diff --git a/lib/Zonemaster/Backend.pm b/lib/Zonemaster/Backend.pm
index 40f65299b..80bb272c8 100644
--- a/lib/Zonemaster/Backend.pm
+++ b/lib/Zonemaster/Backend.pm
@@ -1,6 +1,6 @@
 package Zonemaster::Backend;
 
-our $VERSION = '6.0.0';
+our $VERSION = '6.0.1';
 
 use strict;
 use warnings;

From 159e80bc3510582f0d3f028a60a2d44773f19419 Mon Sep 17 00:00:00 2001
From: Mats Dufberg <mats.dufberg@iis.se>
Date: Tue, 10 Nov 2020 01:02:02 +0100
Subject: [PATCH 92/92] Update master to state of develop

---
 .travis.yml                                   |  41 +-
 Changes                                       |  34 ++
 MANIFEST                                      |  10 +-
 MANIFEST.SKIP                                 |   4 +
 Makefile.PL                                   |  12 +-
 docs/API.md                                   | 111 +++-
 docs/Configuration.md                         |  99 ++-
 docs/Installation.md                          | 102 ++--
 .../maintenance/Garbage-Collection-Testing.md | 129 ++++
 lib/Zonemaster/Backend.pm                     |   2 +-
 lib/Zonemaster/Backend/Config.pm              |  71 +++
 lib/Zonemaster/Backend/DB.pm                  |   2 +-
 lib/Zonemaster/Backend/RPCAPI.pm              | 564 +++++++++++-------
 lib/Zonemaster/Backend/Translator.pm          |  31 +-
 lib/Zonemaster/Backend/Validator.pm           |  10 +-
 script/crontab_job_runner/execute_tests.pl    | 110 ----
 .../execute_zonemaster_P10.pl                 |  37 --
 .../execute_zonemaster_P5.pl                  |  36 --
 script/zonemaster_backend_rpcapi.psgi         |  56 +-
 script/zonemaster_backend_testagent           |  73 ++-
 share/backend_config.ini                      |  12 +-
 share/freebsd-pwd.conf                        |   7 +
 share/travis_mysql_backend_config.ini         |   4 +
 share/travis_postgresql_backend_config.ini    |   4 +
 share/travis_sqlite_backend_config.ini        |   3 +
 share/zm-rpcapi.lsb                           |   5 +-
 share/zm-testagent.lsb                        |   6 +-
 share/zm_rpcapi-bsd                           |   1 +
 share/zmb                                     | 440 ++++++++++++++
 share/zmtest                                  |  79 +++
 t/test01.t                                    |   9 +-
 t/test_DB_backend.pl                          |  16 +-
 32 files changed, 1552 insertions(+), 568 deletions(-)
 create mode 100644 docs/internal-documentation/maintenance/Garbage-Collection-Testing.md
 delete mode 100644 script/crontab_job_runner/execute_tests.pl
 delete mode 100644 script/crontab_job_runner/execute_zonemaster_P10.pl
 delete mode 100644 script/crontab_job_runner/execute_zonemaster_P5.pl
 create mode 100644 share/freebsd-pwd.conf
 create mode 100755 share/zmb
 create mode 100755 share/zmtest

diff --git a/.travis.yml b/.travis.yml
index f0eab3b87..507436983 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,21 +1,29 @@
-services: 
-  - mysql
-  - postgresql
-
-env:
-    - TARGET=MySQL ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_mysql_backend_config.ini
-    - TARGET=PostgreSQL ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_postgresql_backend_config.ini
-    - TARGET=SQLite ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_sqlite_backend_config.ini
-    
 language: perl
 
-perl:
-    - "5.30"
-    - "5.28"
-    - "5.26"
-    - "5.24"
-    - "5.22"
-      # Perl 5.16 is temporarily excluded because Travis fails
+jobs:
+  include:
+  # Cover MySQL and PostgreSQL with the latest supported Perl version
+  - perl: "5.32"
+    env: TARGET=MySQL ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_mysql_backend_config.ini
+    services: mysql
+  - perl: "5.32"
+    env: TARGET=PostgreSQL ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_postgresql_backend_config.ini
+    services: postgresql
+  # Cover all Perl versions with SQLite
+  - perl: "5.32"
+    env: TARGET=SQLite ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_sqlite_backend_config.ini
+  - perl: "5.30"
+    env: TARGET=SQLite ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_sqlite_backend_config.ini
+  - perl: "5.28"
+    env: TARGET=SQLite ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_sqlite_backend_config.ini
+  - perl: "5.26"
+    env: TARGET=SQLite ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_sqlite_backend_config.ini
+  - perl: "5.24"
+    env: TARGET=SQLite ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_sqlite_backend_config.ini
+  - perl: "5.22"
+    env: TARGET=SQLite ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_sqlite_backend_config.ini
+  - perl: "5.16"
+    env: TARGET=SQLite ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_sqlite_backend_config.ini
     
 addons:
   apt:
@@ -67,6 +75,7 @@ addons:
     - librole-tiny-perl
     - librouter-simple-perl
     - libstring-shellquote-perl
+    - libtry-tiny-perl
     - starman
 
 before_install:
diff --git a/Changes b/Changes
index eabcf387e..689fac966 100644
--- a/Changes
+++ b/Changes
@@ -1,5 +1,39 @@
 Release history for Zonemaster component Zonemaster-Backend
 
+v6.0.1 2020-11-09 (public release version)
+ [Fixes]
+ - Fixed a version specification error in Makefile.PL.
+
+
+v6.0.0 2020-11-06 (public release version)
+
+ [Breaking changes]
+ - Updated language tag format in the RPCAPI (#629)
+   - Restricted language tag.
+   - Language tag is set in backend.ini.
+   - New languages can be added without code change.
+   - RPCAPI can report supported language tags.
+
+ [Features]
+ - New tools for command line test (#662, #658, #652, #632,
+   #628, #626, #536, #534)
+ - Improved log handling in RPCAPI (#653, #656, #650, #840)
+ - Improved log handling in testagent (#644, #612)
+
+ [Fixes]
+ - Updated installation instructions (#665, #663, #660, #658,
+   #633, #638, #449, #620)
+ - Corrected MANIFEST (#657)
+ - Improved error handling handling RPCAPI daemon (#545, #213)
+ - Garbage collection testing documentation (#567, #578)
+ - Corrected API documentation (#647, #648)
+ - Updates and corrections in Translator (#655, #649, #631)
+ - Add Norwegian in documentation and configuration (#643)
+ - Clean-up (#642, #598, #597, #639, #638, #641, #193)
+ - Fix warning in test agent (#635, #630, #625, #607)
+ - Fix warning in RPCAPI (#636, #624, #634)
+
+
 v5.0.2 2020-05-22
 
  [Fixes]
diff --git a/MANIFEST b/MANIFEST
index 2c79f4f8c..9ec479bca 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -8,10 +8,10 @@ CodeSnippets/sample_api_call__validate_syntax.pl
 CodeSnippets/sample_api_call__version_info.pl
 docs/API.md
 docs/Architecture.md
+docs/Configuration.md
 docs/files-description.md
 docs/GettingStarted.md
 docs/Installation.md
-docs/Configuration.md
 docs/TypographicConventions.md
 docs/upgrade_db_zonemaster_backend_ver_1.0.3.md
 docs/upgrade_db_zonemaster_backend_ver_1.1.0.md
@@ -46,14 +46,12 @@ README.md
 script/add-test.pl
 script/create_db_mysql.pl
 script/create_db_postgresql_9.3.pl
-script/crontab_job_runner/execute_tests.pl
-script/crontab_job_runner/execute_zonemaster_P10.pl
-script/crontab_job_runner/execute_zonemaster_P5.pl
 script/zonemaster_backend_rpcapi.psgi
 script/zonemaster_backend_testagent
 share/backend_config.ini
 share/cleanup-mysql.sql
 share/cleanup-postgres.sql
+share/freebsd-pwd.conf
 share/initial-mysql.sql
 share/initial-postgres.sql
 share/patch_db_README.txt
@@ -71,11 +69,11 @@ share/zm-rpcapi.lsb
 share/zm-testagent.lsb
 share/zm_rpcapi-bsd
 share/zm_testagent-bsd
+share/zmb
+share/zmtest
 share/zonemaster_backend_testagent.conf
 t/test01.data
 t/test01.t
 t/test_DB_backend.pl
 t/test_runner.pl
 t/test_validate_syntax.t
-
-
diff --git a/MANIFEST.SKIP b/MANIFEST.SKIP
index d588b8c47..7321e5032 100644
--- a/MANIFEST.SKIP
+++ b/MANIFEST.SKIP
@@ -6,6 +6,10 @@ maint
 ^.*\.swp$
 ^MANIFEST\.SKIP$
 
+# Avoid experimental tools
+script/zmb
+script/zmtest
+
 #!start included /usr/share/perl/5.20/ExtUtils/MANIFEST.SKIP
 # Avoid version control files.
 \bRCS\b
diff --git a/Makefile.PL b/Makefile.PL
index ed157e9de..b376c0aa2 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -5,6 +5,9 @@ all_from 'lib/Zonemaster/Backend.pm';
 repository 'https://github.com/zonemaster/zonemaster-backend';
 bugtracker 'https://github.com/zonemaster/zonemaster-backend/issues';
 
+# "2.1.0" could be declared as "2.001" but not as "2.1"
+# (see Zonemaster::LDNS below)
+
 requires
   'Class::Method::Modifiers'    => 1.09,
   'Config::IniFiles'            => 0,
@@ -13,7 +16,6 @@ requires
   'File::ShareDir'              => 0,
   'File::Slurp'                 => 0,
   'HTML::Entities'              => 0,
-  'IO::CaptureOutput'           => 0,
   'JSON::PP'                    => 0,
   'JSON::RPC'                   => 1.01,
   'JSON::Validator'             => 3.12,
@@ -21,17 +23,15 @@ requires
   'Log::Any::Adapter::Dispatch' => 0,
   'Log::Dispatch'               => 0,
   'Moose'                       => 2.04,
-  'Net::IP::XS'                 => 0.14,
   'Parallel::ForkManager'       => 1.12,
   'Plack::Builder'              => 0,
-  'Plack::Middleware::Debug'    => 0.14,
   'Role::Tiny'                  => 1.001003,
   'Router::Simple::Declare'     => 0,
   'Starman'                     => 0,
   'String::ShellQuote'          => 0,
-  'Try::Tiny'                   => 0.30,
-  'Zonemaster::Engine'          => 3.0,
-  'Zonemaster::LDNS'            => 2.0,
+  'Try::Tiny'                   => 0.24,
+  'Zonemaster::Engine'          => 4.0,
+  'Zonemaster::LDNS'            => 2.001,
   ;
 
 test_requires 'DBD::SQLite';
diff --git a/docs/API.md b/docs/API.md
index 01b47b0df..fb06cc043 100644
--- a/docs/API.md
+++ b/docs/API.md
@@ -98,7 +98,7 @@ The unique id of a *batch*.
 Basic data type: string
 
 A string of alphanumerics, hyphens, underscores, pluses (`+`), tildes (`~`),
-full stops (`.`), colons (`:`) and spaces (` `), of at least 1 and at most 512
+full stops (`.`), colons (`:`) and spaces (` `), of at least 1 and at most 50
 characters.
 I.e. a string matching `/^[a-zA-Z0-9-+~_.: ]{1,50}$/`.
 
@@ -111,7 +111,7 @@ Used for monitoring which client (GUI) uses the API.
 Basic data type: string
 
 A string of alphanumerics, hyphens, pluses, tildes, underscores, full stops,
-colons and spaces, of at least 1 and at most 512 characters.
+colons and spaces, of at least 1 and at most 50 characters.
 I.e. a string matching `/^[a-zA-Z0-9-+~_.: ]{1,50}$/`.
 
 Represents the version of the client.
@@ -263,25 +263,38 @@ Default database timestamp format: "Y-M-D H:M:S.ms".
 Example: "2017-12-18 07:56:17.156939"
 
 
-### Translation language
+### Language tag
 
 Basic data type: string
 
-A string of alphanumeric, hyphens, underscores, full stops and at-signs (`@`),
-of at least 1 and at most 30 characters.
-I.e. a string matching `/^[a-zA-Z0-9-_.@]{1,30}$/`.
+A string of A-Z, a-z and underscores matching the regular expression
+`/^[a-z]{2}(_[A-Z]{2})?$/`.
 
-* Any string starting with `"fr"` is interpreted as French.
-* Any string starting with `"sv"` is interpreted as Swedish.
-* Any string starting with `"da"` is interpreted as Danish.
-* Any other string is interpreted as English.
+The `language tag` must match a `locale tag` in the configuration file.
+If the `language tag` is a two-character string, it only needs to match the
+first two characters of the `locale tag` from the configuration file, if
+that is unique (there is only one `locale tag` starting with the same two
+characters), else it is an error.
+
+Any other string is an error.
+
+The two first characters of the `language tag` are intended to be an
+[ISO 639-1] two-character language code and the optional two last characters
+are intended to be an [ISO 3166-1 alpha-2] two-character country code.
+
+A default installation will accept the following `language tags`:
+* `da` or `da_DK` for Danish language.
+* `en` or `en_US` for English language.
+* `fr` or `fr_FR` for French language.
+* `nb` or `nb_NO` for Norwegian language.
+* `sv` or `sv_SE` for Swedish language.
 
 
 ### Unsigned integer
 
- Basic data type: number (integer)
+Basic data type: number (integer)
  
- An unsigned integer is either positive or zero.
+An unsigned integer is either positive or zero.
  
 
 ### Username
@@ -290,7 +303,7 @@ Basic data type: string
 
 A string of alphanumerics, dashes, full stops and at-signs, of at least 1 and at
 most 50 characters.
-I.e. a string matching `/^[a-zA-Z0-9]{1,50}$/`.
+I.e. a string matching `/^[a-zA-Z0-9-.@]{1,50}$/`.
 
 Represents the name of an authenticated account (see *[Privilege levels]*)
 
@@ -367,6 +380,52 @@ Example response:
 An array of *Profile names* in lower case. `"default"` is always included.
 
 
+## API method: `get_language_tags`
+
+Returns all valid [language tags][language tag] generated from the setting in
+the configuration file.
+
+Example request:
+```json
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "get_language_tags"
+}
+```
+
+Example response:
+```json
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "result": [
+    "en",
+    "en_US",
+    "fr",
+    "fr_FR",
+    "sv",
+    "sv_SE"
+  ]
+}
+```
+
+
+#### `"result"`
+
+An array of *language tags*. It is never empty.
+
+
+#### `"error"`
+
+>
+> TODO: List all possible error codes and describe what they mean enough for
+> clients to know how react to them. Or prevent RPCAPI from starting with
+> errors in the configuration file and make it not to reread the configuration
+> file while running.
+>
+
+
 ## API method: `get_host_by_name`
 
 Looks up the A and AAAA records for a hostname (*domain name*) on the public Internet.
@@ -418,6 +477,7 @@ value `0.0.0.0` if the lookup returned no A or AAAA records.
 > TODO: If the name resolves to two or more IPv4 address, how is that represented?
 >
 
+
 #### `"error"`
 
 >
@@ -513,7 +573,7 @@ If an identical *test* was already enqueued and hasn't been started or was enque
 no new *test* is enqueued.
 Instead the id for the already enqueued or run test is returned.
 
-*Tests* enqueud using this method are assigned a *priority* of 10.
+*Tests* enqueued using this method are assigned a *priority* of 10.
 
 Example request:
 ```json
@@ -573,13 +633,15 @@ An object with the following properties:
 > TODO: Clarify the purpose of each `"params"` property.
 >
 
+
 #### `"result"`
 
 A *test id*. 
 
 If the test has been run with the same domain name within an interval of 10 mins (hard coded), 
 then the new request does not trigger a new test, but returns with the results of the last test
- 
+
+
 #### `"error"`
 
 * If the given `profile` is not among the [available profiles], a user
@@ -637,7 +699,8 @@ A *progress percentage*.
 
 ## API method: `get_test_results`
 
-Return all *test result* objects of a *test*, with *messages* in the requested *translation language*.
+Return all *test result* objects of a *test*, with *messages* in the requested language as selected by the
+*language tag*.
 
 Example request:
 ```json
@@ -652,6 +715,9 @@ Example request:
 }
 ```
 
+The `id` parameter must match the `result` in the response to a `start_domain_test` call,
+and that test must have been completed.
+
 Example response:
 ```json
 {
@@ -710,7 +776,7 @@ Example response:
 An object with the following properties:
 
 * `"id"`: A *test id*, required.
-* `"language"`: A *translation language*, required.
+* `"language"`: A *language tag*, required.
 
 
 #### `"result"`
@@ -740,7 +806,6 @@ In the case of a test created with `add_batch_job`:
 >
 
 
-
 #### `"error"`
 
 >
@@ -877,10 +942,12 @@ An object with the following properties:
 * `"username"`: An *username*, required. The name of the user to add.
 * `"api_key"`: An *api key*, required. The API key for the user to add.
 
+
 #### `"result"`
 
 An integer. The value is equal to 1 if the registration is a success, or 0 if it failed.
 
+
 #### `"error"`
 >
 > TODO: List all possible error codes and describe what they mean enough for clients to know how react to them.
@@ -1088,6 +1155,7 @@ Example response:
 }
 ```
 
+
 #### `"params"`
 
 An object with the property:
@@ -1106,5 +1174,8 @@ The `"params"` object sent to `start_domain_test` or `add_batch_job` when the *t
 > TODO: List all possible error codes and describe what they mean enough for clients to know how react to them.
 >
 
-[Available profiles]: Configuration.md#profiles-section
-[Privilege levels]: #privilege-levels
+[Available profiles]:           Configuration.md#profiles-section
+[ISO 3166-1 alpha-2]:           https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2
+[ISO 639-1]:                    https://en.wikipedia.org/wiki/ISO_639-1
+[Privilege levels]:             #privilege-levels
+[Language tag]:                 #language-tag
diff --git a/docs/Configuration.md b/docs/Configuration.md
index 16353f263..8dd86fb83 100644
--- a/docs/Configuration.md
+++ b/docs/Configuration.md
@@ -28,10 +28,86 @@ PostgreSQL        | `PostgreSQL`
 SQLite            | `SQLite`
 
 
-## GEOLOCATION section
-
-TBD
-
+## LANGUAGE section
+
+The LANGUAGE section has one key, `locale`.
+
+The value of the `locale` key is a space separated list of
+`locale tags` where each tag must match the regular expression
+`/^[a-z]{2}_[A-Z]{2}$/`.
+
+If the `locale` key is empty or absent, the `locale tag` value
+"en_US" is set by default.
+
+The two first characters of a `locale tag` are intended to be an
+[ISO 639-1] two-character language code and the two last characters
+are intended to be an [ISO 3166-1 alpha-2] two-character country code.
+A `locale tag` is a locale setting for the available translation
+of messages without ".UTF-8", which is implied.
+
+If a new `locale tag` is added to the configuration then the equivalent
+MO file should be added to Zonemaster-Engine at the correct place so
+that gettext can retrieve it, or else the added `locale tag` will not
+add any actual language support. See the
+[Zonemaster-Engine share directory] for the existing PO files that are
+converted to MO files. (Here we should have a link
+to documentation instead.)
+
+Removing a language from the configuration file just blocks that
+language from being allowed. If there are more than one `locale tag`
+(with different country codes) for the same language, then
+all those must be removed to block that language.
+
+English is the Zonemaster default language, but it can be blocked
+from being allowed by RPC-API by not including it in the
+configuration.
+
+In the RPCAPI, `language tag` is used ([Language tag]). The
+`language tags` are generated from the `locale tags`. Each
+`locale tag` will generate two `language tags`, a short tag
+equal to the first two letters (usually the same as a language
+code) and a long tag which is equal to the full `locale tag`.
+If "en_US" is the `locale tag` then "en" and "en_US" are the
+`language tags`.
+
+If there are two `locale tags` that would give the same short
+`language tag` then that is excluded. E.g. "en_US en_UK" will
+only give "en_US" and "en_UK" as `language tags`.
+
+The default installation and configuration supports the
+following languages.
+
+Language | Locale tag value   | Locale value used
+---------|--------------------|------------------
+Danish   | da_DK              | da_DK.UTF-8
+English  | en_US              | en_US.UTF-8
+French   | fr_FR              | fr_FR.UTF-8
+Norwegian| nb_NO              | nb_NO.UTF-8
+Swedish  | sv_SE              | sv_SE.UTF-8
+
+The following `language tags` are generated:
+* da
+* da_DK
+* en
+* en_US
+* fr
+* fr_FR
+* nb
+* nb_NO
+* sv
+* sv_SE
+
+It is an error to repeat the same `locale tag`.
+
+Setting in the default configuration file:
+
+```
+locale = da_DK en_US fr_FR nb_NO sv_SE
+```
+
+Each locale set in the configuration file, including the implied
+".UTF-8", must also be installed or activate on the system
+running the RPCAPI daemon for the translation to work correctly.
 
 ## LOG section
 
@@ -74,11 +150,14 @@ TBD
 
 --------
 
-[Zonemaster::Engine::Profile]: https://metacpan.org/pod/Zonemaster::Engine::Profile#PROFILE-PROPERTIES
-[Default JSON profile file]: https://github.com/zonemaster/zonemaster-engine/blob/master/share/profile.json
-[Profile JSON files]: https://github.com/zonemaster/zonemaster-engine/blob/master/docs/Profiles.md
-[Profile names]: API.md#profile-name
-[Profiles]: Architecture.md#profile
-
+[Default JSON profile file]:          https://github.com/zonemaster/zonemaster-engine/blob/master/share/profile.json
+[ISO 3166-1 alpha-2]:                 https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2
+[ISO 639-1]:                          https://en.wikipedia.org/wiki/ISO_639-1
+[Profile JSON files]:                 https://github.com/zonemaster/zonemaster-engine/blob/master/docs/Profiles.md
+[Profile names]:                      API.md#profile-name
+[Profiles]:                           Architecture.md#profile
+[Zonemaster-Engine share directory]:  https://github.com/zonemaster/zonemaster-engine/tree/master/share
+[Zonemaster::Engine::Profile]:        https://metacpan.org/pod/Zonemaster::Engine::Profile#PROFILE-PROPERTIES
+[Language tag]:                       API.md#language-tag
 
 
diff --git a/docs/Installation.md b/docs/Installation.md
index b8bd4a4d1..4f8513a29 100644
--- a/docs/Installation.md
+++ b/docs/Installation.md
@@ -43,15 +43,12 @@ Zonemaster::Engine installation].
 > install from your operating system distribution (rather than CPAN).
 > We recommend following the Zonemaster::Engine installation instruction.
 
-Prerequisite for FreeBSD is that the package system is upadated and activated
+Prerequisite for FreeBSD is that the package system is updated and activated
 (see the FreeBSD section of [Zonemaster::Engine installation]).
 
 For details on supported versions of Perl, database engine and operating system
 for Zonemaster::Backend, see the [declaration of prerequisites].
 
-> **Note:** In addition to the normal dependencies, the post-installation
-> smoke test instruction assumes that you have curl installed.
-
 ## 3. Installation on CentOS
 
 ### 3.1 Install Zonemaster::Backend and related dependencies (CentOS)
@@ -59,16 +56,27 @@ for Zonemaster::Backend, see the [declaration of prerequisites].
 > **Note:** Zonemaster::LDNS and Zonemaster::Engine are not listed here as they
 > are dealt with in the [prerequisites](#prerequisites) section.
 
+Optionally install jq (only needed for the post-installation smoke test):
+
+```sh
+sudo yum install jq
+```
+
 Install dependencies available from binary packages:
 
 ```sh
-sudo yum install perl-Module-Install perl-IO-CaptureOutput perl-String-ShellQuote perl-Net-Server redhat-lsb-core
+sudo yum install perl-Class-Method-Modifiers perl-Config-IniFiles perl-JSON-RPC perl-Module-Install perl-Parallel-ForkManager perl-Plack perl-Router-Simple perl-String-ShellQuote perl-Net-Server perl-Role-Tiny redhat-lsb-core
+```
+*  Only for CentOS 7 *
+
+```sh
+sudo yum install perl-Plack-Test
 ```
 
 Install dependencies not available from binary packages:
 
 ```sh
-sudo cpanm Class::Method::Modifiers Config::IniFiles Daemon::Control JSON::RPC::Dispatch Net::IP::XS Parallel::ForkManager Plack::Builder Plack::Middleware::Debug Role::Tiny Router::Simple::Declare Starman
+sudo cpanm Daemon::Control Starman
 ```
 
 Install Zonemaster::Backend:
@@ -115,7 +123,7 @@ the old database first.
 If you keep the database, skip the initialization of the Zonemaster database,
 but if you have removed the old Zonemaster database, then do the initialization.
 
-#### 3.2.1 Instructions for MySQL (CentOS)
+#### 3.2.1 Instructions for MariaDB (CentOS)
 
 Configure Zonemaster::Backend to use the correct database engine:
 
@@ -125,18 +133,33 @@ sudo sed -i '/\bengine\b/ s/=.*/= MySQL/' /etc/zonemaster/backend_config.ini
 
 > **Note:** See the [backend configuration] documentation for details.
 
-Install, configure and start database engine (and Perl bindings):
+Install the database eengine:
+
+```sh
+sudo yum install mariadb-server
+```
+
+Start the database:
+
+```sh
+sudo systemctl start mariadb
+```
+
+Verify that MariaDB has started:
 
 ```sh
-sudo rpm -ivh http://repo.mysql.com/mysql-community-release-el7-5.noarch.rpm
-sudo yum install mysql-server perl-DBD-mysql
-sudo systemctl start mysqld
+sudo systemctl status mariadb
 ```
 
-Verify that MySQL has started:
+Ensure that MariaDB starts at boot:
 
 ```sh
-service mysqld status
+sudo systemctl enable mariadb
+```
+
+Set the root password in case if it is not done:
+```sh
+sudo mysql_secure_installation
 ```
 
 Initialize the database (unless you keep an old database):
@@ -293,7 +316,6 @@ Check that the service has started:
 sudo /etc/init.d/zm-rpcapi status
 sudo /etc/init.d/zm-testagent status
 ```
-*Does not return any status as of now*
 
 
 ### 3.4 Post-installation (CentOS)
@@ -308,32 +330,33 @@ See the [post-installation] section for post-installation matters.
 > **Note:** Zonemaster::LDNS and Zonemaster::Engine are not listed here as they
 > are dealt with in the [prerequisites](#prerequisites) section.
 
-Optionally install Curl (only needed for the post-installation smoke test):
+Optionally install Curl and jq (only needed for the post-installation smoke test):
 
 ```sh
-sudo apt install curl
+sudo apt install curl jq
 ```
 
 Install required locales:
 
 ```sh
+locale -a | grep da_DK.utf8 || echo da_DK.UTF-8 UTF-8 | sudo tee -a /etc/locale.gen
 locale -a | grep en_US.utf8 || echo en_US.UTF-8 UTF-8 | sudo tee -a /etc/locale.gen
-locale -a | grep sv_SE.utf8 || echo sv_SE.UTF-8 UTF-8 | sudo tee -a /etc/locale.gen
 locale -a | grep fr_FR.utf8 || echo fr_FR.UTF-8 UTF-8 | sudo tee -a /etc/locale.gen
-locale -a | grep da_DK.utf8 || echo da_DK.UTF-8 UTF-8 | sudo tee -a /etc/locale.gen
+locale -a | grep nb_NO.utf8 || echo nb_NO.UTF-8 UTF-8 | sudo tee -a /etc/locale.gen
+locale -a | grep sv_SE.utf8 || echo sv_SE.UTF-8 UTF-8 | sudo tee -a /etc/locale.gen
 sudo locale-gen
 ```
 
 Install dependencies available from binary packages:
 
 ```sh
-sudo apt install libclass-method-modifiers-perl libconfig-inifiles-perl libdbd-sqlite3-perl libdbi-perl libfile-sharedir-perl libfile-slurp-perl libhtml-parser-perl libio-captureoutput-perl libjson-pp-perl libjson-rpc-perl liblog-any-adapter-dispatch-perl liblog-any-perl liblog-dispatch-perl libmoose-perl libparallel-forkmanager-perl libplack-perl libplack-middleware-debug-perl librole-tiny-perl librouter-simple-perl libstring-shellquote-perl starman
+sudo apt install libclass-method-modifiers-perl libconfig-inifiles-perl libdbd-sqlite3-perl libdbi-perl libfile-sharedir-perl libfile-slurp-perl libhtml-parser-perl libjson-pp-perl libjson-rpc-perl liblog-any-adapter-dispatch-perl liblog-any-perl liblog-dispatch-perl libmoose-perl libparallel-forkmanager-perl libplack-perl libplack-middleware-debug-perl librole-tiny-perl librouter-simple-perl libstring-shellquote-perl libtry-tiny-perl starman
 ```
 
 Install dependencies not available from binary packages:
 
 ```sh
-sudo cpanm Daemon::Control JSON::Validator Net::IP::XS Try::Tiny
+sudo cpanm Daemon::Control JSON::Validator
 ```
 
 Install Zonemaster::Backend:
@@ -487,19 +510,15 @@ su -l
 Install dependencies available from binary packages:
 
 ```sh
-pkg install p5-Class-Method-Modifiers p5-Config-IniFiles p5-Daemon-Control p5-DBI p5-File-ShareDir p5-File-Slurp p5-HTML-Parser p5-IO-CaptureOutput p5-JSON-PP p5-JSON-RPC p5-Moose p5-Parallel-ForkManager p5-Plack p5-Plack-Middleware-Debug p5-Role-Tiny p5-Router-Simple p5-Starman p5-String-ShellQuote net-mgmt/p5-Net-IP-XS databases/p5-DBD-SQLite devel/p5-Log-Dispatch devel/p5-Log-Any devel/p5-Log-Any-Adapter-Dispatch
+pkg install p5-Class-Method-Modifiers p5-Config-IniFiles p5-Daemon-Control p5-DBI p5-File-ShareDir p5-File-Slurp p5-HTML-Parser p5-JSON-PP p5-JSON-RPC p5-Moose p5-Parallel-ForkManager p5-Plack p5-Role-Tiny p5-Router-Simple p5-Starman p5-String-ShellQuote p5-DBD-SQLite p5-Log-Dispatch p5-Log-Any p5-Log-Any-Adapter-Dispatch p5-JSON-Validator p5-YAML-LibYAML
 ```
+<!-- JSON::Validator requires YAML::PP, but p5-JSON-Validator currently lacks a dependency on p5-YAML-LibYAML -->
 
-Optionally install Curl (only needed for the post-installation smoke test):
 
-```sh
-pkg install curl
-```
-
-Install dependencies not available from binary packages:
+Optionally install Curl and jq (only needed for the post-installation smoke test):
 
 ```sh
-cpanm JSON::Validator
+pkg install curl jq
 ```
 
 Install Zonemaster::Backend:
@@ -515,7 +534,8 @@ Unless they already exist, add `zonemaster` user and `zonemaster` group
 (the group is created automatically):
 
 ```sh
-pw useradd zonemaster -s /sbin/nologin -d /nonexistent -c "Zonemaster daemon user"
+cd `perl -MFile::ShareDir -le 'print File::ShareDir::dist_dir("Zonemaster-Backend")'`
+pw useradd zonemaster -C freebsd-pwd.conf -s /sbin/nologin -d /nonexistent -c "Zonemaster daemon user"
 ```
 
 Install files to their proper locations:
@@ -611,7 +631,7 @@ sed -i '' '/[[:<:]]engine[[:>:]]/ s/=.*/= PostgreSQL/' /usr/local/etc/zonemaster
 Install, configure and start database engine (and Perl bindings):
 
 ```sh
-pkg install databases/postgresql11-server databases/p5-DBD-Pg
+pkg install postgresql12-server p5-DBD-Pg
 sysrc postgresql_enable="YES"
 service postgresql initdb
 service postgresql start
@@ -663,7 +683,7 @@ See the [post-installation] section for post-installation matters.
 
 ## 6. Installation on Ubuntu
 
-Use the procedure for installation on [Debian](#2-installation-on-debian).
+Use the procedure for installation on [Debian](#4-installation-on-debian).
 
 
 ## 7. Post-installation
@@ -672,17 +692,16 @@ Use the procedure for installation on [Debian](#2-installation-on-debian).
 
 If you have followed the installation instructions for Zonemaster::Backend above,
 you should be able to use the
-API on localhost port 5000 as below. The command requires that `curl` is installed.
+API on localhost port 5000 as below. The command requires that `curl` and `jq` are installed.
+
 
 ```sh
-curl -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","method":"version_info","id":"1"}' http://localhost:5000/ && echo
+`perl -MFile::ShareDir -le 'print File::ShareDir::dist_dir("Zonemaster-Backend")'`/zmtest zonemaster.net
 ```
 
-The command is expected to give an immediate JSON response similiar to:
-
-```json
-{"id":"1","jsonrpc":"2.0","result":{"zonemaster_backend":"2.0.2","zonemaster_engine":"v2.0.6"}}
-```
+The command is expected to immediately print out a testid,
+followed by a percentage ticking up from 0% to 100%.
+Once the number reaches 100% a JSON object is printed and zmtest terminates.
 
 
 ### 7.2. What to do next?
@@ -734,10 +753,3 @@ database to use it with the new version of Zonemaster-Backend. Please see the
 [Zonemaster::Engine installation]: https://github.com/zonemaster/zonemaster-engine/blob/master/docs/Installation.md
 [Zonemaster::Engine]: https://github.com/zonemaster/zonemaster-engine/blob/master/README.md
 [Zonemaster::LDNS]: https://github.com/zonemaster/zonemaster-ldns/blob/master/README.md
-
-Copyright (c) 2013 - 2017, IIS (The Internet Foundation in Sweden) \
-Copyright (c) 2013 - 2017, AFNIC \
-Creative Commons Attribution 4.0 International License
-
-You should have received a copy of the license along with this
-work.  If not, see <https://creativecommons.org/licenses/by/4.0/>.
diff --git a/docs/internal-documentation/maintenance/Garbage-Collection-Testing.md b/docs/internal-documentation/maintenance/Garbage-Collection-Testing.md
new file mode 100644
index 000000000..392477eb3
--- /dev/null
+++ b/docs/internal-documentation/maintenance/Garbage-Collection-Testing.md
@@ -0,0 +1,129 @@
+# Testing instructions for the Garbage Collection feature of the Zonemaster Backend module
+
+## Introduction
+The purpose of this instruction is to serve as a notice for manual testing of the new garbage collection feature at release time.
+
+## Testing the unfinished tests garbage collection feature
+
+1. Ensure that the database has the required additionnal columns for this feature:
+     ```
+     SELECT nb_retries FROM test_results LIMIT 0;
+     ```
+     Should return:
+
+     ```
+     nb_retries 
+     ------------
+     (0 rows)
+     ```
+     _Remark: for MySQL use `SHOW COLUMNS FROM test_results` and ensure the `nb_retries` column is present in the list._
+
+2. Check that your `/etc/zonemaster/backend_config.ini` (or, in FreeBSD, `/usr/local/etc/zonemaster/backend_config.ini`) has the proper parameter set
+
+     Either disabled:
+     ```
+     #maximal_number_of_retries=3
+     ```
+     or set to 0:
+     ```
+     maximal_number_of_retries=0
+     ```
+
+3. Start a test and wait for it to be finished
+
+     ```
+     SELECT hash_id, progress FROM test_results LIMIT 1;
+     ```
+     Should return:
+
+     ```
+          hash_id      | progress 
+     ------------------+----------
+     3f7a604683efaf93 |      100
+     (1 row)
+     ```
+
+4. Simulate a crashed test
+     ```
+     UPDATE test_results SET progress = 50, test_start_time = '2020-01-01' WHERE hash_id = '3f7a604683efaf93';
+     ```
+
+5. Check that the backend finishes the test with a result stating it was unfinished
+
+     ```
+     SELECT hash_id, progress FROM test_results WHERE hash_id = '3f7a604683efaf93';
+     ```
+     Should return a finished result:
+     ```
+          hash_id      | progress 
+     ------------------+----------
+     3f7a604683efaf93 |      100
+     (1 row)
+     ```
+
+6. Ensure the test result contains the backend generated critical message:
+     ```
+     {"tag":"UNABLE_TO_FINISH_TEST","level":"CRITICAL","timestamp":"300","module":"BACKEND_TEST_AGENT"}
+     ```
+
+     ```
+     SELECT hash_id, progress FROM test_results WHERE hash_id = '3f7a604683efaf93' AND results::text like '%UNABLE_TO_FINISH_TEST%';
+     ```
+     _Remark: for MySQL queries remove the `::text` from all queries_
+
+     Should return:
+     ```
+          hash_id      | progress 
+     ------------------+----------
+     3f7a604683efaf93 |      100
+     (1 row)
+
+     ```
+
+## Testing the unfinished tests garbage collection feature with a number of retries set to allow finishing tests without a critical error
+
+1. Set the maximal_number_of_retries parameter to a value greater than 0 in the backend_config.ini config file 
+     ```
+     maximal_number_of_retries=1
+     ```
+
+2. Restart the test agent in order for the parameter to be taken into account
+     ```
+     zonemaster_backend_testagent restart
+     ```
+     _Remark: Update accordingly to the OS where the tests are done (ex use init script for FreeBSD, etc.)_
+
+3. Simulate a crashed test
+     ```
+     UPDATE test_results SET progress = 50, test_start_time = '2020-01-01' WHERE hash_id = '3f7a604683efaf93';
+     ```
+
+4. Check that the backend finishes the test WITHOUT a result stating it was unfinished
+
+     ```
+     SELECT hash_id, progress FROM test_results WHERE hash_id = '3f7a604683efaf93';
+     ```
+     Should return a finished result:
+     ```
+          hash_id      | progress 
+     ------------------+----------
+     3f7a604683efaf93 |      100
+     (1 row)
+     ```
+
+5. Ensure the test result does NOT contain the backend generated critical message:
+     ```
+     {"tag":"UNABLE_TO_FINISH_TEST","level":"CRITICAL","timestamp":"300","module":"BACKEND_TEST_AGENT"}
+     ```
+
+     ```
+     SELECT hash_id, progress FROM test_results WHERE hash_id = '3f7a604683efaf93' AND results::text like '%UNABLE_TO_FINISH_TEST%';
+     ```
+     _Remark: for MySQL queries remove the `::text` from all queries_
+     Should return:
+     ```
+     hash_id | progress 
+     ---------+----------
+     (0 rows)
+
+     ```
diff --git a/lib/Zonemaster/Backend.pm b/lib/Zonemaster/Backend.pm
index 6bb172972..80bb272c8 100644
--- a/lib/Zonemaster/Backend.pm
+++ b/lib/Zonemaster/Backend.pm
@@ -1,6 +1,6 @@
 package Zonemaster::Backend;
 
-our $VERSION = '5.0.2';
+our $VERSION = '6.0.1';
 
 use strict;
 use warnings;
diff --git a/lib/Zonemaster/Backend/Config.pm b/lib/Zonemaster/Backend/Config.pm
index e091d227c..0cd66063c 100644
--- a/lib/Zonemaster/Backend/Config.pm
+++ b/lib/Zonemaster/Backend/Config.pm
@@ -76,6 +76,77 @@ sub DB_name {
     return $self->{cfg}->val( 'DB', 'database_name' );
 }
 
+=head2 Language_Locale_hash
+
+Read LANGUAGE.locale from the configuration (.ini) file and returns
+the valid language tags for RPCAPI. The incoming language tag
+from RPCAPI is compared to those. The language tags are mapped to
+locale setting value.
+
+=head3 INPUT
+
+None
+
+=head3 RETURNS
+
+A hash of valid language tags as keys with set locale value as value.
+The hash is never empty.
+
+=cut
+
+sub Language_Locale_hash {
+    # There is one special value to capture ambiguous (and therefore
+    # not permitted) translation language tags.
+    my ($self) = @_;
+    my $data = $self->{cfg}->val( 'LANGUAGE', 'locale' );
+    $data = 'en_US' unless $data;
+    my @localetags = split (/\s+/,$data);
+    my %locale;
+    foreach my $la (@localetags) {
+        die "Incorrect locale tag in LANGUAGE.locale: $la\n" unless $la =~ /^[a-z]{2}_[A-Z]{2}$/;
+        die "Repeated locale tag in LANGUAGE.locale: $la\n" if $locale{$la};
+        (my $a) = split (/_/,$la); # $a is the language code only
+        my $lo = "$la.UTF-8";
+        # Set special value if the same language code is used more than once
+        # with different country codes.
+        if ( $locale{$a} and $locale{$a} ne $lo ) {
+            $locale{$a} = 'NOT-UNIQUE';
+        }
+        else {
+            $locale{$a} = $lo;
+        }
+        $locale{$la} = $lo;
+    }
+    return %locale;
+}
+
+=head2 ListLanguageTags
+
+Read indirectly LANGUAGE.locale from the configuration (.ini) file
+and returns a list of valid language tags for RPCAPI. The list can
+be retrieved via an RPCAPI method.
+
+=head3 INPUT
+
+None
+
+=head3 RETURNS
+
+An array of valid language tags. The array is never empty.
+
+=cut
+
+sub ListLanguageTags {
+    my ($self) = @_;
+    my %locale = &Language_Locale_hash($self);
+    my @langtags;
+    foreach my $key (keys %locale) {
+        push @langtags, $key unless $locale{$key} eq 'NOT-UNIQUE';
+    }
+    return @langtags;
+}
+
+
 sub DB_connection_string {
     my ($self) = @_;
 
diff --git a/lib/Zonemaster/Backend/DB.pm b/lib/Zonemaster/Backend/DB.pm
index bacdbdd3b..ebbba82c1 100644
--- a/lib/Zonemaster/Backend/DB.pm
+++ b/lib/Zonemaster/Backend/DB.pm
@@ -128,7 +128,7 @@ sub process_unfinished_tests {
         }
         else {
             my $result;
-            if ($h->{results} =~ /^\[/) {
+            if ( defined $h->{results} && $h->{results} =~ /^\[/ ) {
                 $result = decode_json( $h->{results} );
             }
             else {
diff --git a/lib/Zonemaster/Backend/RPCAPI.pm b/lib/Zonemaster/Backend/RPCAPI.pm
index 14e435cae..d80f7d95a 100644
--- a/lib/Zonemaster/Backend/RPCAPI.pm
+++ b/lib/Zonemaster/Backend/RPCAPI.pm
@@ -10,15 +10,16 @@ use DBI qw(:utils);
 use Digest::MD5 qw(md5_hex);
 use String::ShellQuote;
 use File::Slurp qw(append_file);
-use Zonemaster::LDNS;
-use Net::IP::XS qw(:PROC);
 use HTML::Entities;
-use JSON::Validator "joi";
+use JSON::Validator::Joi;
 
 # Zonemaster Modules
+use Zonemaster::LDNS;
 use Zonemaster::Engine;
-use Zonemaster::Engine::Nameserver;
 use Zonemaster::Engine::DNSName;
+use Zonemaster::Engine::Logger::Entry;
+use Zonemaster::Engine::Nameserver;
+use Zonemaster::Engine::Net::IP;
 use Zonemaster::Engine::Recursor;
 use Zonemaster::Backend;
 use Zonemaster::Backend::Config;
@@ -29,6 +30,10 @@ my $zm_validator = Zonemaster::Backend::Validator->new;
 my %json_schemas;
 my $recursor = Zonemaster::Engine::Recursor->new;
 
+sub joi {
+    return JSON::Validator::Joi->new;
+}
+
 sub new {
     my ( $type, $params ) = @_;
 
@@ -43,7 +48,9 @@ sub new {
             die "$@ \n" if $@;
             $self->{db} = "$params->{db}"->new( { config => $config } );
         };
-        die "$@ \n" if $@;
+        if ($@) {
+            handle_exception('new', "Failed to initialize the [$params->{db}] database backend module: [$@]", '001');
+        }
     }
     else {
         eval {
@@ -52,20 +59,37 @@ sub new {
             die "$@ \n" if $@;
             $self->{db} = $backend_module->new( { config => $config } );
         };
-        die "$@ \n" if $@;
+        if ($@) {
+            handle_exception('new', "Failed to initialize the [$params->{db}] database backend module: [$@]", '002');
+        }
     }
 
     return ( $self );
 }
 
+sub handle_exception {
+    my ( $method, $exception, $exception_id ) = @_;
+
+    $exception =~ s/\n/ /g;
+    $exception =~ s/^\s+|\s+$//g;
+    warn "Internal error $exception_id: Unexpected error in the $method API call: [$exception] \n";
+    die "Internal error $exception_id \n";
+}
+
 $json_schemas{version_info} = joi->object->strict;
 sub version_info {
     my ( $self ) = @_;
 
     my %ver;
-    $ver{zonemaster_engine} = Zonemaster::Engine->VERSION;
-    $ver{zonemaster_backend} = Zonemaster::Backend->VERSION;
+    eval {
+        $ver{zonemaster_engine} = Zonemaster::Engine->VERSION;
+        $ver{zonemaster_backend} = Zonemaster::Backend->VERSION;
 
+    };
+    if ($@) {
+        handle_exception('version_info', $@, '003');
+    }
+    
     return \%ver;
 }
 
@@ -73,23 +97,47 @@ $json_schemas{profile_names} = joi->object->strict;
 sub profile_names {
     my ($self) = @_;
 
-    my @profiles = Zonemaster::Backend::Config->load_config()->ListPublicProfiles();
+    my @profiles;
+    eval {
+        @profiles = Zonemaster::Backend::Config->load_config()->ListPublicProfiles();
 
+    };
+    if ($@) {
+        handle_exception('profile_names', $@, '004');
+    }
+    
     return \@profiles;
 }
 
+# Return the list of language tags supported by get_test_results(). The tags are
+# derived from the locale tags set in the configuration file.
+$json_schemas{get_language_tags} = joi->object->strict;
+sub get_language_tags {
+    my ($self) = @_;
+
+    my @lang = Zonemaster::Backend::Config->load_config()->ListLanguageTags();
+
+    return \@lang;
+}
+
 $json_schemas{get_host_by_name} = joi->object->strict->props(
     hostname   => $zm_validator->domain_name->required
 );
 sub get_host_by_name {
     my ( $self, $params ) = @_;
-    my $ns_name  = $params->{"hostname"};
+    my @adresses;
+    eval {
+        my $ns_name  = $params->{"hostname"};
 
-    my @adresses = map { {$ns_name => $_->short} } $recursor->get_addresses_for($ns_name);
-    @adresses = { $ns_name => '0.0.0.0' } if not @adresses;
+        @adresses = map { {$ns_name => $_->short} } $recursor->get_addresses_for($ns_name);
+        @adresses = { $ns_name => '0.0.0.0' } if not @adresses;
 
+    };
+    if ($@) {
+        handle_exception('get_host_by_name', $@, '005');
+    }
+    
     return \@adresses;
-
 }
 
 $json_schemas{get_data_from_parent_zone} = joi->object->strict->props(
@@ -99,89 +147,106 @@ sub get_data_from_parent_zone {
     my ( $self, $params ) = @_;
     my $domain = "";
 
-    if (ref \$params eq "SCALAR") {
-        $domain = $params;
-    } else {
-        $domain = $params->{"domain"};
-    }
-
-    my %result;
+    my $result = eval {
+		my %result;
+        if (ref \$params eq "SCALAR") {
+            $domain = $params;
+        } else {
+            $domain = $params->{"domain"};
+        }
 
-    my ( $dn, $dn_syntax ) = $self->_check_domain( $domain, 'Domain name' );
-    return $dn_syntax if ( $dn_syntax->{status} eq 'nok' );
+        my ( $dn, $dn_syntax ) = $self->_check_domain( $domain, 'Domain name' );
+        return $dn_syntax if ( $dn_syntax->{status} eq 'nok' );
 
-    my @ns_list;
-    my @ns_names;
+        my @ns_list;
+        my @ns_names;
 
-    my $zone = Zonemaster::Engine->zone( $domain );
-    push @ns_list, { ns => $_->name->string, ip => $_->address->short} for @{$zone->glue};
+        my $zone = Zonemaster::Engine->zone( $domain );
+        push @ns_list, { ns => $_->name->string, ip => $_->address->short} for @{$zone->glue};
 
-    my @ds_list;
+        my @ds_list;
 
-    $zone = Zonemaster::Engine->zone($domain);
-    my $ds_p = $zone->parent->query_one( $zone->name, 'DS', { dnssec => 1, cd => 1, recurse => 1 } );
-    if ($ds_p) {
-        my @ds = $ds_p->get_records( 'DS', 'answer' );
+        $zone = Zonemaster::Engine->zone($domain);
+        my $ds_p = $zone->parent->query_one( $zone->name, 'DS', { dnssec => 1, cd => 1, recurse => 1 } );
+        if ($ds_p) {
+            my @ds = $ds_p->get_records( 'DS', 'answer' );
 
-        foreach my $ds ( @ds ) {
-            next unless $ds->type eq 'DS';
-            push(@ds_list, { keytag => $ds->keytag, algorithm => $ds->algorithm, digtype => $ds->digtype, digest => $ds->hexdigest });
+            foreach my $ds ( @ds ) {
+                next unless $ds->type eq 'DS';
+                push(@ds_list, { keytag => $ds->keytag, algorithm => $ds->algorithm, digtype => $ds->digtype, digest => $ds->hexdigest });
+            }
         }
-    }
 
-    $result{ns_list} = \@ns_list;
-    $result{ds_list} = \@ds_list;
-
-    return \%result;
+        $result{ns_list} = \@ns_list;
+        $result{ds_list} = \@ds_list;
+		return \%result;
+    };
+    if ($@) {
+        handle_exception('get_data_from_parent_zone', $@, '006');
+    }
+    elsif ($result) {
+		return $result;
+    }
 }
 
 
 sub _check_domain {
     my ( $self, $dn, $type ) = @_;
 
-    if ( !defined( $dn ) ) {
-        return ( $dn, { status => 'nok', message => encode_entities( "$type required" ) } );
-    }
+    my $result = eval {
+        if ( !defined( $dn ) ) {
+            return ( $dn, { status => 'nok', message => encode_entities( "$type required" ) } );
+        }
 
-    if ( $dn =~ m/[^[:ascii:]]+/ ) {
-        if ( Zonemaster::LDNS::has_idn() ) {
-            eval { $dn = Zonemaster::LDNS::to_idn( $dn ); };
-            if ( $@ ) {
+        if ( $dn =~ m/[^[:ascii:]]+/ ) {
+            if ( Zonemaster::LDNS::has_idn() ) {
+                eval { $dn = Zonemaster::LDNS::to_idn( $dn ); };
+                if ( $@ ) {
+                    return (
+                        $dn,
+                        {
+                            status  => 'nok',
+                            message => encode_entities( "The domain name is not a valid IDNA string and cannot be converted to an A-label" )
+                        }
+                    );
+                }
+            }
+            else {
                 return (
                     $dn,
                     {
-                        status  => 'nok',
-                        message => encode_entities( "The domain name is not a valid IDNA string and cannot be converted to an A-label" )
+                        status => 'nok',
+                        message =>
+                        encode_entities( "$type contains non-ascii characters and IDNA is not installed" )
                     }
                 );
             }
         }
-        else {
+
+        if( $dn !~ m/^[\-a-zA-Z0-9\.\_]+$/ ) {
             return (
                 $dn,
                 {
-                    status => 'nok',
-                    message =>
-                      encode_entities( "$type contains non-ascii characters and IDNA conversion is not installed" )
+                status  => 'nok',
+                message => encode_entities( "The domain name character(s) not supported")
                 }
             );
         }
-    }
 
-    if( $dn !~ m/^[\-a-zA-Z0-9\.\_]+$/ ) {
-	    return (
-		   $dn,
-		   {
-			   status  => 'nok',
-			   message => encode_entities( "The domain name contains unauthorized character(s)")
-                   }
-            );
+    };
+    if ($@) {
+        handle_exception('_check_domain', $@, '007');
     }
-
+    elsif ($result) {
+		return $result;
+    }
+    
+    my %levels = Zonemaster::Engine::Logger::Entry::levels();
     my @res;
     @res = Zonemaster::Engine::Test::Basic->basic00($dn);
+    @res = grep { $_->numeric_level >= $levels{ERROR} } @res;
     if (@res != 0) {
-        return ( $dn, { status => 'nok', message => encode_entities( "$type name or label outside allowed length" ) } );
+        return ( $dn, { status => 'nok', message => encode_entities( "$type name or label is too long" ) } );
     }
 
     return ( $dn, { status => 'ok', message => 'Syntax ok' } );
@@ -190,95 +255,107 @@ sub _check_domain {
 sub validate_syntax {
     my ( $self, $syntax_input ) = @_;
 
-    my @allowed_params_keys = (
-        'domain',   'ipv4',      'ipv6', 'ds_info', 'nameservers', 'profile',
-        'client_id', 'client_version', 'config', 'priority', 'queue'
-    );
+    my $result = eval {
+        my @allowed_params_keys = (
+            'domain',   'ipv4',      'ipv6', 'ds_info', 'nameservers', 'profile',
+            'client_id', 'client_version', 'config', 'priority', 'queue'
+        );
 
-    foreach my $k ( keys %$syntax_input ) {
-        return { status => 'nok', message => encode_entities( "Unknown option [$k] in parameters" ) }
-          unless ( grep { $_ eq $k } @allowed_params_keys );
-    }
+        foreach my $k ( keys %$syntax_input ) {
+            return { status => 'nok', message => encode_entities( "Unknown option [$k] in parameters" ) }
+            unless ( grep { $_ eq $k } @allowed_params_keys );
+        }
 
-    if ( ( defined $syntax_input->{nameservers} && @{ $syntax_input->{nameservers} } ) ) {
-        foreach my $ns_ip ( @{ $syntax_input->{nameservers} } ) {
-            foreach my $k ( keys %$ns_ip ) {
-                delete( $ns_ip->{$k} ) unless ( $k eq 'ns' || $k eq 'ip' );
+        if ( ( defined $syntax_input->{nameservers} && @{ $syntax_input->{nameservers} } ) ) {
+            foreach my $ns_ip ( @{ $syntax_input->{nameservers} } ) {
+                foreach my $k ( keys %$ns_ip ) {
+                    delete( $ns_ip->{$k} ) unless ( $k eq 'ns' || $k eq 'ip' );
+                }
             }
         }
-    }
 
-    if ( ( defined $syntax_input->{ds_info} && @{ $syntax_input->{ds_info} } ) ) {
-        foreach my $ds_digest ( @{ $syntax_input->{ds_info} } ) {
-            foreach my $k ( keys %$ds_digest ) {
-                delete( $ds_digest->{$k} ) unless ( $k eq 'algorithm' || $k eq 'digest' || $k eq 'digtype' || $k eq 'keytag' );
+        if ( ( defined $syntax_input->{ds_info} && @{ $syntax_input->{ds_info} } ) ) {
+            foreach my $ds_digest ( @{ $syntax_input->{ds_info} } ) {
+                foreach my $k ( keys %$ds_digest ) {
+                    delete( $ds_digest->{$k} ) unless ( $k eq 'algorithm' || $k eq 'digest' || $k eq 'digtype' || $k eq 'keytag' );
+                }
             }
         }
-    }
 
-    if ( defined $syntax_input->{ipv4} ) {
-        return { status => 'nok', message => encode_entities( "Invalid IPv4 transport option format" ) }
-          unless ( $syntax_input->{ipv4} eq JSON::PP::false
-            || $syntax_input->{ipv4} eq JSON::PP::true
-            || $syntax_input->{ipv4} eq '1'
-            || $syntax_input->{ipv4} eq '0' );
-    }
+        if ( defined $syntax_input->{ipv4} ) {
+            return { status => 'nok', message => encode_entities( "Invalid IPv4 transport option format" ) }
+            unless ( $syntax_input->{ipv4} eq JSON::PP::false
+                || $syntax_input->{ipv4} eq JSON::PP::true
+                || $syntax_input->{ipv4} eq '1'
+                || $syntax_input->{ipv4} eq '0' );
+        }
 
-    if ( defined $syntax_input->{ipv6} ) {
-        return { status => 'nok', message => encode_entities( "Invalid IPv6 transport option format" ) }
-          unless ( $syntax_input->{ipv6} eq JSON::PP::false
-            || $syntax_input->{ipv6} eq JSON::PP::true
-            || $syntax_input->{ipv6} eq '1'
-            || $syntax_input->{ipv6} eq '0' );
-    }
+        if ( defined $syntax_input->{ipv6} ) {
+            return { status => 'nok', message => encode_entities( "Invalid IPv6 transport option format" ) }
+            unless ( $syntax_input->{ipv6} eq JSON::PP::false
+                || $syntax_input->{ipv6} eq JSON::PP::true
+                || $syntax_input->{ipv6} eq '1'
+                || $syntax_input->{ipv6} eq '0' );
+        }
 
-    if ( defined $syntax_input->{profile} ) {
-        my @profiles = map lc, Zonemaster::Backend::Config->load_config()->ListPublicProfiles();
-        return { status => 'nok', message => encode_entities( "Invalid profile option format" ) }
-          unless ( grep { $_ eq lc $syntax_input->{profile} } @profiles );
-    }
+        if ( defined $syntax_input->{profile} ) {
+            my @profiles = map lc, Zonemaster::Backend::Config->load_config()->ListPublicProfiles();
+            return { status => 'nok', message => encode_entities( "Invalid profile option format" ) }
+            unless ( grep { $_ eq lc $syntax_input->{profile} } @profiles );
+        }
 
-    my ( $dn, $dn_syntax ) = $self->_check_domain( $syntax_input->{domain}, 'Domain name' );
+        my ( $dn, $dn_syntax ) = $self->_check_domain( $syntax_input->{domain}, 'Domain name' );
 
-    return $dn_syntax if ( $dn_syntax->{status} eq 'nok' );
+        return $dn_syntax if ( $dn_syntax->{status} eq 'nok' );
 
-    if ( defined $syntax_input->{nameservers} && @{ $syntax_input->{nameservers} } ) {
-        foreach my $ns_ip ( @{ $syntax_input->{nameservers} } ) {
-            my ( $ns, $ns_syntax ) = $self->_check_domain( $ns_ip->{ns}, "NS [$ns_ip->{ns}]" );
-            return $ns_syntax if ( $ns_syntax->{status} eq 'nok' );
-        }
+        if ( defined $syntax_input->{nameservers} && @{ $syntax_input->{nameservers} } ) {
+            foreach my $ns_ip ( @{ $syntax_input->{nameservers} } ) {
+                my ( $ns, $ns_syntax ) = $self->_check_domain( $ns_ip->{ns}, "NS [$ns_ip->{ns}]" );
+                return $ns_syntax if ( $ns_syntax->{status} eq 'nok' );
+            }
 
-        foreach my $ns_ip ( @{ $syntax_input->{nameservers} } ) {
-            return { status => 'nok', message => encode_entities( "Invalid IP address: [$ns_ip->{ip}]" ) }
-                unless( !$ns_ip->{ip} || $ns_ip->{ip} =~ /^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$/ || $ns_ip->{ip} =~ /^([0-9A-Fa-f]{1,4}:[0-9A-Fa-f:]{1,}(:[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})?)|([0-9A-Fa-f]{1,4}::[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})$/);
+            foreach my $ns_ip ( @{ $syntax_input->{nameservers} } ) {
+                # Although counterintuitive both tests are necessary as Zonemaster::Engine::Net::IP accepts incomplete IP adresses (network adresses) as valid IP adresses
+                return { status => 'nok', message => encode_entities( "Invalid IP address: [$ns_ip->{ip}]" ) }
+                    unless( !$ns_ip->{ip} || $ns_ip->{ip} =~ /^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$/ || $ns_ip->{ip} =~ /^([0-9A-Fa-f]{1,4}:[0-9A-Fa-f:]{1,}(:[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})?)|([0-9A-Fa-f]{1,4}::[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})$/);
 
-            return { status => 'nok', message => encode_entities( "Invalid IP address: [$ns_ip->{ip}]" ) }
-              unless ( !$ns_ip->{ip} || ip_is_ipv4( $ns_ip->{ip} ) || ip_is_ipv6( $ns_ip->{ip} ) );
-        }
+                return { status => 'nok', message => encode_entities( "Invalid IP address: [$ns_ip->{ip}]" ) }
+                unless ( !$ns_ip->{ip}
+                    || Zonemaster::Engine::Net::IP::ip_is_ipv4( $ns_ip->{ip} )
+                    || Zonemaster::Engine::Net::IP::ip_is_ipv6( $ns_ip->{ip} ) );
+            }
 
-        foreach my $ds_digest ( @{ $syntax_input->{ds_info} } ) {
-            return {
-                status  => 'nok',
-                message => encode_entities( "Invalid algorithm type: [$ds_digest->{algorithm}]" )
-              }
-              if ( $ds_digest->{algorithm} =~ /\D/ );
-        }
+            foreach my $ds_digest ( @{ $syntax_input->{ds_info} } ) {
+                return {
+                    status  => 'nok',
+                    message => encode_entities( "Invalid algorithm type: [$ds_digest->{algorithm}]" )
+                }
+                if ( $ds_digest->{algorithm} =~ /\D/ );
+            }
 
-        foreach my $ds_digest ( @{ $syntax_input->{ds_info} } ) {
-            return {
-                status  => 'nok',
-                message => encode_entities( "Invalid digest format: [$ds_digest->{digest}]" )
+            foreach my $ds_digest ( @{ $syntax_input->{ds_info} } ) {
+                return {
+                    status  => 'nok',
+                    message => encode_entities( "Invalid digest format: [$ds_digest->{digest}]" )
+                }
+                if (
+                    ( length( $ds_digest->{digest} ) != 96 &&
+                        length( $ds_digest->{digest} ) != 64 &&
+                        length( $ds_digest->{digest} ) != 40 ) ||
+                        $ds_digest->{digest} =~ /[^A-Fa-f0-9]/
+                );
             }
-            if (
-                ( length( $ds_digest->{digest} ) != 96 &&
-                      length( $ds_digest->{digest} ) != 64 &&
-                      length( $ds_digest->{digest} ) != 40 ) ||
-                      $ds_digest->{digest} =~ /[^A-Fa-f0-9]/
-            );
         }
+    };
+    if ($@) {
+        handle_exception('validate_syntax', $@, '008');
+    }
+    elsif ($result) {
+        return $result;
+    }
+    else {
+        return { status => 'ok', message =>  encode_entities( 'Syntax ok' ) };
     }
-
-    return { status => 'ok', message => encode_entities( 'Syntax ok' ) };
 }
 
 $json_schemas{start_domain_test} = joi->object->strict->props(
@@ -303,22 +380,27 @@ sub start_domain_test {
 
     my $result = 0;
 
-    $params->{domain} =~ s/^\.// unless ( !$params->{domain} || $params->{domain} eq '.' );
-    my $syntax_result = $self->validate_syntax( $params );
-    die "$syntax_result->{message} \n" unless ( $syntax_result && $syntax_result->{status} eq 'ok' );
+    eval {
+        $params->{domain} =~ s/^\.// unless ( !$params->{domain} || $params->{domain} eq '.' );
+        my $syntax_result = $self->validate_syntax( $params );
+        die "$syntax_result->{message} \n" unless ( $syntax_result && $syntax_result->{status} eq 'ok' );
 
-    die "No domain in parameters\n" unless ( $params->{domain} );
+        die "No domain in parameters\n" unless ( $params->{domain} );
 
-    if ($params->{config}) {
-        $params->{config} =~ s/[^\w_]//isg;
-        die "Unknown test configuration: [$params->{config}]\n" unless ( Zonemaster::Backend::Config->load_config()->GetCustomConfigParameter('ZONEMASTER', $params->{config}) );
-    }
+        if ($params->{config}) {
+            $params->{config} =~ s/[^\w_]//isg;
+            die "Unknown test configuration: [$params->{config}]\n" unless ( Zonemaster::Backend::Config->load_config()->GetCustomConfigParameter('ZONEMASTER', $params->{config}) );
+        }
 
-    $params->{priority}  //= 10;
-    $params->{queue}     //= 0;
-    my $minutes_between_tests_with_same_params = 10;
+        $params->{priority}  //= 10;
+        $params->{queue}     //= 0;
+        my $minutes_between_tests_with_same_params = 10;
 
-    $result = $self->{db}->create_new_test( $params->{domain}, $params, $minutes_between_tests_with_same_params );
+        $result = $self->{db}->create_new_test( $params->{domain}, $params, $minutes_between_tests_with_same_params );
+    };
+    if ($@) {
+        handle_exception('start_domain_test', $@, '009');
+    }
 
     return $result;
 }
@@ -328,16 +410,22 @@ $json_schemas{test_progress} = joi->object->strict->props(
 );
 sub test_progress {
     my ( $self, $params ) = @_;
-    my $test_id = "";
-    if (ref \$params eq "SCALAR") {
-        $test_id = $params;
-    } else {
-        $test_id = $params->{"test_id"};
-    }
-
+    
     my $result = 0;
+    eval {
+        my $test_id = "";
+        if (ref \$params eq "SCALAR") {
+            $test_id = $params;
+        } else {
+            $test_id = $params->{"test_id"};
+        }
 
-    $result = $self->{db}->test_progress( $test_id );
+
+        $result = $self->{db}->test_progress( $test_id );
+    };
+    if ($@) {
+        handle_exception('test_progress', $@, '010');
+    }
 
     return $result;
 }
@@ -351,74 +439,101 @@ sub get_test_params {
 
     my $result = 0;
 
-    $result = $self->{db}->get_test_params( $test_id );
+    eval {
+        $result = $self->{db}->get_test_params( $test_id );
+    };
+    if ($@) {
+        handle_exception('get_test_params', $@, '011');
+    }
 
     return $result;
 }
 
 $json_schemas{get_test_results} = joi->object->strict->props(
     id => $zm_validator->test_id->required,
-    language => $zm_validator->translation_language->required
+    language => $zm_validator->language_tag->required
 );
 sub get_test_results {
     my ( $self, $params ) = @_;
 
     my $result;
-
     my $translator;
     $translator = Zonemaster::Backend::Translator->new;
-    my ( $browser_lang ) = ( $params->{language} =~ /^(\w{2})/ );
+
+    my %locale = Zonemaster::Backend::Config->load_config()->Language_Locale_hash();
+    if ( $locale{$params->{language}} ) {
+        if ( $locale{$params->{language}} eq 'NOT-UNIQUE') {
+            die "Language string not unique: '$params->{language}'\n";
+        }
+    }
+    else {
+        die "Undefined language string: '$params->{language}'\n";
+    }
+
+    my $previous_locale = $translator->locale;
+    $translator->locale( $locale{$params->{language}} );
 
     eval { $translator->data } if $translator;    # Provoke lazy loading of translation data
 
-    my $test_info = $self->{db}->test_results( $params->{id} );
+    my $test_info;
     my @zm_results;
-    foreach my $test_res ( @{ $test_info->{results} } ) {
-        my $res;
-        if ( $test_res->{module} eq 'NAMESERVER' ) {
-            $res->{ns} = ( $test_res->{args}->{ns} ) ? ( $test_res->{args}->{ns} ) : ( 'All' );
-        }
-        elsif ($test_res->{module} eq 'SYSTEM'
-            && $test_res->{tag} eq 'POLICY_DISABLED'
-            && $test_res->{args}->{name} eq 'Example' )
-        {
-            next;
-        }
-
-        $res->{module} = $test_res->{module};
-        $res->{message} = $translator->translate_tag( $test_res, $browser_lang ) . "\n";
-        $res->{message} =~ s/,/, /isg;
-        $res->{message} =~ s/;/; /isg;
-        $res->{level} = $test_res->{level};
-
-        if ( $test_res->{module} eq 'SYSTEM' ) {
-            if ( $res->{message} =~ /policy\.json/ ) {
-                my ( $policy ) = ( $res->{message} =~ /\s(\/.*)$/ );
-                if ( $policy ) {
-                    my $policy_description = 'DEFAULT POLICY';
-                    $policy_description = 'SOME OTHER POLICY' if ( $policy =~ /some\/other\/policy\/path/ );
-                    $res->{message} =~ s/$policy/$policy_description/;
-                }
-                else {
-                    $res->{message} = 'UNKNOWN POLICY FORMAT';
-                }
+    eval{
+        $test_info = $self->{db}->test_results( $params->{id} );
+        foreach my $test_res ( @{ $test_info->{results} } ) {
+            my $res;
+            if ( $test_res->{module} eq 'NAMESERVER' ) {
+                $res->{ns} = ( $test_res->{args}->{ns} ) ? ( $test_res->{args}->{ns} ) : ( 'All' );
+            }
+            elsif ($test_res->{module} eq 'SYSTEM'
+                && $test_res->{tag} eq 'POLICY_DISABLED'
+                && $test_res->{args}->{name} eq 'Example' )
+            {
+                next;
             }
-            elsif ( $res->{message} =~ /config\.json/ ) {
-                my ( $config ) = ( $res->{message} =~ /\s(\/.*)$/ );
-                if ( $config ) {
-                    my $config_description = 'DEFAULT CONFIGURATION';
-                    $config_description = 'SOME OTHER CONFIGURATION' if ( $config =~ /some\/other\/configuration\/path/ );
-                    $res->{message} =~ s/$config/$config_description/;
+
+            $res->{module} = $test_res->{module};
+            $res->{message} = $translator->translate_tag( $test_res, $params->{language} ) . "\n";
+            $res->{message} =~ s/,/, /isg;
+            $res->{message} =~ s/;/; /isg;
+            $res->{level} = $test_res->{level};
+
+            if ( $test_res->{module} eq 'SYSTEM' ) {
+                if ( $res->{message} =~ /policy\.json/ ) {
+                    my ( $policy ) = ( $res->{message} =~ /\s(\/.*)$/ );
+                    if ( $policy ) {
+                        my $policy_description = 'DEFAULT POLICY';
+                        $policy_description = 'SOME OTHER POLICY' if ( $policy =~ /some\/other\/policy\/path/ );
+                        $res->{message} =~ s/$policy/$policy_description/;
+                    }
+                    else {
+                        $res->{message} = 'UNKNOWN POLICY FORMAT';
+                    }
                 }
-                else {
-                    $res->{message} = 'UNKNOWN CONFIG FORMAT';
+                elsif ( $res->{message} =~ /config\.json/ ) {
+                    my ( $config ) = ( $res->{message} =~ /\s(\/.*)$/ );
+                    if ( $config ) {
+                        my $config_description = 'DEFAULT CONFIGURATION';
+                        $config_description = 'SOME OTHER CONFIGURATION' if ( $config =~ /some\/other\/configuration\/path/ );
+                        $res->{message} =~ s/$config/$config_description/;
+                    }
+                    else {
+                        $res->{message} = 'UNKNOWN CONFIG FORMAT';
+                    }
                 }
             }
+
+            push( @zm_results, $res );
         }
 
-        push( @zm_results, $res );
+        $result = $test_info;
+        $result->{results} = \@zm_results;
+    };
+    if ($@) {
+        handle_exception('get_test_results', $@, '012');
     }
 
+    $translator->locale( $previous_locale );
+
     $result = $test_info;
     $result->{results} = \@zm_results;
 
@@ -438,11 +553,16 @@ sub get_test_history {
 
     my $results;
 
-    $p->{offset} //= 0;
-    $p->{limit} //= 200;
-    $p->{filter} //= "all";
+    eval {
+        $p->{offset} //= 0;
+        $p->{limit} //= 200;
+        $p->{filter} //= "all";
 
-    $results = $self->{db}->get_test_history( $p );
+        $results = $self->{db}->get_test_history( $p );
+    };
+    if ($@) {
+        handle_exception('get_test_history', $@, '013');
+    }
 
     return $results;
 }
@@ -456,16 +576,21 @@ sub add_api_user {
 
     my $result = 0;
 
-    my $allow = 0;
-    if ( defined $remote_ip ) {
-        $allow = 1 if ( $remote_ip eq '::1' || $remote_ip eq '127.0.0.1' );
-    }
-    else {
-        $allow = 1;
-    }
+    eval {
+        my $allow = 0;
+        if ( defined $remote_ip ) {
+            $allow = 1 if ( $remote_ip eq '::1' || $remote_ip eq '127.0.0.1' );
+        }
+        else {
+            $allow = 1;
+        }
 
-    if ( $allow ) {
-        $result = 1 if ( $self->{db}->add_api_user( $p->{username}, $p->{api_key} ) eq '1' );
+        if ( $allow ) {
+            $result = 1 if ( $self->{db}->add_api_user( $p->{username}, $p->{api_key} ) eq '1' );
+        }
+    };
+    if ($@) {
+        handle_exception('add_api_user', $@, '014');
     }
 
     return $result;
@@ -500,7 +625,13 @@ sub add_batch_job {
     $params->{test_params}->{priority}  //= 5;
     $params->{test_params}->{queue}     //= 0;
 
-    my $results = $self->{db}->add_batch_job( $params );
+    my $results;
+    eval {
+        $results = $self->{db}->add_batch_job( $params );
+    };
+    if ($@) {
+        handle_exception('add_batch_job', $@, '015');
+    }
 
     return $results;
 }
@@ -511,9 +642,18 @@ $json_schemas{get_batch_job_result} = joi->object->strict->props(
 sub get_batch_job_result {
     my ( $self, $params ) = @_;
 
-    my $batch_id = $params->{"batch_id"};
-
-    return $self->{db}->get_batch_job_result($batch_id);
+    my $result;
+    
+    eval {
+        my $batch_id = $params->{"batch_id"};
+
+        $result = $self->{db}->get_batch_job_result($batch_id);
+    };
+    if ($@) {
+        handle_exception('get_batch_job_result', $@, '016');
+    }
+    
+    return $result;
 }
 
 my $rpc_request = joi->object->props(
diff --git a/lib/Zonemaster/Backend/Translator.pm b/lib/Zonemaster/Backend/Translator.pm
index f250c5f9f..32b974a71 100644
--- a/lib/Zonemaster/Backend/Translator.pm
+++ b/lib/Zonemaster/Backend/Translator.pm
@@ -7,6 +7,7 @@ use 5.14.2;
 use Moose;
 use Encode;
 use POSIX qw[setlocale LC_MESSAGES LC_CTYPE];
+use Zonemaster::Backend::Config;
 
 # Zonemaster Modules
 require Zonemaster::Engine::Translator;
@@ -15,22 +16,9 @@ require Zonemaster::Engine::Logger::Entry;
 extends 'Zonemaster::Engine::Translator';
 
 sub translate_tag {
-    my ( $self, $entry, $browser_lang ) = @_;
-
-    my $previous_locale = $self->locale;
-    if ( $browser_lang eq 'fr' ) {
-        $self->locale( "fr_FR.UTF-8" );
-    }
-    elsif ( $browser_lang eq 'sv' ) {
-        $self->locale( "sv_SE.UTF-8" );
-    }
-    elsif ( $browser_lang eq 'da' ) {
-        $self->locale( "da_DK.UTF-8" );
-    }
-    else {
-        $self->locale( "en_US.UTF-8" );
-    }
+    my ( $self, $hashref, $browser_lang ) = @_;
 
+    # Workaround for broken Zonemaster::Engine::translate_tag in Zonemaster-Engine 3.1.2.
     # Make locale really be set. Fix that makes translation work on FreeBSD 12.1. Solution copied from
     # CLI.pm in the Zonemaster-CLI repository.
     undef $ENV{LANGUAGE};
@@ -44,17 +32,10 @@ sub translate_tag {
         $ENV{LC_ALL} || $ENV{LC_CTYPE};
     }
 
-    my $string = $self->data->{ $entry->{module} }{ $entry->{tag} };
-
-    if ( not $string ) {
-        return $entry->{string};
-    }
+    my $entry = Zonemaster::Engine::Logger::Entry->new( %{ $hashref } );
+    my $octets = Zonemaster::Engine::Translator::translate_tag( $self, $entry );
 
-    my $blessed_entry = bless($entry, 'Zonemaster::Engine::Logger::Entry');
-    my $octets = Zonemaster::Engine::Translator::translate_tag( $self, $blessed_entry );
-    $self->locale( $previous_locale );
-    my $str = decode_utf8( $octets );
-    return $str;
+    return decode_utf8( $octets );
 }
 
 1;
diff --git a/lib/Zonemaster/Backend/Validator.pm b/lib/Zonemaster/Backend/Validator.pm
index 7da59b8fe..e51e32961 100644
--- a/lib/Zonemaster/Backend/Validator.pm
+++ b/lib/Zonemaster/Backend/Validator.pm
@@ -6,7 +6,11 @@ use strict;
 use warnings;
 use 5.14.2;
 
-use JSON::Validator "joi";
+use JSON::Validator::Joi;
+
+sub joi {
+    return JSON::Validator::Joi->new;
+}
 
 sub new {
     my ( $type ) = @_;
@@ -64,8 +68,8 @@ sub queue {
 sub test_id {
     return joi->string->regex('^[0-9]$|^[1-9][0-9]{1,8}$|^[0-9a-f]{16}$');
 }
-sub translation_language {
-    return joi->string->regex('^[a-zA-Z0-9-_.@]{1,30}$');
+sub language_tag {
+    return joi->string->regex('^[a-z]{2}(_[A-Z]{2})?$');
 }
 sub username {
     return joi->string->regex('^[a-zA-Z0-9-.@]{1,50}$');
diff --git a/script/crontab_job_runner/execute_tests.pl b/script/crontab_job_runner/execute_tests.pl
deleted file mode 100644
index 4aeb1c9db..000000000
--- a/script/crontab_job_runner/execute_tests.pl
+++ /dev/null
@@ -1,110 +0,0 @@
-use strict;
-use warnings;
-use utf8;
-use 5.10.1;
-
-use Data::Dumper;
-use DBI qw(:utils);
-use IO::CaptureOutput qw/capture_exec/;
-use POSIX;
-use Time::HiRes;
-use Proc::ProcessTable;
-
-local $| = 1;
-
-use Zonemaster::Backend::Config;
-
-use FindBin qw($RealScript $Script $RealBin $Bin);
-FindBin::again();
-##################################################################
-my $PROJECT_NAME = "zonemaster-backend";
-
-my $SCRITP_DIR = __FILE__;
-$SCRITP_DIR = $Bin unless ($SCRITP_DIR =~ /^\//);
-
-#warn "SCRITP_DIR:$SCRITP_DIR\n";
-#warn "SCRITP_DIR:$SCRITP_DIR\n";
-#warn "RealScript:$RealScript\n";
-#warn "Script:$Script\n";
-#warn "RealBin:$RealBin\n";
-#warn "Bin:$Bin\n";
-#warn "__PACKAGE__:".__PACKAGE__;
-#warn "__FILE__:".__FILE__;
-
-my ($PROD_DIR) = ($SCRITP_DIR =~ /(.*?\/)$PROJECT_NAME/);
-#warn "PROD_DIR:$PROD_DIR\n";
-
-my $PROJECT_BASE_DIR = $PROD_DIR.$PROJECT_NAME."/";
-#warn "PROJECT_BASE_DIR:$PROJECT_BASE_DIR\n";
-unshift(@INC, $PROJECT_BASE_DIR);
-###################################################################
-
-my $JOB_RUNNER_DIR = $PROD_DIR."zonemaster-backend/script/crontab_job_runner/";
-my $LOG_DIR = Zonemaster::Backend::Config->load_config()->LogDir();
-my $perl_command = Zonemaster::Backend::Config->load_config()->PerlInterpreter();
-my $polling_interval = Zonemaster::Backend::Config->load_config()->PollingInterval();
-my $zonemaster_timeout_interval = Zonemaster::Backend::Config->load_config()->MaxZonemasterExecutionTime();
-my $frontend_slots = Zonemaster::Backend::Config->load_config()->NumberOfProcessesForFrontendTesting();
-my $batch_slots = Zonemaster::Backend::Config->load_config()->NumberOfProcessesForBatchTesting();
-
-my $connection_string = Zonemaster::Backend::Config->load_config()->DB_connection_string();
-my $dbh = DBI->connect($connection_string, Zonemaster::Backend::Config->load_config()->DB_user(), Zonemaster::Backend::Config->load_config()->DB_password(), {RaiseError => 1, AutoCommit => 1});
-
-
-sub clean_hung_processes {
-	my $t = new Proc::ProcessTable;
-
-	foreach my $p (@{$t->table}) {
-		if (($p->cmndline =~ /execute_zonemaster_P10\.pl/ || $p->cmndline =~ /execute_zonemaster_P5\.pl/) && $p->cmndline !~ /sh -c/) {
-			if (time() - $p->start > $zonemaster_timeout_interval) {
-				say "Killing hung Zonemaster test process: [".$p->cmndline."]";
-				$p->kill(9);
-			}
-		}
-	}
-}
-
-sub can_start_new_worker {
-	my ($priority, $test_id) = @_;
-	my $result = 0;
-	
-	my @nb_instances = split(/\n+/, `ps -ef | grep "execute_zonemaster_P$priority.pl" | grep -v "sh -c" | grep -v grep | grep -v tail`);
-	my @same_test_id = split(/\n+/, `ps -ef | grep "execute_zonemaster_P$priority.pl $test_id " | grep -v "sh -c" | grep -v grep | grep -v tail`);
-	
-	my $max_slots = 0;
-	if ($priority == 5) {
-		$max_slots = $batch_slots;
-	}
-	elsif ($priority == 10) {
-		$max_slots = $frontend_slots;
-	}
-	
-	$result = 1 if (scalar @nb_instances < $max_slots && !@same_test_id);
-}
-
-sub process_jobs {
-	my ($priority, $start_time) = @_;
-
-	my $query = "SELECT id FROM test_results WHERE progress=0 AND priority=$priority ORDER BY id LIMIT 10";
-	my $sth1 = $dbh->prepare($query);
-	$sth1->execute;
-	while (my $h = $sth1->fetchrow_hashref) {
-		if (can_start_new_worker($priority, $h->{id})) {
-			my $command = "$perl_command $JOB_RUNNER_DIR/execute_zonemaster_P$priority.pl $h->{id} > $LOG_DIR/execute_zonemaster_P$priority"."_$h->{id}_$start_time.log 2>&1 &";
-			say $command;
-			system($command);
-		}
-	}
-	$sth1->finish();
-}
-
-my $start_time = time();
-do {
-	clean_hung_processes();
-	process_jobs(10, $start_time);
-	process_jobs(5, $start_time);
-	say '----------------------- '.strftime("%F %T", localtime()).' ------------------------';
-	Time::HiRes::sleep($polling_interval);
-} while (time() - $start_time < (15*60 - 15));
-
-say "WORKED FOR 15 minutes LEAVING";
diff --git a/script/crontab_job_runner/execute_zonemaster_P10.pl b/script/crontab_job_runner/execute_zonemaster_P10.pl
deleted file mode 100644
index 13d116aeb..000000000
--- a/script/crontab_job_runner/execute_zonemaster_P10.pl
+++ /dev/null
@@ -1,37 +0,0 @@
-use strict;
-use warnings;
-use utf8;
-use 5.10.1;
-
-use Data::Dumper;
-
-use FindBin qw($RealScript $Script $RealBin $Bin);
-FindBin::again();
-##################################################################
-my $PROJECT_NAME = "zonemaster-backend";
-
-my $SCRITP_DIR = __FILE__;
-$SCRITP_DIR = $Bin unless ($SCRITP_DIR =~ /^\//);
-
-#warn "SCRITP_DIR:$SCRITP_DIR\n";
-#warn "SCRITP_DIR:$SCRITP_DIR\n";
-#warn "RealScript:$RealScript\n";
-#warn "Script:$Script\n";
-#warn "RealBin:$RealBin\n";
-#warn "Bin:$Bin\n";
-#warn "__PACKAGE__:".__PACKAGE__;
-#warn "__FILE__:".__FILE__;
-
-my ($PROD_DIR) = ($SCRITP_DIR =~ /(.*?\/)$PROJECT_NAME/);
-#warn "PROD_DIR:$PROD_DIR\n";
-
-my $PROJECT_BASE_DIR = $PROD_DIR.$PROJECT_NAME."/";
-#warn "PROJECT_BASE_DIR:$PROJECT_BASE_DIR\n";
-unshift(@INC, $PROJECT_BASE_DIR);
-##################################################################
-
-unshift(@INC, $PROD_DIR."zonemaster-backend/lib/") unless $INC{$PROD_DIR."zonemaster-backend/lib/"};
-require Zonemaster::Backend::TestAgent;
-
-Zonemaster::Backend::TestAgent->new()->run($ARGV[0]);
-
diff --git a/script/crontab_job_runner/execute_zonemaster_P5.pl b/script/crontab_job_runner/execute_zonemaster_P5.pl
deleted file mode 100644
index 67d7f5c84..000000000
--- a/script/crontab_job_runner/execute_zonemaster_P5.pl
+++ /dev/null
@@ -1,36 +0,0 @@
-use strict;
-use warnings;
-use utf8;
-use 5.10.1;
-
-use Data::Dumper;
-
-use FindBin qw($RealScript $Script $RealBin $Bin);
-FindBin::again();
-##################################################################
-my $PROJECT_NAME = "zonemaster-backend";
-
-my $SCRITP_DIR = __FILE__;
-$SCRITP_DIR = $Bin unless ($SCRITP_DIR =~ /^\//);
-
-#warn "SCRITP_DIR:$SCRITP_DIR\n";
-#warn "SCRITP_DIR:$SCRITP_DIR\n";
-#warn "RealScript:$RealScript\n";
-#warn "Script:$Script\n";
-#warn "RealBin:$RealBin\n";
-#warn "Bin:$Bin\n";
-#warn "__PACKAGE__:".__PACKAGE__;
-#warn "__FILE__:".__FILE__;
-
-my ($PROD_DIR) = ($SCRITP_DIR =~ /(.*?\/)$PROJECT_NAME/);
-#warn "PROD_DIR:$PROD_DIR\n";
-
-my $PROJECT_BASE_DIR = $PROD_DIR.$PROJECT_NAME."/";
-#warn "PROJECT_BASE_DIR:$PROJECT_BASE_DIR\n";
-unshift(@INC, $PROJECT_BASE_DIR);
-##################################################################
-
-unshift(@INC, $PROD_DIR."zonemaster-backend/lib/") unless $INC{$PROD_DIR."zonemaster-backend/lib/"};
-require Zonemaster::Backend::TestAgent;
-
-Zonemaster::Backend::TestAgent->new()->run($ARGV[0]);
diff --git a/script/zonemaster_backend_rpcapi.psgi b/script/zonemaster_backend_rpcapi.psgi
index 4b6dcb2cf..19b12d909 100644
--- a/script/zonemaster_backend_rpcapi.psgi
+++ b/script/zonemaster_backend_rpcapi.psgi
@@ -6,14 +6,18 @@ our $VERSION = '1.1.0';
 
 use 5.14.2;
 
-use JSON::RPC::Dispatch;
-use Router::Simple::Declare;
+use English qw( $PID );
 use JSON::PP;
+use JSON::RPC::Dispatch;
+use Log::Any qw( $log );
+use Log::Any::Adapter;
+use Log::Any::Adapter::Util qw( logging_methods logging_aliases );
+use Log::Dispatch;
 use POSIX;
-use Try::Tiny;
-
 use Plack::Builder;
 use Plack::Response;
+use Router::Simple::Declare;
+use Try::Tiny;
 
 BEGIN { $ENV{PERL_JSON_BACKEND} = 'JSON::PP' };
 
@@ -23,7 +27,6 @@ use Zonemaster::Backend::Config;
 local $| = 1;
 
 builder {
-    enable 'Debug';
     enable sub {
         my $app = shift;
 
@@ -42,11 +45,16 @@ my $router = router {
 	};
 
 	connect "profile_names" => {
-        handler => "+Zonemaster::Backend::RPCAPI",
-        action => "profile_names"
-  };
-  
-  connect "get_host_by_name" => {
+                handler => "+Zonemaster::Backend::RPCAPI",
+                action => "profile_names"
+        };
+
+	connect "get_language_tags" => {
+                handler => "+Zonemaster::Backend::RPCAPI",
+                action => "get_language_tags"
+        };
+
+        connect "get_host_by_name" => {
 		handler => "+Zonemaster::Backend::RPCAPI",
 		action => "get_host_by_name"
 	};
@@ -99,6 +107,34 @@ my $router = router {
 	};
 };
 
+# Returns a Log::Any-compatible log level string, or throws an exception.
+sub get_loglevel {
+    my $value = $ENV{ZM_BACKEND_RPCAPI_LOGLEVEL} || 'warning';
+    for my $method ( logging_methods(), logging_aliases() ) {
+        if ( $value eq $method ) {
+            return $method;
+        }
+    }
+    die "Error: Unrecognized ZM_BACKEND_RPCAPI_LOGLEVEL $value\n";
+}
+
+Log::Any::Adapter->set(
+    'Dispatch',
+    dispatcher => Log::Dispatch->new(
+        outputs => [
+            [
+                'Screen',
+                min_level => get_loglevel(),
+                stderr    => 1,
+                callbacks => sub {
+                    my %args = @_;
+                    $args{message} = sprintf "%s [%d] %s - %s\n", strftime( "%FT%TZ", gmtime ), $PID, uc $args{level}, $args{message};
+                },
+            ],
+        ]
+    ),
+);
+
 my $dispatch = JSON::RPC::Dispatch->new(
 	router => $router,
 );
diff --git a/script/zonemaster_backend_testagent b/script/zonemaster_backend_testagent
index f7f8e9c2b..4f32bebb5 100755
--- a/script/zonemaster_backend_testagent
+++ b/script/zonemaster_backend_testagent
@@ -61,22 +61,51 @@ $loglevel = lc $loglevel;
 
 $loglevel =~ /^(?:trace|debug|info|inform|notice|warning|warn|error|err|critical|crit|fatal|alert|emergency)$/ or die "Error: Unrecognized --loglevel $loglevel\n";
 
-print STDERR "Logging to $logfile\n";
-
-{
-    my $dispatcher = Log::Dispatch->new(outputs => [
-        [
-            'File',
-            min_level => $loglevel,
-            filename => $logfile,
-            mode => '>>',
-            callbacks => sub {
-                my %args = @_;
-                $args{message} = sprintf "%s [%d] %s - %s\n", strftime("%FT%TZ", gmtime), $PID, uc $args{level}, $args{message};
-            },
+# Returns a Log::Dispatch object logging to STDOUT
+#
+# This procedure duplicates the STDOUT file descriptor to make sure that it keeps
+# logging to the same place even if STDOUT is later redirected.
+sub log_dispatcher_dup_stdout {
+    my $min_level = shift;
+
+    open( my $fd, '>&', \*STDOUT ) or die "Can't dup STDOUT: $!";
+    my $handle = IO::Handle->new_from_fd( $fd, "w" ) or die "Can't fdopen duplicated STDOUT: $!";
+    $handle->autoflush(1);
+
+    return Log::Dispatch->new(
+        outputs => [
+            [
+                'Handle',
+                handle    => $handle,
+                min_level => $min_level,
+                callbacks => sub {
+                    my %args = @_;
+                    $args{message} = sprintf "%s: %s\n", uc $args{level}, $args{message};
+                },
+            ],
         ]
-    ]);
-    Log::Any::Adapter->set( 'Dispatch', dispatcher => $dispatcher );
+    );
+}
+
+# Returns a Log::Dispatch object logging to a file
+sub log_dispatcher_file {
+    my $min_level = shift;
+    my $log_file  = shift;
+
+    return Log::Dispatch->new(
+        outputs => [
+            [
+                'File',
+                filename  => $log_file,
+                mode      => '>>',
+                min_level => $min_level,
+                callbacks => sub {
+                    my %args = @_;
+                    $args{message} = sprintf "%s [%d] %s - %s\n", strftime( "%FT%TZ", gmtime ), $PID, uc $args{level}, $args{message};
+                },
+            ],
+        ]
+    );
 }
 
 ###
@@ -136,6 +165,18 @@ sub main {
 # Make sure the environment is alright before forking
 my $initial_config;
 eval {
+    # Initialize logging
+    my $dispatcher;
+    if ( $logfile eq '-' ) {
+        $dispatcher = log_dispatcher_dup_stdout( $loglevel );
+    }
+    else {
+        $dispatcher = log_dispatcher_file( $loglevel, $logfile );
+        print STDERR "zonemaster-testagent logging to file $logfile\n";
+    }
+    Log::Any::Adapter->set( 'Dispatch', dispatcher => $dispatcher );
+
+    # Make sure we can load the configuration file
     $log->debug("Starting pre-flight check");
     $initial_config = Zonemaster::Backend::Config->load_config();
 
@@ -216,6 +257,8 @@ The location of the PID file to use.
 
 The location of the log file to use.
 
+When FILE is -, the log is written to standard output.
+
 =item B<--loglevel=LEVEL>
 
 The location of the log level to use.
diff --git a/share/backend_config.ini b/share/backend_config.ini
index b8a36fd88..05be3f0df 100755
--- a/share/backend_config.ini
+++ b/share/backend_config.ini
@@ -1,3 +1,6 @@
+# For documentation of the backend_config.ini file see
+# https://github.com/zonemaster/zonemaster-backend/blob/master/docs/Configuration.md
+
 [DB]
 engine            = MySQL
 user              = zonemaster
@@ -23,6 +26,9 @@ number_of_processes_for_batch_testing    = 20
 #
 #maximal_number_of_retries=3
 
+[LANGUAGE]
+locale = da_DK en_US fr_FR nb_NO sv_SE
+
 [PUBLIC PROFILES]
 #example_profile_1=/example/directory/test1_profile.json
 #default=/example/directory/default_profile.json
@@ -30,9 +36,3 @@ number_of_processes_for_batch_testing    = 20
 [PRIVATE PROFILES]
 #example_profile_2=/example/directory/test2_profile.json
 
-[GEOLOCATION]
-# Requires the Geo::IP Perl module to be installed
-#maxmind_isp_db_file  =
-
-# Requires the GeoIP2::Database::Reader Perl module to be installed
-#maxmind_city_db_file =
diff --git a/share/freebsd-pwd.conf b/share/freebsd-pwd.conf
new file mode 100644
index 000000000..894a3cb52
--- /dev/null
+++ b/share/freebsd-pwd.conf
@@ -0,0 +1,7 @@
+# Range of free system UIDs that can be used for Zonemaster user
+minuid = 736
+maxuid = 769
+
+# Range of free system GIDs that can be used for Zonemaster group
+mingid = 736
+maxgid = 769
diff --git a/share/travis_mysql_backend_config.ini b/share/travis_mysql_backend_config.ini
index 2cf612a0e..3ae50b45f 100644
--- a/share/travis_mysql_backend_config.ini
+++ b/share/travis_mysql_backend_config.ini
@@ -27,3 +27,7 @@ number_of_processes_for_batch_testing=20
 # upt to that value (the id value given here will be the highest posible value allowed as 
 # simple id, for everything above hash_id will be forced).
 force_hash_id_use_in_API_starting_from_id=1
+
+[LANGUAGE]
+locale = da_DK en_US fr_FR nb_NO sv_SE
+
diff --git a/share/travis_postgresql_backend_config.ini b/share/travis_postgresql_backend_config.ini
index 5a14431a4..51c62eb63 100644
--- a/share/travis_postgresql_backend_config.ini
+++ b/share/travis_postgresql_backend_config.ini
@@ -27,3 +27,7 @@ number_of_professes_for_batch_testing=20
 # upt to that value (the id value given here will be the highest posible value allowed as 
 # simple id, for everything above hash_id will be forced).
 force_hash_id_use_in_API_starting_from_id=1
+
+[LANGUAGE]
+locale = da_DK en_US fr_FR nb_NO sv_SE
+
diff --git a/share/travis_sqlite_backend_config.ini b/share/travis_sqlite_backend_config.ini
index 44098cdc0..832fd0f8c 100644
--- a/share/travis_sqlite_backend_config.ini
+++ b/share/travis_sqlite_backend_config.ini
@@ -21,3 +21,6 @@ number_of_processes_for_frontend_testing=20
 number_of_processes_for_batch_testing=20
 #seconds
 
+[LANGUAGE]
+locale = da_DK en_US fr_FR nb_NO sv_SE
+
diff --git a/share/zm-rpcapi.lsb b/share/zm-rpcapi.lsb
index 1ae491f2e..61ffa06a9 100644
--- a/share/zm-rpcapi.lsb
+++ b/share/zm-rpcapi.lsb
@@ -14,14 +14,15 @@
 ### END INIT INFO
 
 BINDIR=${ZM_BACKEND_BINDIR:-/usr/local/bin}
-LOGFILE=${ZM_BACKEND_LOGDIR:-/var/log/zonemaster}/zm-rpcapi.log
-PIDFILE=${ZM_BACKEND_PIDDIR:-/var/run/zonemaster}/zm-rpcapi.pid
+LOGFILE=${ZM_BACKEND_LOGFILE:-/var/log/zonemaster/zm-rpcapi.log}
+PIDFILE=${ZM_BACKEND_PIDFILE:-/var/run/zonemaster/zm-rpcapi.pid}
 LISTENIP=${ZM_BACKEND_LISTENIP:-127.0.0.1}
 LISTENPORT=${ZM_BACKEND_LISTENPORT:-5000}
 USER=${ZM_BACKEND_USER:-zonemaster}
 GROUP=${ZM_BACKEND_GROUP:-zonemaster}
 
 STARMAN=`PATH="$PATH:/usr/local/bin" /usr/bin/which starman`
+export ZM_BACKEND_RPCAPI_LOGLEVEL='warning' # Lowest level that will be logged
 
 . /lib/lsb/init-functions
 
diff --git a/share/zm-testagent.lsb b/share/zm-testagent.lsb
index 4f0e8d6df..6efd7841d 100644
--- a/share/zm-testagent.lsb
+++ b/share/zm-testagent.lsb
@@ -14,9 +14,9 @@
 ### END INIT INFO
 
 BINDIR=${ZM_BACKEND_BINDIR:-/usr/local/bin}
-LOGFILE=${ZM_BACKEND_LOGDIR:-/var/log/zonemaster}/zm-testagent.log
-OUTFILE=${ZM_BACKEND_LOGDIR:-/var/log/zonemaster}/zm-testagent.out
-PIDFILE=${ZM_BACKEND_PIDDIR:-/var/run/zonemaster}/zm-testagent.pid
+LOGFILE=${ZM_BACKEND_LOGFILE:-/var/log/zonemaster/zm-testagent.log}
+OUTFILE=${ZM_BACKEND_OUTFILE:-/var/log/zonemaster/zm-testagent.out}
+PIDFILE=${ZM_BACKEND_PIDFILE:-/var/run/zonemaster/zm-testagent.pid}
 USER=${ZM_BACKEND_USER:-zonemaster}
 GROUP=${ZM_BACKEND_GROUP:-zonemaster}
 
diff --git a/share/zm_rpcapi-bsd b/share/zm_rpcapi-bsd
index 4f0c0fbe3..d89cb2600 100755
--- a/share/zm_rpcapi-bsd
+++ b/share/zm_rpcapi-bsd
@@ -18,6 +18,7 @@ load_rc_config $name
 : ${zm_rpcapi_listen="127.0.0.1:5000"}
 
 export ZONEMASTER_BACKEND_CONFIG_FILE="/usr/local/etc/zonemaster/backend_config.ini"
+export ZM_BACKEND_RPCAPI_LOGLEVEL='warning' # Lowest level that will be logged
 
 command="/usr/local/bin/starman"
 command_args="--daemonize --user=${zm_rpcapi_user} --group=${zm_rpcapi_group} --pid=${zm_rpcapi_pidfile} --error-log=${zm_rpcapi_logfile} --listen=${zm_rpcapi_listen} --app /usr/local/bin/zonemaster_backend_rpcapi.psgi"
diff --git a/share/zmb b/share/zmb
new file mode 100755
index 000000000..5a292cda8
--- /dev/null
+++ b/share/zmb
@@ -0,0 +1,440 @@
+#!/usr/bin/env perl
+use strict;
+use warnings;
+use feature 'say';
+
+use Getopt::Long qw( GetOptionsFromArray :config require_order );
+
+use Pod::Usage;
+
+use JSON::PP;
+use LWP::UserAgent;
+
+=head1 NAME
+
+B<zmb> - Shell bindings for the Zonemaster::Backend RPC API
+
+Zmb is meant to be pronounced I<Zimba>.
+
+=head1 SYNOPSIS
+
+zmb [GLOBAL OPTIONS] COMMAND [OPTIONS]
+
+=head1 GLOBAL OPTIONS
+
+ --help         Show usage
+ --verbose      Show RPC query
+ --server URL   The server to connect to. Default is http://localhost:5000/.
+
+=cut
+
+sub main {
+    my @argv = @_;
+
+    my $opt_help;
+    my $opt_verbose;
+    my $opt_server = 'http://localhost:5000/';
+    GetOptionsFromArray(
+        \@argv,
+        'help'     => \$opt_help,
+        'verbose'  => \$opt_verbose,
+        'server=s' => \$opt_server,
+    ) or pod2usage( 2 );
+    if ( !@argv ) {
+        pod2usage( -verbose => 99, -sections => ['SYNOPSIS', 'GLOBAL OPTIONS'], -exitval => 'NOEXIT' );
+        show_commands();
+        exit 1;
+    }
+    my $cmd = shift @argv;
+    pod2usage( 1 ) if !defined $cmd;
+    my $cmd_sub = \&{ "cmd_" . $cmd };
+    pod2usage( "'$cmd' is not a command" ) if !defined &$cmd_sub;
+    pod2usage( -verbose => 99, -sections => ["COMMANDS/$cmd"] ) if $opt_help;
+
+    my $json = &$cmd_sub( @argv );
+
+    if ( $json ) {
+        say $json if $opt_verbose;
+        my $request  = to_request( $opt_server, $json );
+        my $response = submit( $request );
+        say $response;
+    }
+}
+
+
+=head1 COMMANDS
+
+=head2 man
+
+Show the full manual page.
+
+ zmb [GLOBAL OPTIONS] man
+
+=cut
+
+sub cmd_man {
+    pod2usage( -verbose => 2 );
+}
+
+
+=head2 non_existing_method
+
+Call a non-existing RPC method.
+
+ zmb [GLOBAL OPTIONS] non_existing_method
+
+=cut
+
+sub cmd_non_existing_method {
+    return to_jsonrpc(
+        id     => 1,
+        method => 'non_existing_method',
+    );
+}
+
+
+=head2 version_info
+
+ zmb [GLOBAL OPTIONS] version_info
+
+=cut
+
+sub cmd_version_info {
+    return to_jsonrpc(
+        id     => 1,
+        method => 'version_info',
+    );
+}
+
+
+=head2 start_domain_test
+
+ zmb [GLOBAL OPTIONS] start_domain_test [OPTIONS]
+
+ Options:
+
+    --domain DOMAIN_NAME
+    --ipv4 true|false|null
+    --ipv6 true|false|null
+    --nameserver DOMAIN_NAME:IP_ADDRESS
+    --ds-info DS_INFO
+    --client-id CLIENT_ID
+    --client-version CLIENT_VERSION
+
+ DS_INFO is a comma separated list of key-value pairs. The expected pairs are:
+
+    keytag=UNSIGNED_INTEGER
+    algorithm=UNSIGNED_INTEGER
+    digtype=UNSIGNED_INTEGER
+    digest=HEX_STRING
+
+=cut
+
+sub cmd_start_domain_test {
+    my @opts = @_;
+
+    my @opt_nameserver;
+    my $opt_domain;
+    my $opt_client_id;
+    my $opt_client_version;
+    my @opt_ds_info;
+    my $opt_ipv4;
+    my $opt_ipv6;
+    GetOptionsFromArray(
+        \@opts,
+        'domain|d=s'       => \$opt_domain,
+        'nameserver|n=s'   => \@opt_nameserver,
+        'client-id=s'      => \$opt_client_id,
+        'client-version=s' => \$opt_client_version,
+        'ds-info=s'        => \@opt_ds_info,
+        'ipv4=s'           => \$opt_ipv4,
+        'ipv6=s'           => \$opt_ipv6,
+    ) or pod2usage( 2 );
+
+    my %params = ( domain => $opt_domain, );
+
+    if ( $opt_client_id ) {
+        $params{client_id} = $opt_client_id,
+    }
+
+    if ( $opt_client_version ) {
+        $params{client_version} = $opt_client_version,
+    }
+
+    if ( @opt_ds_info ) {
+        my @info_objects;
+        for my $property_value_pairs ( @opt_ds_info ) {
+            my %info_object;
+            for my $pair ( split /,/, $property_value_pairs ) {
+                my ( $property, $value ) = split /=/, $pair;
+                if ( $property =~ /^(?:keytag|algorithm|digtype)$/ ) {
+                    $value = 0 + $value;
+                }
+                $info_object{$property} = $value;
+            }
+            push @info_objects, \%info_object;
+        }
+        $params{ds_info} = \@info_objects;
+    }
+
+    if ( @opt_nameserver ) {
+        my @nameserver_objects;
+        for my $domain_ip_pair ( @opt_nameserver ) {
+            my ( $domain, $ip ) = split /:/, $domain_ip_pair, 2;
+            push @nameserver_objects,
+              {
+                ns => $domain,
+                ip => $ip,
+              };
+        }
+        $params{nameservers} = \@nameserver_objects;
+    }
+
+    if ( $opt_ipv4 ) {
+        $params{ipv4} = json_tern( $opt_ipv4 );
+    }
+
+    if ( $opt_ipv6 ) {
+        $params{ipv6} = json_tern( $opt_ipv6 );
+    }
+
+    return to_jsonrpc(
+        id     => 1,
+        method => 'start_domain_test',
+        params => \%params,
+    );
+}
+
+
+=head2 test_progress
+
+ zmb [GLOBAL OPTIONS] test_progress [OPTIONS]
+
+ Options:
+   --testid TEST_ID
+
+=cut
+
+sub cmd_test_progress {
+    my @opts = @_;
+
+    my $opt_lang;
+    my $opt_testid;
+    GetOptionsFromArray( \@opts, 'testid|t=s' => \$opt_testid, )
+      or pod2usage( 2 );
+
+    return to_jsonrpc(
+        id     => 1,
+        method => 'test_progress',
+        params => {
+            test_id => $opt_testid,
+        },
+    );
+}
+
+
+=head2 get_test_results
+
+ zmb [GLOBAL OPTIONS] get_test_results [OPTIONS]
+
+ Options:
+   --testid TEST_ID
+   --lang LANGUAGE
+
+=cut
+
+sub cmd_get_test_results {
+    my @opts = @_;
+
+    my $opt_lang;
+    my $opt_testid;
+    GetOptionsFromArray(
+        \@opts,
+        'testid|t=s' => \$opt_testid,
+        'lang|l=s'   => \$opt_lang,
+    ) or pod2usage( 2 );
+
+    return to_jsonrpc(
+        id     => 1,
+        method => 'get_test_results',
+        params => {
+            id       => $opt_testid,
+            language => $opt_lang,
+        },
+    );
+}
+
+
+=head2 get_test_history
+
+ zmb [GLOBAL OPTIONS] get_test_history [OPTIONS]
+
+ Options:
+   --domain DOMAIN_NAME
+   --nameserver true|false|null
+   --offset COUNT
+   --limit COUNT
+
+=cut
+
+sub cmd_get_test_history {
+    my @opts = @_;
+    my $opt_nameserver;
+    my $opt_domain;
+    my $opt_offset;
+    my $opt_limit;
+
+    GetOptionsFromArray(
+        \@opts,
+        'domain|d=s'     => \$opt_domain,
+        'nameserver|n=s' => \$opt_nameserver,
+        'offset|o=i'     => \$opt_offset,
+        'limit|l=i'      => \$opt_limit,
+    ) or pod2usage( 2 );
+
+    my %params = (
+        frontend_params => {
+            domain => $opt_domain,
+        },
+    );
+
+    if ( $opt_nameserver ) {
+        $params{frontend_params}{nameservers} = json_tern( $opt_nameserver );
+    }
+
+    if ( defined $opt_offset ) {
+        $params{offset} = $opt_offset;
+    }
+
+    if ( defined $opt_limit ) {
+        $params{limit} = $opt_limit;
+    }
+
+    return to_jsonrpc(
+        id     => 1,
+        method => 'get_test_history',
+        params => \%params,
+    );
+}
+
+
+=head2 add_api_user
+
+ zmb [GLOBAL OPTIONS] add_api_user [OPTIONS]
+
+ Options:
+   --username USERNAME
+   --api-key API_KEY
+
+=cut
+
+sub cmd_add_api_user {
+    my @opts = @_;
+
+    my $opt_username;
+    my $opt_api_key;
+    GetOptionsFromArray(
+        \@opts,
+        'username|u=s' => \$opt_username,
+        'api-key|a=s'  => \$opt_api_key,
+    ) or pod2usage( 2 );
+
+    return to_jsonrpc(
+        id     => 1,
+        method => 'add_api_user',
+        params => {
+            username => $opt_username,
+            api_key  => $opt_api_key,
+        },
+    );
+}
+
+
+sub show_commands {
+    my %specials = (
+        man                 => 'Show the full manual page.',
+        non_existing_method => 'Call a non-existing RPC method.',
+    );
+    my @commands  = get_commands();
+    my $max_width = 0;
+    for my $command ( @commands ) {
+        $max_width = length $command if length $command > $max_width;
+    }
+    say "Commands:";
+    for my $command ( @commands ) {
+        if ( exists $specials{$command} ) {
+            printf "     %-*s     %s\n", $max_width, $command, $specials{$command};
+        }
+        else {
+            say "     ", $command;
+        }
+    }
+}
+
+
+sub get_commands {
+    no strict 'refs';
+
+    return sort
+      map { $_ =~ s/^cmd_//r }
+      grep { $_ =~ /^cmd_/ } grep { defined &{"main\::$_"} } keys %{"main\::"};
+}
+
+sub json_tern {
+    my $value = shift;
+    if ( $value eq 'true' ) {
+        return JSON::PP::true;
+    }
+    elsif ( $value eq 'false' ) {
+        return JSON::PP::false;
+    }
+    elsif ( $value eq 'null' ) {
+        return undef;
+    }
+    else {
+        die "unknown ternary value";
+    }
+}
+
+sub to_jsonrpc {
+    my %args   = @_;
+    my $id     = $args{id};
+    my $method = $args{method};
+
+    my $request = {
+        jsonrpc => "2.0",
+        method  => $method,
+        id      => $id,
+    };
+    if ( exists $args{params} ) {
+        $request->{params} = $args{params};
+    }
+    return encode_json( $request );
+}
+
+sub to_request {
+    my $server = shift;
+    my $json   = shift;
+
+    my $req = HTTP::Request->new( POST => $server );
+    $req->content_type( 'application/json' );
+    $req->content( $json );
+
+    return $req;
+}
+
+sub submit {
+    my $req = shift;
+
+    my $ua  = LWP::UserAgent->new;
+    my $res = $ua->request( $req );
+
+    if ( $res->is_success ) {
+        return $res->decoded_content;
+    }
+    else {
+        die $res->status_line;
+    }
+}
+
+main( @ARGV );
diff --git a/share/zmtest b/share/zmtest
new file mode 100755
index 000000000..680ce4869
--- /dev/null
+++ b/share/zmtest
@@ -0,0 +1,79 @@
+#!/bin/sh
+
+bindir="$(dirname "$0")"
+
+ZMB="${bindir}/zmb"
+JQ="$(which jq)"
+
+usage () {
+    status="$1"
+    message="$2"
+    printf "%s" "${message}" >&2
+    echo "Usage: zmtest [OPTIONS] DOMAIN" >&2
+    echo >&2
+    echo "Options:" >&2
+    echo "  -s URL --server URL   Zonemaster Backend to query. Default is http://localhost:5000/" >&2
+    echo "  --noipv4              Run the test without ipv4." >&2
+    echo "  --noipv6              Run the test without ipv6." >&2
+    echo "  --lang LANG           A language tag. Default is \"en\"." >&2
+    echo "                        Valid values are determined by backend_config.ini." >&2
+    exit "${status}"
+}
+
+error () {
+    status="$1"
+    message="$2"
+    printf "error: %s\n" "${message}" >&2
+    exit "${status}"
+}
+
+zmb () {
+    server_url="$1"; shift
+    output="$("${ZMB}" --server="${server_url}" "$@" 2>&1)" || error 1 "method $1: ${output}"
+    json="$(printf "%s" "${output}" | "${JQ}" -S . 2>/dev/null)" || error 1 "method $1 did not return valid JSON output: ${output}"
+    error="$(printf "%s" "${json}" | "${JQ}" -e .error 2>/dev/null)" && error 1 "method $1: ${error}"
+    printf "%s" "${json}"
+}
+
+[ -n "${JQ}" ] || error 2 "Dependency not found: jq"
+
+domain=""
+server_url="http://localhost:5000/"
+ipv4="true"
+ipv6="true"
+lang="en"
+while [ $# -gt 0 ] ; do
+    case "$1" in
+        -s|--server) server_url="$2"; shift 2;;
+        --noipv4) ipv4="false"; shift 1;;
+        --noipv6) ipv6="false"; shift 1;;
+        --lang) lang="$2"; shift 2;;
+        *) domain="$1" ; shift 1;;
+    esac
+done
+[ -n "${domain}" ] || usage 2 "No domain specified"
+
+# Start test
+output="$(zmb "${server_url}" start_domain_test --domain "${domain}" --ipv4 "${ipv4}" --ipv6 "${ipv6}")" || exit $?
+testid="$(printf "%s" "${output}" | "${JQ}" -r .result)" || exit $?
+printf "testid: %s\n" "${testid}" >&2
+
+if echo "${testid}" | grep -qE '[^0-9a-fA-F]' ; then
+    error 1 "start_domain_test did not return a testid: ${testid}"
+fi
+
+# Wait for test to finish
+while true
+do
+    output="$(zmb "${server_url}" test_progress --testid "${testid}")" || exit $?
+    progress="$(printf "%s" "${output}" | "${JQ}" -r .result)" || exit $?
+    printf "\r${progress}%% done" >&2
+    if [ "${progress}" -eq 100 ] ; then
+        echo >&2
+        break
+    fi
+    sleep 1
+done
+
+# Get test results
+zmb "${server_url}" get_test_results --testid "${testid}" --lang "${lang}"
diff --git a/t/test01.t b/t/test01.t
index 388cd78a7..5e48113c8 100644
--- a/t/test01.t
+++ b/t/test01.t
@@ -3,6 +3,7 @@ use warnings;
 use 5.14.2;
 
 use Test::More;    # see done_testing()
+use Test::Exception;
 use Zonemaster::Engine;
 use JSON::PP;
 
@@ -82,13 +83,19 @@ sub run_zonemaster_test_with_backend_API {
 		last if ( $progress == 100 );
 	}
 	ok( $engine->test_progress( $test_id ) == 100 );
-	my $test_results = $engine->get_test_results( { id => $test_id, language => 'fr-FR' } );
+
+        my $test_results = $engine->get_test_results( { id => $test_id, language => 'fr_FR' } );
 	ok( defined $test_results->{id},                 'TEST1 $test_results->{id} defined' );
 	ok( defined $test_results->{params},             'TEST1 $test_results->{params} defined' );
 	ok( defined $test_results->{creation_time},      'TEST1 $test_results->{creation_time} defined' );
 	ok( defined $test_results->{results},            'TEST1 $test_results->{results} defined' );
 	ok( scalar( @{ $test_results->{results} } ) > 1, 'TEST1 got some results' );
 
+        dies_ok { $engine->get_test_results( { id => $test_id, language => 'fr-FR' } ); }
+        'API get_test_results -> [results] parameter not present (wrong language tag)'; # Should be underscore, not hyphen.
+
+        dies_ok { $engine->get_test_results( { id => $test_id, language => 'zz' } ); }
+        'API get_test_results -> [results] parameter not present (wrong language tag)'; # "zz" is not our configuration file.
 }
 
 run_zonemaster_test_with_backend_API(1);
diff --git a/t/test_DB_backend.pl b/t/test_DB_backend.pl
index 5b351d112..104f9a524 100644
--- a/t/test_DB_backend.pl
+++ b/t/test_DB_backend.pl
@@ -4,9 +4,10 @@
 
 use Test::More;    # see done_testing()
 use JSON::PP;
+use Test::Exception;
 
-my $db_backend = $ARGV[0];
-ok( $db_backend eq 'PostgreSQL' || $db_backend eq 'MySQL' , "Testing a supported database backend: $db_backend" );
+my $db_backend = $ARGV[0] // BAIL_OUT( "No database backend specified" );
+( $db_backend eq 'PostgreSQL' || $db_backend eq 'MySQL' ) or BAIL_OUT( "Unsupported database backend: $db_backend" );
 
 my $frontend_params_1 = {
 	client_id      => "$db_backend Unit Test",         # free string
@@ -59,12 +60,20 @@ sub run_zonemaster_test_with_backend_API {
         last if ( $progress == 100 );
     }
     ok( $engine->test_progress( $api_test_id ) == 100 , 'API test_progress -> Test finished' );
-    my $test_results = $engine->get_test_results( { id => $api_test_id, language => 'fr-FR' } );
+
+    my $test_results = $engine->get_test_results( { id => $api_test_id, language => 'fr_FR' } );
     ok( defined $test_results->{id} , 'API get_test_results -> [id] paramater present' );
     ok( defined $test_results->{params} , 'API get_test_results -> [params] paramater present' );
     ok( defined $test_results->{creation_time} , 'API get_test_results -> [creation_time] paramater present' );
     ok( defined $test_results->{results} , 'API get_test_results -> [results] paramater present' );
     ok( scalar( @{ $test_results->{results} } ) > 1 , 'API get_test_results -> [results] paramater contains data' );
+
+    dies_ok { $engine->get_test_results( { id => $api_test_id, language => 'fr-FR' } ); }
+    'API get_test_results -> [results] parameter not present (wrong language tag)';
+
+    dies_ok { $engine->get_test_results( { id => $api_test_id, language => 'zz' } ); }
+    'API get_test_results -> [results] parameter not present (wrong language tag)';
+
 }
 
 # add test user
@@ -99,3 +108,4 @@ sub run_zonemaster_test_with_backend_API {
 ok( length($test_history->[0]->{id}) == 16 || length($test_history->[1]->{id}) == 16 );
 
 done_testing();
+