Merge pull request #927 from matsduf/merge-develop-into-master

Merge develop into master (Backend)

.travis.yml

@@ -9,7 +9,7 @@ jobs:
   - perl: "5.32.0"
     env: TARGET=PostgreSQL ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_postgresql_backend_config.ini
     services: postgresql
-  # Cover all Perl versions with SQLite
+  # Cover supported Perl versions with SQLite
   - perl: "5.32.0"
     env: TARGET=SQLite ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_sqlite_backend_config.ini
   - perl: "5.30"
@@ -18,12 +18,6 @@ jobs:
     env: TARGET=SQLite ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_sqlite_backend_config.ini
   - perl: "5.26"
     env: TARGET=SQLite ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_sqlite_backend_config.ini
-  - perl: "5.24"
-    env: TARGET=SQLite ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_sqlite_backend_config.ini
-  - perl: "5.22"
-    env: TARGET=SQLite ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_sqlite_backend_config.ini
-  - perl: "5.16"
-    env: TARGET=SQLite ZONEMASTER_RECORD=0 ZONEMASTER_BACKEND_CONFIG_FILE=./share/travis_sqlite_backend_config.ini
 
 addons:
   apt:
@@ -107,19 +101,20 @@ before_install:
   - git clone --depth=1 --branch=$TRAVIS_BRANCH https://github.com/zonemaster/zonemaster-engine.git
   - ( cd zonemaster-engine && cpanm --verbose --notest . ) && rm -rf zonemaster-engine
 
+    # Install share files
+  - mkdir -p ./lib/auto/share/dist/
+  - ln -s ../../../../share ./lib/auto/share/dist/Zonemaster-Backend
+
 before_script:
-    - if [[ "$TARGET" == "PostgreSQL" ]]; then psql -c "create user travis_zonemaster WITH PASSWORD 'travis_zonemaster';" -U postgres; fi
-    - if [[ "$TARGET" == "PostgreSQL" ]]; then psql -c 'create database travis_zonemaster OWNER travis_zonemaster;' -U postgres; fi
+    - if [[ "$TARGET" == "PostgreSQL" ]]; then psql -U postgres -c "CREATE USER travis_zonemaster WITH PASSWORD 'travis_zonemaster';"; fi
+    - if [[ "$TARGET" == "PostgreSQL" ]]; then psql -U postgres -c 'CREATE DATABASE travis_zonemaster OWNER travis_zonemaster;'; fi
     - if [[ "$TARGET" == "PostgreSQL" ]]; then cpanm DBD::Pg; fi
-    - if [[ "$TARGET" == "PostgreSQL" ]]; then perl -I./lib ./script/create_db_postgresql_9.3.pl; fi
 
-    - if [[ "$TARGET" == "MySQL" ]]; then mysql -e "CREATE USER 'travis_zm'@'localhost' IDENTIFIED BY 'travis_zonemaster';" -u root; fi
-    - if [[ "$TARGET" == "MySQL" ]]; then mysql -e "CREATE DATABASE travis_zonemaster CHARACTER SET utf8 COLLATE utf8_bin;" -u root; fi
-    - if [[ "$TARGET" == "MySQL" ]]; then mysql -e "GRANT ALL ON travis_zonemaster.* TO 'travis_zm'@'localhost';" -u root; fi
-    - if [[ "$TARGET" == "MySQL" ]]; then mysql -e "FLUSH PRIVILEGES;" -u root; fi
-    - if [[ "$TARGET" == "MySQL" ]]; then cpanm --force DBD::mysql; fi
-    - if [[ "$TARGET" == "MySQL" ]]; then perl -I./lib ./script/create_db_mysql.pl; fi
+    - if [[ "$TARGET" == "MySQL" ]]; then mysql -u root -e "CREATE USER 'travis_zm'@'localhost' IDENTIFIED BY 'travis_zonemaster';"; fi
+    - if [[ "$TARGET" == "MySQL" ]]; then mysql -u root -e "CREATE DATABASE travis_zonemaster CHARACTER SET utf8 COLLATE utf8_bin;"; fi
+    - if [[ "$TARGET" == "MySQL" ]]; then mysql -u root -e "GRANT ALL ON travis_zonemaster.* TO 'travis_zm'@'localhost';"; fi
+    - if [[ "$TARGET" == "MySQL" ]]; then cpanm DBD::mysql; fi
 
 script:
+    - perl -I./lib ./share/create_db.pl
     - perl Makefile.PL && make test
-

Changes

@@ -1,5 +1,47 @@
 Release history for Zonemaster component Zonemaster-Backend
 
+v8.0.0 2021-12-03 (public release version)
+
+ [Breaking changes]
+ - Removes support of PostgreSQL before version 10 (#892)
+ - Removes "retry" configuration setting (#896, 881)
+
+ [Deprecation]
+ - Deprecates country code in RPC API calls (#796, #794)
+
+ [Features]
+ - Adds translation of error messages (#891)
+ - Adds Finnish translation of error messages (#921, #911)
+ - Adds Swedish translation of error messages (#899)
+ - Replaces CentOS with Rocky Linux (#908, #895)
+ - Adds upgrade documentation (#907, #879, #618)
+ - Adds support for collecting metrics (#844)
+ - Optimization of batches when PostgreSQL is used (#890)
+ - Improved result history response in API (#837, #830)
+ - Makes database port configurable (#755, #496)
+
+ [Fixes]
+ - Document clean-up and improvement (#913, #907, #882, #873, #871,
+   #822, #814, #816, #797)
+ - Improves error catching (#916, #914)
+ - Database clean-up and improvement (#906, #887, #859, #833, #839, #831,
+   #815, #824, #826, #812, #798, #775, #804, #805, 806)
+ - Updates zmb command line tool (#825, #628, #810, #768, #787, #780)
+ - Updates dependencies (#902, #901, 903)
+ - Fix translation issue (#894, #811, #809)
+ - Improves error message when creation of API user is blocked (#889)
+ - Improves RPC API error messages (#853, #789, #847, #819, #817, #820,
+   #703)
+ - Harmonize database code (#841, #832, #840, #865, #834, #689, #805)
+ - Use SQLite by default (#855)
+ - Improves handling of crashed tests (#845)
+ - Fixes configuration loading error (#851, #813)
+ - Improves log handling (#843)
+ - Fixes local adress mapping (#836)
+ - Updates default location for configuration file (#835)
+ - Improves validation (#801, #808, #685, #808, #802, #799, #757)
+
+
 v7.0.0 2021-09-15 (public security release)
  [Fixes]
  - By design adding a API user (needed for the batch function) is limited to

LICENSE

@@ -1,9 +1,11 @@
 ### Code license
 
-Copyright (c) 2013-2017, IIS (The Internet Foundation in Sweden)
-Copyright (c) 2013-2017, AFNIC
+Copyright (c) The Swedish Internet Foundation (<https://internetstiftelsen.se/en/>)
+Copyright (c) AFNIC (<https://www.afnic.fr/en/>)
 All rights reserved.
 
+Copyright belongs to external contributor where applicable.
+
 Redistribution and use in source and binary forms, with or without modification,
 are permitted provided that the following conditions are met:
 
@@ -28,8 +30,12 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 ### Documentation license
 
-Copyright (c) 2013-2017, IIS (The Internet Foundation in Sweden)
-Copyright (c) 2013-2017, AFNIC
+Copyright (c) The Swedish Internet Foundation (<https://internetstiftelsen.se/en/>)
+Copyright (c) AFNIC (<https://www.afnic.fr/en/>)
+All rights reserved.
+
+Copyright belongs to external contributor where applicable.
+
 Creative Commons Attribution 4.0 International License
 
 You should have received a copy of the license along with this

MANIFEST

@@ -6,11 +6,14 @@ docs/Configuration.md
 docs/files-description.md
 docs/GettingStarted.md
 docs/Installation.md
+docs/Telemetry.md
 docs/TypographicConventions.md
-docs/upgrade_db_zonemaster_backend_ver_1.0.3.md
-docs/upgrade_db_zonemaster_backend_ver_1.1.0.md
-docs/upgrade_db_zonemaster_backend_ver_5.0.0.md
-docs/upgrade_db_zonemaster_backend_ver_5.0.2.md
+docs/Upgrade.md
+docs/upgrade/upgrade_zonemaster_backend_ver_1.0.3.md
+docs/upgrade/upgrade_zonemaster_backend_ver_1.1.0.md
+docs/upgrade/upgrade_zonemaster_backend_ver_5.0.0.md
+docs/upgrade/upgrade_zonemaster_backend_ver_5.0.2.md
+docs/upgrade/upgrade_zonemaster_backend_ver_8.0.0.md
 inc/Module/Install.pm
 inc/Module/Install/Base.pm
 inc/Module/Install/Can.pm
@@ -21,13 +24,16 @@ inc/Module/Install/Scripts.pm
 inc/Module/Install/Share.pm
 inc/Module/Install/Win32.pm
 inc/Module/Install/WriteAll.pm
+inc/Module/Install/External.pm
 lib/Zonemaster/Backend.pm
 lib/Zonemaster/Backend/Config.pm
 lib/Zonemaster/Backend/Config/DCPlugin.pm
 lib/Zonemaster/Backend/DB.pm
 lib/Zonemaster/Backend/DB/MySQL.pm
 lib/Zonemaster/Backend/DB/PostgreSQL.pm
 lib/Zonemaster/Backend/DB/SQLite.pm
+lib/Zonemaster/Backend/Errors.pm
+lib/Zonemaster/Backend/Metrics.pm
 lib/Zonemaster/Backend/RPCAPI.pm
 lib/Zonemaster/Backend/TestAgent.pm
 lib/Zonemaster/Backend/Translator.pm
@@ -38,25 +44,24 @@ MANIFEST			This list of files
 META.yml
 README.md
 script/add-batch-job.pl
-script/create_db_mysql.pl
-script/create_db_postgresql_9.3.pl
 script/zmb
 script/zmtest
 script/zonemaster_backend_rpcapi.psgi
 script/zonemaster_backend_testagent
 share/backend_config.ini
 share/cleanup-mysql.sql
 share/cleanup-postgres.sql
-share/create_db_sqlite.pl
+share/create_db.pl
 share/freebsd-pwd.conf
-share/initial-mysql.sql
-share/initial-postgres.sql
-share/patch_db_README.txt
-share/patch_mysql_db_zonemaster_backend_ver_1.0.3.pl
-share/patch_mysql_db_zonemaster_backend_ver_5.0.0.pl
-share/patch_mysql_db_zonemaster_backend_ver_5.0.2.pl
-share/patch_postgresql_db_zonemaster_backend_ver_1.0.3.pl
-share/patch_postgresql_db_zonemaster_backend_ver_5.0.0.pl
+share/patch/README.txt
+share/patch/patch_mysql_db_zonemaster_backend_ver_1.0.3.pl
+share/patch/patch_mysql_db_zonemaster_backend_ver_5.0.0.pl
+share/patch/patch_mysql_db_zonemaster_backend_ver_5.0.2.pl
+share/patch/patch_mysql_db_zonemaster_backend_ver_8.0.0.pl
+share/patch/patch_postgresql_db_zonemaster_backend_ver_1.0.3.pl
+share/patch/patch_postgresql_db_zonemaster_backend_ver_5.0.0.pl
+share/patch/patch_postgresql_db_zonemaster_backend_ver_8.0.0.pl
+share/patch/patch_sqlite_db_zonemaster_backend_ver_8.0.0.pl
 share/tmpfiles.conf
 share/travis_mysql_backend_config.ini
 share/travis_postgresql_backend_config.ini
@@ -65,7 +70,17 @@ share/zm-rpcapi.lsb
 share/zm-testagent.lsb
 share/zm_rpcapi-bsd
 share/zm_testagent-bsd
+share/locale/da/LC_MESSAGES/Zonemaster-Backend.mo
+share/locale/es/LC_MESSAGES/Zonemaster-Backend.mo
+share/locale/fi/LC_MESSAGES/Zonemaster-Backend.mo
+share/locale/fr/LC_MESSAGES/Zonemaster-Backend.mo
+share/locale/nb/LC_MESSAGES/Zonemaster-Backend.mo
+share/locale/sv/LC_MESSAGES/Zonemaster-Backend.mo
+share/Makefile
+share/GNUmakefile
 t/config.t
+t/db.t
+t/parameters_validation.t
 t/test01.data
 t/test01.t
 t/test_validate_syntax.t

MANIFEST.SKIP

@@ -8,6 +8,10 @@
 ^\.github/pull_request_template\.md$
 ^\.travis\.yml$
 ^docs/internal-documentation/
+\.po$
+^share/[^/]*\.mo$
+^share/Zonemaster-Backend.pot$
+^share/update-po$
 
 #!start included /usr/share/perl/5.20/ExtUtils/MANIFEST.SKIP
 # Avoid version control files.

Makefile.PL

@@ -1,4 +1,5 @@
 use inc::Module::Install;
+use Module::Install::Share;
 
 name 'Zonemaster-Backend';
 all_from 'lib/Zonemaster/Backend.pm';
@@ -8,6 +9,8 @@ bugtracker 'https://github.com/zonemaster/zonemaster-backend/issues';
 # "2.1.0" could be declared as "2.001" but not as "2.1"
 # (see Zonemaster::LDNS below)
 
+configure_requires 'Locale::Msgfmt' => 0.15;
+
 requires
   'Class::Method::Modifiers'    => 1.09,
   'Config::IniFiles'            => 0,
@@ -18,11 +21,12 @@ requires
   'HTML::Entities'              => 0,
   'JSON::PP'                    => 0,
   'JSON::RPC'                   => 1.01,
-  'JSON::Validator'             => 3.12,
+  'JSON::Validator'             => 4.00,
   'Log::Any'                    => 0,
   'Log::Any::Adapter::Dispatch' => 0,
   'Log::Dispatch'               => 0,
   'LWP::UserAgent'              => 0,
+  'Mojolicious'                 => 7.28,
   'Moose'                       => 2.04,
   'Parallel::ForkManager'       => 1.12,
   'Plack::Builder'              => 0,
@@ -31,9 +35,10 @@ requires
   'Starman'                     => 0,
   'String::ShellQuote'          => 0,
   'Try::Tiny'                   => 0.12,
-  'Zonemaster::Engine'          => 4.002,
+  'Zonemaster::Engine'          => 4.003,
   'Zonemaster::LDNS'            => 2.002,
   'Plack::Middleware::ReverseProxy' => 0,
+  'Locale::TextDomain'          => 1.20,
   ;
 
 test_requires 'DBD::SQLite';
@@ -53,4 +58,26 @@ install_script 'zmb';
 no_index directory => 'CodeSnippets';
 no_index directory => 'Doc';
 
+# Make all platforms include inc/Module/Install/External.pm
+requires_external_bin 'find';
+if ($^O eq "freebsd") {
+    requires_external_bin 'gmake';
+};
+
+sub MY::postamble {
+    my $text;
+    if ($^O eq "freebsd") {
+        # Make FreeBSD use gmake for share/Makefile
+        $text = 'GMAKE ?= "gmake"' . "\n"
+            . 'pure_all :: share/Makefile' . "\n"
+            . "\t" . 'cd share && $(GMAKE) all' . "\n";
+    } else {
+        $text = 'pure_all :: share/Makefile' . "\n"
+            . "\t" . 'cd share && $(MAKE) all' . "\n";
+    };
+    return $text;
+};
+
+install_share;
+
 WriteAll;

README.md

@@ -25,28 +25,10 @@ Zonemaster Engine installed. Please see the
 instructions](https://github.com/zonemaster/zonemaster-engine/blob/master/docs/Installation.md).
 
 
-### Upgrade 
+### Upgrade
 
-If you upgrade Zonemaster-Backend and want to keep the content of the database
-(MySQL/MariaDB or PostgrSQL) then you should not reset the database when you
-follow the [installation instructions]. In some cases you need to patch the
-database when you update Zonemaster-Backend.
+See the [upgrade document].
 
-Always take a backup first if the database is valuable.
-
-Current version                     | Link to instructions  | Comments
-------------------------------------|-----------------------|-----------------------
-Older than 1.0.3                    | [Upgrade to 1.0.3]    |
-At least 1.0.3 but older than 1.1.0 | [Upgrade to 1.1.0]    |
-At least 1.1.0 but older than 5.0.0 | [Upgrade to 5.0.0]    |
-At least 5.0.0 but older than 5.0.2 | [Upgrade to 5.0.2]    | For MySQL/MariaDB only
-
-If the database was created before Zonemaster-Backend version 5.0.0, then you
-have to upgrade in several steps.
-
-To complete the upgrade follow the [installation instructions], except for creating
-the database. If you instead want to start from an empty database, then you remove the database
-and create a new database using the [installation instructions].
 
 ### Installation
 
@@ -83,7 +65,4 @@ The software is released under the 2-clause BSD license. See separate
 
 [Configuration documentation]: docs/Configuration.md
 [Installation instructions]:   docs/Installation.md
-[Upgrade to 1.0.3]:            docs/upgrade_db_zonemaster_backend_ver_1.0.3.md
-[Upgrade to 1.1.0]:            docs/upgrade_db_zonemaster_backend_ver_1.1.0.md
-[Upgrade to 5.0.0]:            docs/upgrade_db_zonemaster_backend_ver_5.0.0.md
-[Upgrade to 5.0.2]:            docs/upgrade_db_zonemaster_backend_ver_5.0.2.md
+[Upgrade document]:            docs/Upgrade.md