Merge branch '263-logical-copy-multiple-databases' into 'master'

feat: provide tools to logical copy multiple or all databases (#263)

Closes #263

See merge request postgres-ai/database-lab!292

Author: agneum

configs/config.example.logical_generic.yml

@@ -172,13 +172,19 @@ retrieval:
             # The environment variable has a higher priority.
             password: postgres
 
-        # Options for a partial dump.
-        # partial:
-        #   tables:
-        #     - test
+        # Option for specifying the database list that must be copied.
+        # By default, DLE dumps and restores all available databases.
+        # Do not specify the databases section to take all databases.
+        # databases:
+        #   database1:
+        #     # Option for a partial dump. Do not specify the tables section to dump all available tables.
+        #     tables:
+        #       - test
+        #   database2:
+        #   databaseN:
 
         # Use parallel jobs to dump faster.
-        # It's ignored if "restore" is present because "pg_dump | pg_restore" is always single-threaded.
+        # It’s ignored if “immediateRestore.enabled: true” is present because “pg_dump | pg_restore” is always single-threaded.
         parallelJobs: 2
 
         # Options for direct restore to Database Lab Engine instance.
@@ -187,19 +193,20 @@ retrieval:
         # this option does not support parallelization, it is always a single-threaded (both for
         # dumping on the source, and restoring on the destination end).
         # immediateRestore:
+        #   # Enable immediate restore.
+        #   enabled: true
         #   # Restore data even if the Postgres directory (`global.dataDir`) is not empty.
         #   # Note the existing data might be overwritten.
         #   forceInit: false
-        #
+        #   # Option to adjust PostgreSQL configuration for a logical dump job.
+        #   # It's useful if a dumped database contains non-standard extensions.
         #   configs:
         #      shared_preload_libraries: "pg_stat_statements"
 
     # Restores PostgreSQL database from the provided dump. If you use this block, do not use
     # "restore" option in the "logicalDump" job.
     logicalRestore:
       options:
-        dbname: "test"
-
         # The location of the archive file (or directory, for a directory-format archive) to be restored.
         dumpLocation: "/var/lib/dblab/dblab_pool/dump/db.dump"
 
@@ -214,15 +221,25 @@ retrieval:
         # Note the existing data might be overwritten.
         forceInit: false
 
-        # Options for a partial dump.
-        # partial:
-        #   tables:
-        #     - test
-
-        #
+        # Option to adjust PostgreSQL configuration for a logical restore job
+        # It's useful if a restored database contains non-standard extensions.
         # configs:
         #  shared_preload_libraries: "pg_stat_statements"
 
+        # Option for specifying the database list that must be restored.
+        # By default, DLE restores all available databases.
+        # Do not specify the databases section to restore all available databases.
+        # databases:
+        #   database1:
+        #     # Dump format. Available formats: directory, custom, plain. Default format: directory.
+        #     format: directory
+        #     # Option for a partial restore. Do not specify the tables section to restore all available tables.
+        #     tables:
+        #       - table1
+        #       - table2
+        #   database2:
+        #   databaseN:
+
     logicalSnapshot:
       options:
         # Define pre-precessing SQL queries for data patching. For example, "/tmp/scripts/sql".

configs/config.example.logical_rds_iam.yml

@@ -173,13 +173,20 @@ retrieval:
             # Path to the SSL root certificate: https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem
             sslRootCert: "/cert/rds-combined-ca-bundle.pem"
 
-        # Options for a partial dump.
-        # partial:
-        #   tables:
-        #     - test
+        # Option for specifying the database list that must be copied.
+        # By default, DLE dumps and restores all available databases.
+        # Do not specify the databases section to take all databases.
+        # databases:
+        #   database1:
+        #     # Option for a partial dump. Do not specify the tables section to dump all available tables.
+        #     tables:
+        #       - table1
+        #       - table2
+        #   database2:
+        #   databaseN:
 
         # Use parallel jobs to dump faster.
-        # It's ignored if "restore" is present because "pg_dump | pg_restore" is always single-threaded.
+        # It’s ignored if “immediateRestore.enabled: true” is present because “pg_dump | pg_restore” is always single-threaded.
         parallelJobs: 2
 
         # Options for direct restore to Database Lab Engine instance.
@@ -188,19 +195,20 @@ retrieval:
         # this option does not support parallelization, it is always a single-threaded (both for
         # dumping on the source, and restoring on the destination end).
         # immediateRestore:
+        #   # Enable immediate restore.
+        #   enabled: true
         #   # Restore data even if the Postgres directory (`global.dataDir`) is not empty.
         #   # Note the existing data might be overwritten.
         #   forceInit: false
-        #
+        #   # Option to adjust PostgreSQL configuration for a logical dump job.
+        #   # It's useful if a dumped database contains non-standard extensions.
         #   configs:
         #      shared_preload_libraries: "pg_stat_statements"
 
     # Restores PostgreSQL database from the provided dump. If you use this block, do not use
     # "restore" option in the "logicalDump" job.
     logicalRestore:
       options:
-        dbname: "test"
-
         # The location of the archive file (or directory, for a directory-format archive) to be restored.
         dumpLocation: "/var/lib/dblab/dblab_pool/dump/rds_db.dump"
 
@@ -214,15 +222,24 @@ retrieval:
         # Note the existing data might be overwritten.
         forceInit: false
 
-        # Options for a partial dump.
-        # partial:
-        #   tables:
-        #     - test
-
-        #
+        # Option to adjust PostgreSQL configuration for a logical restore job
+        # It's useful if a restored database contains non-standard extensions.
         # configs:
         #  shared_preload_libraries: "pg_stat_statements"
 
+        # Option for specifying the database list that must be restored.
+        # By default, DLE restores all available databases.
+        # Do not specify the databases section to restore all available databases.
+        # databases:
+        #   database1:
+        #     # Dump format. Available formats: directory, custom, plain. Default format: directory.
+        #     format: directory
+        #     # Option for a partial restore. Do not specify the tables section to restore all available tables.
+        #     tables:
+        #       - test
+        #   database2:
+        #   databaseN:
+
     logicalSnapshot:
       options:
         # Define pre-precessing SQL queries for data patching. For example, "/tmp/scripts/sql".