* make integration tests harmless in dev and staging environments:

  * synthetic
  * logical generic
  * physical WAL-G
  * physical basebackup
  * logical RDS
* enable logical generic and physical basebackup cases
* install yq from the official repository

Author: agneum

Diff for: .gitlab-ci.yml

@@ -326,8 +326,9 @@ build-image-swagger-latest:
   variables:
     IMAGE_TAG: "${CI_COMMIT_REF_SLUG}"
   script:
-    - zfs list
     - bash test/1.synthetic.sh
+    - bash test/2.logical_generic.sh
+    - bash test/4.physical_basebackup.sh
   after_script:
     - bash test/_cleanup.sh
   tags:

Diff for: test/1.synthetic.sh

@@ -1,11 +1,17 @@
 #!/bin/bash
 set -euxo pipefail
 
-TAG=${TAG:-${CI_COMMIT_REF_SLUG}}
+TAG=${TAG:-${CI_COMMIT_REF_SLUG:-"master"}}
 IMAGE2TEST="registry.gitlab.com/postgres-ai/database-lab/dblab-server:${TAG}"
+DLE_SERVER_NAME="dblab_server_test"
 
 # Environment variables for replacement rules
 export POSTGRES_VERSION="${POSTGRES_VERSION:-13}"
+export DLE_SERVER_PORT=${DLE_SERVER_PORT:-12345}
+export DLE_PORT_POOL_FROM=${DLE_PORT_POOL_FROM:-9000}
+export DLE_PORT_POOL_TO=${DLE_PORT_POOL_TO:-9100}
+export DLE_TEST_MOUNT_DIR="/var/lib/test/dblab"
+export DLE_TEST_POOL_NAME="test_dblab_pool"
 
 DIR=${0%/*}
 
@@ -17,12 +23,15 @@ source "${DIR}/_zfs.file.sh"
 ### Step 2. Configure and launch the Database Lab Engine
 
 ## Prepare database data directory.
+sudo docker rm dblab_pg_initdb || true
+
 sudo docker run \
   --name dblab_pg_initdb \
   --label dblab_sync \
+  --label dblab_test \
   --env PGDATA=/var/lib/postgresql/pgdata \
   --env POSTGRES_HOST_AUTH_METHOD=trust \
-  --volume /var/lib/dblab/dblab_pool/data:/var/lib/postgresql/pgdata \
+  --volume ${DLE_TEST_MOUNT_DIR}/${DLE_TEST_POOL_NAME}/data:/var/lib/postgresql/pgdata \
   --detach \
   postgres:"${POSTGRES_VERSION}"-alpine
 
@@ -64,6 +73,10 @@ yq eval -i '
   .global.debug = true |
   .global.telemetry.enabled = false |
   .localUI.enabled = false |
+  .server.port = env(DLE_SERVER_PORT) |
+  .provision.portPool.from = env(DLE_PORT_POOL_FROM) |
+  .provision.portPool.to = env(DLE_PORT_POOL_TO) |
+  .poolManager.mountDir = env(DLE_TEST_MOUNT_DIR) |
   del(.retrieval.jobs[] | select(. == "logicalDump")) |
   del(.retrieval.jobs[] | select(. == "logicalRestore")) |
   .databaseContainer.dockerImage = "postgresai/extended-postgres:" + strenv(POSTGRES_VERSION)
@@ -76,13 +89,14 @@ fi
 
 ## Launch Database Lab server
 sudo docker run \
-  --name dblab_server \
+  --name ${DLE_SERVER_NAME} \
   --label dblab_control \
+  --label dblab_test \
   --privileged \
-  --publish 2345:2345 \
+  --publish ${DLE_SERVER_PORT}:${DLE_SERVER_PORT} \
   --volume /var/run/docker.sock:/var/run/docker.sock \
-  --volume /var/lib/dblab/dblab_pool/dump:/var/lib/dblab/dblab_pool/dump \
-  --volume /var/lib/dblab:/var/lib/dblab/:rshared \
+  --volume ${DLE_TEST_MOUNT_DIR}/${DLE_TEST_POOL_NAME}/dump:${DLE_TEST_MOUNT_DIR}/${DLE_TEST_POOL_NAME}/dump \
+  --volume ${DLE_TEST_MOUNT_DIR}:${DLE_TEST_MOUNT_DIR}/:rshared \
   --volume "${configDir}":/home/dblab/configs:ro \
   --volume "${metaDir}":/home/dblab/meta \
   --volume /sys/kernel/debug:/sys/kernel/debug:rw \
@@ -93,11 +107,11 @@ sudo docker run \
   "${IMAGE2TEST}"
 
 # Check the Database Lab Engine logs
-sudo docker logs dblab_server -f 2>&1 | awk '{print "[CONTAINER dblab_server]: "$0}' &
+sudo docker logs ${DLE_SERVER_NAME} -f 2>&1 | awk '{print "[CONTAINER ${DLE_SERVER_PORT}]: "$0}' &
 
 ### Waiting for the Database Lab Engine initialization.
 for i in {1..300}; do
-  curl http://localhost:2345 > /dev/null 2>&1 && break || echo "dblab is not ready yet"
+  curl http://localhost:${DLE_SERVER_PORT} > /dev/null 2>&1 && break || echo "dblab is not ready yet"
   sleep 1
 done
 
@@ -113,7 +127,7 @@ dblab --version
 # Initialize CLI configuration
 dblab init \
   --environment-id=test \
-  --url=http://localhost:2345 \
+  --url=http://localhost:${DLE_SERVER_PORT} \
   --token=secret_token \
   --insecure
 
@@ -129,14 +143,14 @@ dblab clone create \
 
 # Connect to a clone and check the available table
 PGPASSWORD=secret_password psql \
-  "host=localhost port=6000 user=dblab_user_1 dbname=test" -c '\dt+'
+  "host=localhost port=${DLE_PORT_POOL_FROM} user=dblab_user_1 dbname=test" -c '\dt+'
 
 # Drop table
 PGPASSWORD=secret_password psql \
-  "host=localhost port=6000 user=dblab_user_1 dbname=test" -c 'drop table pgbench_accounts'
+  "host=localhost port=${DLE_PORT_POOL_FROM} user=dblab_user_1 dbname=test" -c 'drop table pgbench_accounts'
 
 PGPASSWORD=secret_password psql \
-  "host=localhost port=6000 user=dblab_user_1 dbname=test" -c '\dt+'
+  "host=localhost port=${DLE_PORT_POOL_FROM} user=dblab_user_1 dbname=test" -c '\dt+'
 
 ## Reset clone
 dblab clone reset testclone
@@ -146,17 +160,17 @@ dblab clone status testclone
 
 # Check the database objects (everything should be the same as when we started)
 PGPASSWORD=secret_password psql \
-  "host=localhost port=6000 user=dblab_user_1 dbname=test" -c '\dt+'
+  "host=localhost port=${DLE_PORT_POOL_FROM} user=dblab_user_1 dbname=test" -c '\dt+'
 
 
 ### Step 4. Check clone durability on DLE restart.
 
 ## Restart DLE.
-sudo docker restart dblab_server
+sudo docker restart ${DLE_SERVER_NAME}
 
 ### Waiting for the Database Lab Engine to start.
 for i in {1..300}; do
-  curl http://localhost:2345 > /dev/null 2>&1 && break || echo "dblab is not ready yet"
+  curl http://localhost:${DLE_SERVER_PORT} > /dev/null 2>&1 && break || echo "dblab is not ready yet"
   sleep 1
 done
 
@@ -168,15 +182,15 @@ dblab clone status testclone
 
 # Check the database objects (everything should be the same as when we started)
 PGPASSWORD=secret_password psql \
-  "host=localhost port=6000 user=dblab_user_1 dbname=test" -c '\dt+'
+  "host=localhost port=${DLE_PORT_POOL_FROM} user=dblab_user_1 dbname=test" -c '\dt+'
 
 
 ### Step 5. Destroy clone
 dblab clone destroy testclone
 dblab clone list
 
 ## Stop DLE.
-sudo docker stop dblab_server
+sudo docker stop ${DLE_SERVER_NAME}
 
 ### Finish. clean up
 source "${DIR}/_cleanup.sh"

Diff for: test/2.logical_generic.sh

@@ -1,8 +1,9 @@
 #!/bin/bash
 set -euxo pipefail
 
-TAG="${TAG:-"master"}"
+TAG=${TAG:-${CI_COMMIT_REF_SLUG:-"master"}}
 IMAGE2TEST="registry.gitlab.com/postgres-ai/database-lab/dblab-server:${TAG}"
+DLE_SERVER_NAME="dblab_server_test"
 
 # Environment variables for replacement rules
 export SOURCE_DBNAME="${SOURCE_DBNAME:-test}"
@@ -11,37 +12,52 @@ export SOURCE_PORT="${SOURCE_PORT:-7432}"
 export SOURCE_USERNAME="${SOURCE_USERNAME:-postgres}"
 export SOURCE_PASSWORD="${SOURCE_PASSWORD:-secretpassword}"
 export POSTGRES_VERSION="${POSTGRES_VERSION:-13}"
+export DLE_TEST_MOUNT_DIR="/var/lib/test/dblab"
+export DLE_TEST_POOL_NAME="test_dblab_pool"
+export DLE_SERVER_PORT=${DLE_SERVER_PORT:-12345}
+export DLE_PORT_POOL_FROM=${DLE_PORT_POOL_FROM:-9000}
+export DLE_PORT_POOL_TO=${DLE_PORT_POOL_TO:-9100}
 
 DIR=${0%/*}
 
 if [[ "${SOURCE_HOST}" = "172.17.0.1" ]]; then
 ### Step 0. Create source database
-  sudo rm -rf "$(pwd)"/postgresql/"${POSTGRES_VERSION}"/test || true
+  TMP_DATA_DIR="/tmp/dle_test/logical_generic"
+  cleanup_testdata_dir() {
+    sudo rm -rf "${TMP_DATA_DIR}"/postgresql/"${POSTGRES_VERSION}"/test || true
+  }
+
+  trap cleanup_testdata_dir EXIT
+
+  cleanup_testdata_dir
+  sudo docker rm postgres"${POSTGRES_VERSION}" || true
+
   sudo docker run \
     --name postgres"${POSTGRES_VERSION}" \
     --label pgdb \
+    --label dblab_test \
     --privileged \
     --publish 172.17.0.1:"${SOURCE_PORT}":5432 \
     --env PGDATA=/var/lib/postgresql/pgdata \
     --env POSTGRES_USER="${SOURCE_USERNAME}" \
     --env POSTGRES_PASSWORD="${SOURCE_PASSWORD}" \
     --env POSTGRES_DB="${SOURCE_DBNAME}" \
     --env POSTGRES_HOST_AUTH_METHOD=md5 \
-    --volume "$(pwd)"/postgresql/"${POSTGRES_VERSION}"/test:/var/lib/postgresql/pgdata \
+    --volume "${TMP_DATA_DIR}"/postgresql/"${POSTGRES_VERSION}"/test:/var/lib/postgresql/pgdata \
     --detach \
     postgres:"${POSTGRES_VERSION}-alpine"
 
   for i in {1..300}; do
-    sudo docker exec -it postgres"${POSTGRES_VERSION}" psql -d "${SOURCE_DBNAME}" -U postgres -c 'select' > /dev/null 2>&1  && break || echo "test database is not ready yet"
+    sudo docker exec postgres"${POSTGRES_VERSION}" psql -d "${SOURCE_DBNAME}" -U postgres -c 'select' > /dev/null 2>&1  && break || echo "test database is not ready yet"
     sleep 1
   done
 
   # Generate data in the test database using pgbench
   # 1,000,000 accounts, ~0.14 GiB of data.
-  sudo docker exec -it postgres"${POSTGRES_VERSION}" pgbench -U postgres -i -s 10 "${SOURCE_DBNAME}"
+  sudo docker exec postgres"${POSTGRES_VERSION}" pgbench -U postgres -i -s 10 "${SOURCE_DBNAME}"
 
   # Database info
-  sudo docker exec -it postgres"${POSTGRES_VERSION}" psql -U postgres -c "\l+ ${SOURCE_DBNAME}"
+  sudo docker exec postgres"${POSTGRES_VERSION}" psql -U postgres -c "\l+ ${SOURCE_DBNAME}"
 fi
 
 ### Step 1. Prepare a machine with disk, Docker, and ZFS
@@ -65,23 +81,35 @@ yq eval -i '
   .global.debug = true |
   .global.telemetry.enabled = false |
   .localUI.enabled = false |
+  .server.port = env(DLE_SERVER_PORT) |
+  .poolManager.mountDir = env(DLE_TEST_MOUNT_DIR) |
+  .provision.portPool.from = env(DLE_PORT_POOL_FROM) |
+  .provision.portPool.to = env(DLE_PORT_POOL_TO) |
+  .retrieval.spec.logicalDump.options.dumpLocation = env(DLE_TEST_MOUNT_DIR) + "/" + env(DLE_TEST_POOL_NAME) + "/dump" |
   .retrieval.spec.logicalDump.options.source.connection.dbname = strenv(SOURCE_DBNAME) |
   .retrieval.spec.logicalDump.options.source.connection.host = strenv(SOURCE_HOST) |
   .retrieval.spec.logicalDump.options.source.connection.port = env(SOURCE_PORT) |
   .retrieval.spec.logicalDump.options.source.connection.username = strenv(SOURCE_USERNAME) |
   .retrieval.spec.logicalDump.options.source.connection.password = strenv(SOURCE_PASSWORD) |
+  .retrieval.spec.logicalRestore.options.dumpLocation = env(DLE_TEST_MOUNT_DIR) + "/" + env(DLE_TEST_POOL_NAME) + "/dump" |
   .databaseContainer.dockerImage = "postgresai/extended-postgres:" + strenv(POSTGRES_VERSION)
 ' "${configDir}/server.yml"
 
+# logerrors is not supported in PostgreSQL 9.6
+if [ "${POSTGRES_VERSION}" = "9.6" ]; then
+  yq eval -i '.databaseConfigs.configs.shared_preload_libraries = "pg_stat_statements, auto_explain"' "${configDir}/server.yml"
+fi
+
 ## Launch Database Lab server
 sudo docker run \
-  --name dblab_server \
+  --name ${DLE_SERVER_NAME} \
   --label dblab_control \
+  --label dblab_test \
   --privileged \
-  --publish 2345:2345 \
+  --publish ${DLE_SERVER_PORT}:${DLE_SERVER_PORT} \
   --volume /var/run/docker.sock:/var/run/docker.sock \
-  --volume /var/lib/dblab/dblab_pool/dump:/var/lib/dblab/dblab_pool/dump \
-  --volume /var/lib/dblab:/var/lib/dblab/:rshared \
+  --volume ${DLE_TEST_MOUNT_DIR}/${DLE_TEST_POOL_NAME}/dump:${DLE_TEST_MOUNT_DIR}/${DLE_TEST_POOL_NAME}/dump \
+  --volume ${DLE_TEST_MOUNT_DIR}:${DLE_TEST_MOUNT_DIR}/:rshared \
   --volume "${configDir}":/home/dblab/configs:ro \
   --volume "${metaDir}":/home/dblab/meta \
   --volume /sys/kernel/debug:/sys/kernel/debug:rw \
@@ -92,11 +120,11 @@ sudo docker run \
   "${IMAGE2TEST}"
 
 # Check the Database Lab Engine logs
-sudo docker logs dblab_server -f 2>&1 | awk '{print "[CONTAINER dblab_server]: "$0}' &
+sudo docker logs ${DLE_SERVER_NAME} -f 2>&1 | awk '{print "[CONTAINER dblab_server]: "$0}' &
 
 ### Waiting for the Database Lab Engine initialization.
 for i in {1..300}; do
-  curl http://localhost:2345 > /dev/null 2>&1 && break || echo "dblab is not ready yet"
+  curl http://localhost:${DLE_SERVER_PORT} > /dev/null 2>&1 && break || echo "dblab is not ready yet"
   sleep 1
 done
 
@@ -112,7 +140,7 @@ dblab --version
 # Initialize CLI configuration
 dblab init \
   --environment-id=test \
-  --url=http://localhost:2345 \
+  --url=http://localhost:${DLE_SERVER_PORT} \
   --token=secret_token \
   --insecure
 
@@ -128,14 +156,14 @@ dblab clone create \
 
 # Connect to a clone and check the available table
 PGPASSWORD=secret_password psql \
-  "host=localhost port=6000 user=dblab_user_1 dbname=${SOURCE_DBNAME}" -c '\dt+'
+  "host=localhost port=${DLE_PORT_POOL_FROM} user=dblab_user_1 dbname=${SOURCE_DBNAME}" -c '\dt+'
 
 # Drop table
 PGPASSWORD=secret_password psql \
-  "host=localhost port=6000 user=dblab_user_1 dbname=${SOURCE_DBNAME}" -c 'drop table pgbench_accounts'
+  "host=localhost port=${DLE_PORT_POOL_FROM} user=dblab_user_1 dbname=${SOURCE_DBNAME}" -c 'drop table pgbench_accounts'
 
 PGPASSWORD=secret_password psql \
-  "host=localhost port=6000 user=dblab_user_1 dbname=${SOURCE_DBNAME}" -c '\dt+'
+  "host=localhost port=${DLE_PORT_POOL_FROM} user=dblab_user_1 dbname=${SOURCE_DBNAME}" -c '\dt+'
 
 ## Reset clone
 dblab clone reset testclone
@@ -145,11 +173,14 @@ dblab clone status testclone
 
 # Check the database objects (everything should be the same as when we started)
 PGPASSWORD=secret_password psql \
-  "host=localhost port=6000 user=dblab_user_1 dbname=${SOURCE_DBNAME}" -c '\dt+'
+  "host=localhost port=${DLE_PORT_POOL_FROM} user=dblab_user_1 dbname=${SOURCE_DBNAME}" -c '\dt+'
 
 ### Step 4. Destroy clone
 dblab clone destroy testclone
 dblab clone list
 
+## Stop DLE.
+sudo docker stop ${DLE_SERVER_NAME}
+
 ### Finish. clean up
 source "${DIR}/_cleanup.sh"