Update seed data and refactor its processing

Use Azure resources from the new development subscription.

Make data restoration more robust so that reinitialising data works when there
already exists data in the databases.

Author: jarkkoka

README.md

@@ -194,7 +194,7 @@ Internally the script calls `start-dependencies.sh` and forwards any arguments (
 
 ## Loading single dump into development database
 
-The jore3-importer microservice imports data from JORE3 and transforms it into the JORE4 data model. Currently, existing database dumps of the transformed data can be found from Azure Blob container at `hsl-jore4-common / jore4storage / jore4-dump`.
+The jore3-importer microservice imports data from JORE3 and transforms it into the JORE4 data model. Currently, existing database dumps of the transformed data can be found from Azure Blob container at `rg-jore4-dev-001 / stjore4dev001 / jore4-dump`.
 
 To download a single dump file to your local workspace and import it into your local development database instance, run `./scripts/development.sh dump:import <azure_blob_filepath> <database_name>` and follow the instructions. _Warning!_ This will empty the target database and overwrite all the data in it!
 
@@ -204,7 +204,7 @@ If you just want to download a single dump file to your local workspace (but not
 
 To update database dump files (with the `.pgdump` extension), do the following:
 
-- Check the latest suitable dump files from the `jore4-dump` container under the `jore4storage` storage account in the `hsl-jore4-common` resource group. Make sure that the `docker-compose.custom.yml` file does not specify `jore4-hasura` and `jore4-tiamat` microservices with versions older than the versions the dump files were created with. If needed, restart dependencies and generate new GraphQL schema for new Hasura version and make necessary changes to achieve ui - hasura compatibility.
+- Check the latest suitable dump files from the `jore4-dump` container under the `stjore4dev001` storage account in the `rg-jore4-dev-001` resource group. As of 2025-05, dump files are organised into directories in Azure Blob storage. They should be accompanied by a README file that states which microservice versions the dumps were created with. Make sure that the `docker-compose.custom.yml` file does not specify `jore4-hasura` and `jore4-tiamat` microservices with versions older than the versions the dump files were created with. If needed, restart dependencies and generate new GraphQL schema for new Hasura version and make necessary changes to achieve ui - hasura compatibility.
 - Update the dump filenames in the `./scripts/development.sh` file. Remove the existing `.pgdump` files from your project directory. Then stop the dependencies, and run `./scripts/setup-dependencies-and-seed.sh`.
 - If everything goes right, after running the script and following the instructions you should now have your databases seeded with the new dumps.
 

scripts/development.sh

@@ -16,9 +16,9 @@ DOCKER_COMPOSE_BUNDLE_REF=${BUNDLE_REF:-main}
 # this project from others.
 export COMPOSE_PROJECT_NAME=jore4-ui
 
-DUMP_ROUTES_FILENAME="routes-12-2024.pgdump"
-DUMP_TIMETABLES_FILENAME="timetables-12-2024.pgdump"
-DUMP_STOPS_FILENAME="stopdb-12-2024.pgdump"
+DUMP_ROUTES_FILENAME="2025-04-03_test/2025-04-03-jore4-local-jore4e2e.pgdump"
+DUMP_TIMETABLES_FILENAME="2025-04-03_test/2025-04-03-jore4-local-timetablesdb-nodata.pgdump"
+DUMP_STOPS_FILENAME="2025-04-03_test/2025-04-03-jore4-local-stopdb.pgdump"
 
 DOCKER_TESTDB_IMAGE="jore4-testdb"
 DOCKER_IMAGES="jore4-auth jore4-hasura jore4-mbtiles jore4-mapmatchingdb jore4-mapmatching jore4-hastus jore4-tiamat jore4-timetablesapi"
@@ -208,58 +208,68 @@ start_dependencies() {
 }
 
 download_dump() {
-  echo "Downloading database dump for JORE4 network & routes from Azure Blob Storage..."
-
-  # Here is a breakdown of the dump name used below:
-  # - "jore4e2e"        ~ The name of the database to which the data dump applies
-  # - "test-20240104"   ~ The data originates from the Jore3 test database (not production) and specifically the snapshot taken on 4.1.2024.
-  # - "data-only"       ~ The dump contains only data. It does not contain DDL, i.e. table and other schema element definitions.
-  # - "8a28ef5f"        ~ The dump is based on the database migrations of the jore4-hasura image version starting with this hash.
-  # - "20240104" (2nd)  ~ The day when the jore3-importer was run
-  if [ -z ${1+x} ]; then
-    read -p "Dump file name (default: jore4e2e-test-20240104-data-only-8a28ef5f-20240104.pgdump): " DUMP_FILENAME
-    DUMP_FILENAME="${DUMP_FILENAME:-jore4e2e-test-20240104-data-only-8a28ef5f-20240104.pgdump}"
+  local az_blob_filepath
+
+  if [ -z "$1" ]; then
+    read -rp "Dump file name: " az_blob_filepath
+    if [ -z "$az_blob_filepath" ]; then
+      echo "Error: empty Azure Blob container filepath given. Exiting..."
+      exit 1
+    fi
   else
-    DUMP_FILENAME=$1
+    az_blob_filepath="$1"
   fi
 
   login
 
-  # Check dump file
-  if [ ! -f "$1" ]; then
-    echo "Downloading dump file as $DUMP_FILENAME"
+  # Download the dump file, if it does not already exist.
+  if [ ! -f "$az_blob_filepath" ]; then
+    echo "Downloading dump file: $az_blob_filepath"
+
     az storage blob download \
-      --account-name "jore4storage" \
+      --subscription "HSLAZ-CORP-DEV-JORE4" \
+      --account-name "stjore4dev001" \
       --container-name "jore4-dump" \
-      --name "$DUMP_FILENAME" \
-      --file "$DUMP_FILENAME" \
+      --name "$az_blob_filepath" \
+      --file "$(basename "$az_blob_filepath")" \
       --auth-mode login
   fi
 }
 
 import_dump() {
-  if [[ -z ${1+x} || -z ${2+x} ]]; then
-    echo "File and target database need to be defined!"
+  local az_blob_filepath="$1"
+  local target_database="$2"
+
+  if [[ -z ${az_blob_filepath} || -z ${target_database} ]]; then
+    echo "Azure Blob container filepath and target database need to be defined!"
     echo "usage:"
-    echo "       development.sh dump:import file database"
+    echo "   development.sh dump:import <azure_blob_filepath> <database_name>"
     exit
   fi
 
-  echo "Importing JORE4 dump to $2 database"
+  # Extract the filename from the full path, which may include directory names.
+  local az_blob_filename
+  az_blob_filename=$(basename "$az_blob_filepath")
 
   # Download dump if it is missing
-  if [ ! -f "$1" ]; then
-    download_dump "$1"
+  if [ ! -f "$az_blob_filename" ]; then
+    download_dump "$az_blob_filepath"
   fi
 
-  docker exec -i testdb pg_restore -U dbadmin --dbname="$2" --format=c < "$1"
+  echo "Importing database dump from the file '$az_blob_filename' to the '${target_database}' database..."
+
+  docker exec -i testdb bash -c "
+    set -eux
+    dropdb --username=dbadmin --force $target_database
+    pg_restore --username=dbadmin --dbname=postgres --format=custom --create
+  " < "$az_blob_filename"
 }
 
 download_digitransit_key() {
   login
 
   echo "Downloading secret value to ui/.env.local"
-  { echo -n "NEXT_PUBLIC_DIGITRANSIT_API_KEY=" && az keyvault secret show --name "hsl-jore4-digitransit-api-key" --vault-name "hsl-jore4-dev-vault" --query "value"; } > ui/.env.local
+  { echo -n "NEXT_PUBLIC_DIGITRANSIT_API_KEY=" && az keyvault secret show --name "hsl-jore4-digitransit-api-key" --vault-name "kv-jore4-dev-001" --query "value"; } > ui/.env.local
 }
 
 setup_environment() {
@@ -362,14 +372,14 @@ print_usage() {
   dump:download [<azure_blob_filepath>]
     Downloads a JORE4 database dump from Azure Blob Storage. A full file path
     may be given as a parameter. The file path is used to refer to a file inside
-    the 'jore4-dump' container under the 'jore4storage' storage account in the
-    'hsl-jore4-common' resource group.
+    the 'jore4-dump' container under the 'stjore4dev001' storage account in the
+    'rg-jore4-dev-001' resource group.
 
   dump:import <azure_blob_filepath> <database_name>
     Imports a database dump from the given file to the specified database.
     The dump file must be given as a Azure Blob storage reference where a full
     file path needs to be given within the 'jore4-dump' container under the
-    'jore4storage' storage account in the 'hsl-jore4-common' resource group.
+    'stjore4dev001' storage account in the 'rg-jore4-dev-001' resource group.
 
   digitransit:fetch
     Download Digitransit map API key for JORE4 account.