From 01e80b9f9d3b3d8b813829027a4dc6ec04227dd2 Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Wed, 20 Dec 2023 16:18:22 +0100
Subject: [PATCH 01/35] update script to handle mariadb migration testing

---
 .github/workflows/release.yml | 15 +++++++++++++++
 scripts/test-migrations.sh    | 14 +++++++++++++-
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 5a909515909..c56e33a1064 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -66,6 +66,21 @@ jobs:
     - name: Test migrations across versions
       run: bash scripts/test-migrations.sh sqlite
 
+  mariadb-db-migration-testing:
+    runs-on: ubuntu-dind-runners
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4.1.1
+
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v4.8.0
+      with:
+        python-version: "3.9"
+
+    - name: Test migrations across versions
+      run: bash scripts/test-migrations.sh mariadb
+
   publish-python-package:
     needs: [setup-and-test, mlstacks-compatibility-check, sqlite-db-migration-testing, mysql-db-migration-testing]
     uses: ./.github/workflows/publish_to_pypi.yml
diff --git a/scripts/test-migrations.sh b/scripts/test-migrations.sh
index 8747ec197af..92c064b13a7 100755
--- a/scripts/test-migrations.sh
+++ b/scripts/test-migrations.sh
@@ -42,9 +42,14 @@ if [ "$1" == "mysql" ]; then
     echo "===== Testing MySQL ====="
     # run a mysql instance in docker
     docker run --name mysql -d -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password mysql:latest
-
     # mysql takes a while to start up
     sleep 30
+elif [ "$1" == "mariadb" ]; then
+    echo "===== Testing MariaDB ====="
+    # run a mariadb instance in docker
+    docker run --name mariadb -d -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password mariadb:latest
+    # mariadb takes a while to start up
+    sleep 30
 fi
 
 # List of versions to test
@@ -76,6 +81,8 @@ do
 
     if [ "$1" == "mysql" ]; then
         zenml connect --url mysql://127.0.0.1/zenml --username root --password password
+    elif [ "$1" == "mariadb" ]; then
+        zenml connect --url mysql://127.0.0.1/zenml --username root --password password
     fi
 
     # Run the tests for this version
@@ -101,6 +108,8 @@ pip3 install importlib_metadata
 
 if [ "$1" == "mysql" ]; then
     zenml connect --url mysql://127.0.0.1/zenml --username root --password password
+elif [ "$1" == "mariadb" ]; then
+    zenml connect --url mysql://127.0.0.1/zenml --username root --password password
 fi
 
 run_tests_for_version current_branch_mysql
@@ -108,6 +117,9 @@ run_tests_for_version current_branch_mysql
 if [ "$1" == "mysql" ]; then
     zenml disconnect
     docker rm -f mysql
+elif [ "$1" == "mariadb" ]; then
+    zenml disconnect
+    docker rm -f mariadb
 fi
 
 deactivate

From 996479bf0167e0dd4d19bae1509164457df7f2fa Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Wed, 20 Dec 2023 16:44:03 +0100
Subject: [PATCH 02/35] WIP

---
 .github/workflows/testing_this_now.yml | 23 +++++++++++++++++++++++
 scripts/test-migrations.sh             |  2 +-
 2 files changed, 24 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/testing_this_now.yml

diff --git a/.github/workflows/testing_this_now.yml b/.github/workflows/testing_this_now.yml
new file mode 100644
index 00000000000..2f90b660729
--- /dev/null
+++ b/.github/workflows/testing_this_now.yml
@@ -0,0 +1,23 @@
+# This is a basic workflow to help you get started with Actions
+name: Release Package & Docker Image
+
+on:
+    pull_request:
+        types: [opened, synchronize, ready_for_review]
+        paths-ignore: ["docs/**", "docker/**", "*", "!pyproject.toml", "**.md"]
+
+jobs:
+    mariadb-db-migration-testing:
+    runs-on: ubuntu-dind-runners
+
+    steps:
+    - name: Checkout code
+        uses: actions/checkout@v4.1.1
+
+    - name: Set up Python 3.9
+        uses: actions/setup-python@v4.8.0
+        with:
+        python-version: "3.9"
+
+    - name: Test migrations across versions
+        run: bash scripts/test-migrations.sh mariadb
diff --git a/scripts/test-migrations.sh b/scripts/test-migrations.sh
index 92c064b13a7..7cf5b32ae7b 100755
--- a/scripts/test-migrations.sh
+++ b/scripts/test-migrations.sh
@@ -47,7 +47,7 @@ if [ "$1" == "mysql" ]; then
 elif [ "$1" == "mariadb" ]; then
     echo "===== Testing MariaDB ====="
     # run a mariadb instance in docker
-    docker run --name mariadb -d -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password mariadb:latest
+    docker run --name mariadb -d -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password mariadb:10.6
     # mariadb takes a while to start up
     sleep 30
 fi

From 75d27eebaf82db5ae6db260058eadfbfb0d88bc7 Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Wed, 20 Dec 2023 16:55:28 +0100
Subject: [PATCH 03/35] reformat temporary script

---
 .github/workflows/testing_this_now.yml | 25 ++++++++++---------------
 1 file changed, 10 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/testing_this_now.yml b/.github/workflows/testing_this_now.yml
index 2f90b660729..180f6bb22d2 100644
--- a/.github/workflows/testing_this_now.yml
+++ b/.github/workflows/testing_this_now.yml
@@ -1,23 +1,18 @@
-# This is a basic workflow to help you get started with Actions
-name: Release Package & Docker Image
-
+---
+name: TEST MariaDB DB migrations
 on:
-    pull_request:
-        types: [opened, synchronize, ready_for_review]
-        paths-ignore: ["docs/**", "docker/**", "*", "!pyproject.toml", "**.md"]
-
+  pull_request:
+    types: [opened, synchronize, ready_for_review]
+    paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md']
 jobs:
-    mariadb-db-migration-testing:
+  mariadb-db-migration-testing:
     runs-on: ubuntu-dind-runners
-
     steps:
-    - name: Checkout code
+      - name: Checkout code
         uses: actions/checkout@v4.1.1
-
-    - name: Set up Python 3.9
+      - name: Set up Python 3.9
         uses: actions/setup-python@v4.8.0
         with:
-        python-version: "3.9"
-
-    - name: Test migrations across versions
+          python-version: '3.9'
+      - name: Test migrations across versions
         run: bash scripts/test-migrations.sh mariadb

From 387554907552c237b4211002519008b0e54382d0 Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Wed, 20 Dec 2023 17:04:35 +0100
Subject: [PATCH 04/35] parameterize DB startup delay

---
 scripts/test-migrations.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scripts/test-migrations.sh b/scripts/test-migrations.sh
index a6a5a5e2cd9..28e11134613 100755
--- a/scripts/test-migrations.sh
+++ b/scripts/test-migrations.sh
@@ -1,6 +1,8 @@
 #!/bin/bash
 
 DB="sqlite"
+DB_STARTUP_DELAY=30 # Time in seconds to wait for the database container to start
+
 if [ -z "$1" ]; then
   echo "No argument passed, using default: $DB"
 else
@@ -42,13 +44,13 @@ if [ "$1" == "mysql" ]; then
     # run a mysql instance in docker
     docker run --name mysql -d -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password mysql:latest
     # mysql takes a while to start up
-    sleep 30
+    sleep $DB_STARTUP_DELAY
 elif [ "$1" == "mariadb" ]; then
     echo "===== Testing MariaDB ====="
     # run a mariadb instance in docker
     docker run --name mariadb -d -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password mariadb:10.6
     # mariadb takes a while to start up
-    sleep 30
+    sleep $DB_STARTUP_DELAY
 fi
 
 # List of versions to test

From eddf46364e0cd180ced95ea40da858eb13798d63 Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Wed, 20 Dec 2023 18:43:45 +0100
Subject: [PATCH 05/35] add a check for mysql vs mariadb

---
 ...aa_add_name_column_to_input_artifact_pk.py | 28 +++++++++++--------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py b/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py
index 1df3e241df8..13c165b9406 100644
--- a/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py
+++ b/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py
@@ -126,19 +126,23 @@ def downgrade() -> None:
 
 def _disable_primary_key_requirement_if_necessary() -> None:
     """Disables MySQL primary key requirement if necessary."""
-    if op.get_bind().engine.name == "mysql":
-        server_version_info = op.get_bind().engine.dialect.server_version_info
+    connection = op.get_bind()
+    engine_name = connection.engine.name
+
+    if engine_name == "mysql":
+        server_version_info = connection.engine.dialect.server_version_info
         if server_version_info and server_version_info >= (8, 0, 13):
-            potential_session_var = (
-                op.get_bind()
-                .execute(
+            try:
+                potential_session_var = connection.execute(
                     text(
                         'SHOW SESSION VARIABLES LIKE "sql_require_primary_key";'
                     )
-                )
-                .fetchone()
-            )
-            if potential_session_var and potential_session_var[1] == "ON":
-                # Temporarily disable this MySQL setting so we can update the
-                # primary key
-                op.execute("SET SESSION sql_require_primary_key = 0;")
+                ).fetchone()
+                if potential_session_var and potential_session_var[1] == "ON":
+                    # Temporarily disable this MySQL setting so we can update the
+                    # primary key
+                    op.execute("SET SESSION sql_require_primary_key = 0;")
+            except Exception as e:
+                # Handle the case where the variable does not exist
+                # (e.g., in MariaDB)
+                print("Skipping setting sql_require_primary_key: ", e)

From 40deb3a185af15e1f59abe3f8acbc942660adf12 Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Wed, 20 Dec 2023 19:01:03 +0100
Subject: [PATCH 06/35] try another way to isolate mariadb vs mysql

---
 ...faa_add_name_column_to_input_artifact_pk.py | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py b/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py
index 13c165b9406..0195e28abe3 100644
--- a/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py
+++ b/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py
@@ -131,7 +131,17 @@ def _disable_primary_key_requirement_if_necessary() -> None:
 
     if engine_name == "mysql":
         server_version_info = connection.engine.dialect.server_version_info
-        if server_version_info and server_version_info >= (8, 0, 13):
+
+        # Check if MariaDB
+        is_mariadb = (
+            "MariaDB" in connection.execute("SELECT VERSION();").fetchone()[0]
+        )
+
+        if (
+            not is_mariadb
+            and server_version_info
+            and server_version_info >= (8, 0, 13)
+        ):
             try:
                 potential_session_var = connection.execute(
                     text(
@@ -139,10 +149,10 @@ def _disable_primary_key_requirement_if_necessary() -> None:
                     )
                 ).fetchone()
                 if potential_session_var and potential_session_var[1] == "ON":
-                    # Temporarily disable this MySQL setting so we can update the
-                    # primary key
+                    # Temporarily disable this MySQL setting so we can
+                    # update the primary key
                     op.execute("SET SESSION sql_require_primary_key = 0;")
             except Exception as e:
                 # Handle the case where the variable does not exist
-                # (e.g., in MariaDB)
+                # or other errors
                 print("Skipping setting sql_require_primary_key: ", e)

From e835ca418c828a1e15b7824ac160daf132cb43b4 Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Wed, 20 Dec 2023 19:48:56 +0100
Subject: [PATCH 07/35] try a different mariadb connection path

---
 scripts/test-migrations.sh                    |  4 +-
 ...aa_add_name_column_to_input_artifact_pk.py | 40 ++++++-------------
 2 files changed, 15 insertions(+), 29 deletions(-)

diff --git a/scripts/test-migrations.sh b/scripts/test-migrations.sh
index 8bb55be0aa5..21da55af6a1 100755
--- a/scripts/test-migrations.sh
+++ b/scripts/test-migrations.sh
@@ -88,7 +88,7 @@ do
     if [ "$1" == "mysql" ]; then
         zenml connect --url mysql://127.0.0.1/zenml --username root --password password
     elif [ "$1" == "mariadb" ]; then
-        zenml connect --url mysql://127.0.0.1/zenml --username root --password password
+        zenml connect --url mariadb://127.0.0.1/zenml --username root --password password
     fi
 
     # Run the tests for this version
@@ -115,7 +115,7 @@ pip3 install importlib_metadata
 if [ "$1" == "mysql" ]; then
     zenml connect --url mysql://127.0.0.1/zenml --username root --password password
 elif [ "$1" == "mariadb" ]; then
-    zenml connect --url mysql://127.0.0.1/zenml --username root --password password
+    zenml connect --url mariadb://127.0.0.1/zenml --username root --password password
 fi
 
 run_tests_for_version current_branch_mysql
diff --git a/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py b/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py
index 0195e28abe3..1df3e241df8 100644
--- a/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py
+++ b/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py
@@ -126,33 +126,19 @@ def downgrade() -> None:
 
 def _disable_primary_key_requirement_if_necessary() -> None:
     """Disables MySQL primary key requirement if necessary."""
-    connection = op.get_bind()
-    engine_name = connection.engine.name
-
-    if engine_name == "mysql":
-        server_version_info = connection.engine.dialect.server_version_info
-
-        # Check if MariaDB
-        is_mariadb = (
-            "MariaDB" in connection.execute("SELECT VERSION();").fetchone()[0]
-        )
-
-        if (
-            not is_mariadb
-            and server_version_info
-            and server_version_info >= (8, 0, 13)
-        ):
-            try:
-                potential_session_var = connection.execute(
+    if op.get_bind().engine.name == "mysql":
+        server_version_info = op.get_bind().engine.dialect.server_version_info
+        if server_version_info and server_version_info >= (8, 0, 13):
+            potential_session_var = (
+                op.get_bind()
+                .execute(
                     text(
                         'SHOW SESSION VARIABLES LIKE "sql_require_primary_key";'
                     )
-                ).fetchone()
-                if potential_session_var and potential_session_var[1] == "ON":
-                    # Temporarily disable this MySQL setting so we can
-                    # update the primary key
-                    op.execute("SET SESSION sql_require_primary_key = 0;")
-            except Exception as e:
-                # Handle the case where the variable does not exist
-                # or other errors
-                print("Skipping setting sql_require_primary_key: ", e)
+                )
+                .fetchone()
+            )
+            if potential_session_var and potential_session_var[1] == "ON":
+                # Temporarily disable this MySQL setting so we can update the
+                # primary key
+                op.execute("SET SESSION sql_require_primary_key = 0;")

From efb26631a5f7197325e7459c73278379be15abf5 Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Fri, 5 Jan 2024 14:37:41 +0100
Subject: [PATCH 08/35] Fix database connection URL in test-migrations.sh
 script

---
 scripts/test-migrations.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/test-migrations.sh b/scripts/test-migrations.sh
index 21da55af6a1..8bb55be0aa5 100755
--- a/scripts/test-migrations.sh
+++ b/scripts/test-migrations.sh
@@ -88,7 +88,7 @@ do
     if [ "$1" == "mysql" ]; then
         zenml connect --url mysql://127.0.0.1/zenml --username root --password password
     elif [ "$1" == "mariadb" ]; then
-        zenml connect --url mariadb://127.0.0.1/zenml --username root --password password
+        zenml connect --url mysql://127.0.0.1/zenml --username root --password password
     fi
 
     # Run the tests for this version
@@ -115,7 +115,7 @@ pip3 install importlib_metadata
 if [ "$1" == "mysql" ]; then
     zenml connect --url mysql://127.0.0.1/zenml --username root --password password
 elif [ "$1" == "mariadb" ]; then
-    zenml connect --url mariadb://127.0.0.1/zenml --username root --password password
+    zenml connect --url mysql://127.0.0.1/zenml --username root --password password
 fi
 
 run_tests_for_version current_branch_mysql

From 915ccb5539404e3934e0584620204b660af52361 Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Fri, 5 Jan 2024 15:23:49 +0100
Subject: [PATCH 09/35] Adjust settings for MySQL and MariaDB engines in
 migration script

---
 ...aa_add_name_column_to_input_artifact_pk.py | 34 +++++++++----------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py b/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py
index 1df3e241df8..e13c3ad3a54 100644
--- a/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py
+++ b/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py
@@ -125,20 +125,20 @@ def downgrade() -> None:
 
 
 def _disable_primary_key_requirement_if_necessary() -> None:
-    """Disables MySQL primary key requirement if necessary."""
-    if op.get_bind().engine.name == "mysql":
-        server_version_info = op.get_bind().engine.dialect.server_version_info
-        if server_version_info and server_version_info >= (8, 0, 13):
-            potential_session_var = (
-                op.get_bind()
-                .execute(
-                    text(
-                        'SHOW SESSION VARIABLES LIKE "sql_require_primary_key";'
-                    )
-                )
-                .fetchone()
-            )
-            if potential_session_var and potential_session_var[1] == "ON":
-                # Temporarily disable this MySQL setting so we can update the
-                # primary key
-                op.execute("SET SESSION sql_require_primary_key = 0;")
+    """Adjusts settings based on database engine requirements."""
+    engine = op.get_bind().engine
+    engine_name = engine.name
+    server_version_info = engine.dialect.server_version_info
+
+    if engine_name == "mysql" and server_version_info >= (8, 0, 13):
+        potential_session_var = engine.execute(
+            text('SHOW SESSION VARIABLES LIKE "sql_require_primary_key";')
+        ).fetchone()
+        if potential_session_var and potential_session_var[1] == "ON":
+            # Temporarily disable this MySQL setting for primary key modification
+            op.execute("SET SESSION sql_require_primary_key = 0;")
+    elif engine_name == "mariadb":
+        # MariaDB does not require a similar setting, so skip this step
+        pass
+    else:
+        raise NotImplementedError(f"Unsupported engine: {engine_name}")

From f02471f31ac9424fd7712570a44c5203a6a8af7d Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Fri, 5 Jan 2024 15:30:14 +0100
Subject: [PATCH 10/35] log values

---
 .../728c6369cfaa_add_name_column_to_input_artifact_pk.py    | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py b/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py
index e13c3ad3a54..ac4bc58799c 100644
--- a/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py
+++ b/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py
@@ -14,6 +14,10 @@
 branch_labels = None
 depends_on = None
 
+import logging
+
+logger = logging.getLogger(__name__)
+
 
 def upgrade() -> None:
     """Upgrade database schema and/or data, creating a new revision.
@@ -129,6 +133,8 @@ def _disable_primary_key_requirement_if_necessary() -> None:
     engine = op.get_bind().engine
     engine_name = engine.name
     server_version_info = engine.dialect.server_version_info
+    logger.info(f"Detected database engine: {engine_name}")
+    logger.info(f"Detected database version: {server_version_info}")
 
     if engine_name == "mysql" and server_version_info >= (8, 0, 13):
         potential_session_var = engine.execute(

From 4337882ba4f9bd800f2bf50fe8dd701e2ad10fdc Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Fri, 5 Jan 2024 15:31:49 +0100
Subject: [PATCH 11/35] temporarily disable full CI runs for testing

---
 .github/workflows/ci.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index b545d7c0286..ac7432a5dc3 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -3,12 +3,12 @@ name: Build, Lint, Unit, Integration and Templates Tests
 on:
   workflow_dispatch:
   workflow_call:
-  push:
-    branches: ["main"]
-    paths-ignore: ["docs/**", "docker/**", "*", "!pyproject.toml", "**.md"]
-  pull_request:
-    types: [opened, synchronize, ready_for_review]
-    paths-ignore: ["docs/**", "docker/**", "*", "!pyproject.toml", "**.md"]
+  # push:
+  #   branches: ["main"]
+  #   paths-ignore: ["docs/**", "docker/**", "*", "!pyproject.toml", "**.md"]
+  # pull_request:
+  #   types: [opened, synchronize, ready_for_review]
+  #   paths-ignore: ["docs/**", "docker/**", "*", "!pyproject.toml", "**.md"]
 
 concurrency:
   # New commit on branch cancels running workflows of the same branch

From 0b6190f95ff4d40215fcda0cca810f552a9b8e69 Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Fri, 5 Jan 2024 16:58:08 +0100
Subject: [PATCH 12/35] print values

---
 .../728c6369cfaa_add_name_column_to_input_artifact_pk.py  | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py b/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py
index ac4bc58799c..8207dfcb2a0 100644
--- a/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py
+++ b/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py
@@ -14,10 +14,6 @@
 branch_labels = None
 depends_on = None
 
-import logging
-
-logger = logging.getLogger(__name__)
-
 
 def upgrade() -> None:
     """Upgrade database schema and/or data, creating a new revision.
@@ -133,8 +129,8 @@ def _disable_primary_key_requirement_if_necessary() -> None:
     engine = op.get_bind().engine
     engine_name = engine.name
     server_version_info = engine.dialect.server_version_info
-    logger.info(f"Detected database engine: {engine_name}")
-    logger.info(f"Detected database version: {server_version_info}")
+    print(f"Detected database engine: {engine_name}")
+    print(f"Detected database version: {server_version_info}")
 
     if engine_name == "mysql" and server_version_info >= (8, 0, 13):
         potential_session_var = engine.execute(

From 9b8ee69e7849943b76aa7f62780a1cd2d9ae7321 Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Fri, 5 Jan 2024 18:26:37 +0100
Subject: [PATCH 13/35] trying a different connection approach

---
 scripts/test-migrations.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/test-migrations.sh b/scripts/test-migrations.sh
index 8bb55be0aa5..ced615dbbc7 100755
--- a/scripts/test-migrations.sh
+++ b/scripts/test-migrations.sh
@@ -88,7 +88,7 @@ do
     if [ "$1" == "mysql" ]; then
         zenml connect --url mysql://127.0.0.1/zenml --username root --password password
     elif [ "$1" == "mariadb" ]; then
-        zenml connect --url mysql://127.0.0.1/zenml --username root --password password
+        zenml connect --url mariadb+mariadbconnector://127.0.0.1/zenml --username root --password password
     fi
 
     # Run the tests for this version

From fed45a4a92b1fd49a182aa18014a7eccf0c64280 Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Fri, 5 Jan 2024 18:32:17 +0100
Subject: [PATCH 14/35] trying innodb

---
 scripts/test-migrations.sh                                  | 2 +-
 .../728c6369cfaa_add_name_column_to_input_artifact_pk.py    | 6 ++----
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/scripts/test-migrations.sh b/scripts/test-migrations.sh
index ced615dbbc7..8bb55be0aa5 100755
--- a/scripts/test-migrations.sh
+++ b/scripts/test-migrations.sh
@@ -88,7 +88,7 @@ do
     if [ "$1" == "mysql" ]; then
         zenml connect --url mysql://127.0.0.1/zenml --username root --password password
     elif [ "$1" == "mariadb" ]; then
-        zenml connect --url mariadb+mariadbconnector://127.0.0.1/zenml --username root --password password
+        zenml connect --url mysql://127.0.0.1/zenml --username root --password password
     fi
 
     # Run the tests for this version
diff --git a/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py b/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py
index 8207dfcb2a0..6a862cba8d6 100644
--- a/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py
+++ b/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py
@@ -49,9 +49,7 @@ def upgrade() -> None:
         )
 
         # Update the PK
-        batch_op.drop_constraint(
-            constraint_name=constraint_name, type_="primary"
-        )
+        batch_op.drop_constraint(constraint_name=constraint_name, type_="primary")
         batch_op.create_primary_key(
             constraint_name="pk_step_run_input_artifact",
             columns=["step_id", "artifact_id", "name"],
@@ -139,7 +137,7 @@ def _disable_primary_key_requirement_if_necessary() -> None:
         if potential_session_var and potential_session_var[1] == "ON":
             # Temporarily disable this MySQL setting for primary key modification
             op.execute("SET SESSION sql_require_primary_key = 0;")
-    elif engine_name == "mariadb":
+    elif engine_name == "InnoDB":
         # MariaDB does not require a similar setting, so skip this step
         pass
     else:

From d4e617186c295d488669d828b73dd9a4be4fd4e2 Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Mon, 8 Jan 2024 09:24:33 +0100
Subject: [PATCH 15/35] Add support for MariaDB in primary key modification

---
 .../728c6369cfaa_add_name_column_to_input_artifact_pk.py        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py b/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py
index 6a862cba8d6..11030f245f6 100644
--- a/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py
+++ b/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py
@@ -137,7 +137,7 @@ def _disable_primary_key_requirement_if_necessary() -> None:
         if potential_session_var and potential_session_var[1] == "ON":
             # Temporarily disable this MySQL setting for primary key modification
             op.execute("SET SESSION sql_require_primary_key = 0;")
-    elif engine_name == "InnoDB":
+    elif engine_name == "InnoDB" or engine_name.lower() == "mariadb":
         # MariaDB does not require a similar setting, so skip this step
         pass
     else:

From 8b628ceff6d199c4363971b7530736c436933d4f Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Mon, 8 Jan 2024 09:26:16 +0100
Subject: [PATCH 16/35] split mariadb from mysql db migration testing

---
 ...grations.sh => test-migrations-mariadb.sh} |   0
 scripts/test-migrations-mysql.sh              | 132 ++++++++++++++++++
 2 files changed, 132 insertions(+)
 rename scripts/{test-migrations.sh => test-migrations-mariadb.sh} (100%)
 create mode 100755 scripts/test-migrations-mysql.sh

diff --git a/scripts/test-migrations.sh b/scripts/test-migrations-mariadb.sh
similarity index 100%
rename from scripts/test-migrations.sh
rename to scripts/test-migrations-mariadb.sh
diff --git a/scripts/test-migrations-mysql.sh b/scripts/test-migrations-mysql.sh
new file mode 100755
index 00000000000..8bb55be0aa5
--- /dev/null
+++ b/scripts/test-migrations-mysql.sh
@@ -0,0 +1,132 @@
+#!/bin/bash
+
+DB="sqlite"
+DB_STARTUP_DELAY=30 # Time in seconds to wait for the database container to start
+
+if [ -z "$1" ]; then
+  echo "No argument passed, using default: $DB"
+else
+  DB="$1"
+fi
+
+function run_tests_for_version() {
+    set -e  # Exit immediately if a command exits with a non-zero status
+    local VERSION=$1
+
+    echo "===== Testing version $VERSION ====="
+    # Initialize zenml with the appropriate template
+    # hardcoded to 0.43.0 since this is the latest template-starter repo
+    # release tag
+    copier copy -l --trust -r release/0.43.0 https://github.com/zenml-io/template-starter.git test_starter
+    cd test_starter
+
+    export ZENML_ANALYTICS_OPT_IN=false
+    export ZENML_DEBUG=true
+
+    echo "===== Installing sklearn integration ====="
+    zenml integration install sklearn -y
+
+    echo "===== Running starter template pipeline ====="
+    python3 run.py
+    # Add additional CLI tests here
+    zenml version
+
+    # Confirm DB works and is accessible
+    zenml pipeline runs list
+
+    cd ..
+    rm -rf test_starter template-starter
+    echo "===== Finished testing version $VERSION ====="
+}
+
+if [ "$1" == "mysql" ]; then
+    echo "===== Testing MySQL ====="
+    # run a mysql instance in docker
+    docker run --name mysql -d -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password mysql:latest
+    # mysql takes a while to start up
+    sleep $DB_STARTUP_DELAY
+elif [ "$1" == "mariadb" ]; then
+    echo "===== Testing MariaDB ====="
+    # run a mariadb instance in docker
+    docker run --name mariadb -d -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password mariadb:10.6
+    # mariadb takes a while to start up
+    sleep $DB_STARTUP_DELAY
+fi
+
+# List of versions to test
+VERSIONS=("0.40.0" "0.40.3" "0.41.0" "0.43.0" "0.44.1" "0.44.3" "0.45.2" "0.45.3" "0.45.4" "0.45.5" "0.45.6" "0.46.0" "0.47.0" "0.50.0" "0.51.0" "0.52.0")
+
+# Start completely fresh
+rm -rf ~/.config/zenml
+
+for VERSION in "${VERSIONS[@]}"
+do
+    set -e  # Exit immediately if a command exits with a non-zero status
+    # Create a new virtual environment
+    python3 -m venv ".venv-$VERSION"
+    source ".venv-$VERSION/bin/activate"
+
+    # Install the specific version
+    pip3 install -U pip setuptools wheel
+    pip3 install "zenml[templates,server]==$VERSION"
+    # handles unpinned sqlmodel dependency in older versions
+    pip3 install "sqlmodel==0.0.8" "bcrypt==4.0.1"
+
+    # Get the major and minor version of Python
+    PYTHON_VERSION=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
+
+    # Check if the Python version is 3.9 and VERSION is > 0.47.0
+    if [[ "$PYTHON_VERSION" == "3.9" ]]; then
+        case "$VERSION" in
+            "0.47.0"|"0.50.0"|"0.51.0"|"0.52.0")
+                pip3 install importlib_metadata
+                ;;
+        esac
+    fi
+
+
+    if [ "$1" == "mysql" ]; then
+        zenml connect --url mysql://127.0.0.1/zenml --username root --password password
+    elif [ "$1" == "mariadb" ]; then
+        zenml connect --url mysql://127.0.0.1/zenml --username root --password password
+    fi
+
+    # Run the tests for this version
+    run_tests_for_version $VERSION
+
+    if [ "$1" == "mysql" ]; then
+        zenml disconnect
+        sleep 5
+    fi
+
+    deactivate
+done
+
+
+# Test the most recent migration with MySQL
+set -e
+python3 -m venv ".venv-current-branch"
+source ".venv-current-branch/bin/activate"
+
+pip3 install -U pip setuptools wheel
+pip3 install -e ".[templates,server]"
+pip3 install importlib_metadata
+
+if [ "$1" == "mysql" ]; then
+    zenml connect --url mysql://127.0.0.1/zenml --username root --password password
+elif [ "$1" == "mariadb" ]; then
+    zenml connect --url mysql://127.0.0.1/zenml --username root --password password
+fi
+
+run_tests_for_version current_branch_mysql
+
+if [ "$1" == "mysql" ]; then
+    zenml disconnect
+    docker rm -f mysql
+elif [ "$1" == "mariadb" ]; then
+    zenml disconnect
+    docker rm -f mariadb
+fi
+
+deactivate
+

From 1bf185e17f1e0c50a3106de67bded153eb3b4ca9 Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Mon, 8 Jan 2024 10:35:40 +0100
Subject: [PATCH 17/35] split mariadb from mysql db migration testing (core)

---
 scripts/test-migrations-mariadb.sh | 71 ++++++------------------------
 1 file changed, 13 insertions(+), 58 deletions(-)

diff --git a/scripts/test-migrations-mariadb.sh b/scripts/test-migrations-mariadb.sh
index 8bb55be0aa5..0e91337cc8d 100755
--- a/scripts/test-migrations-mariadb.sh
+++ b/scripts/test-migrations-mariadb.sh
@@ -1,14 +1,8 @@
 #!/bin/bash
 
-DB="sqlite"
+DB="mariadb"
 DB_STARTUP_DELAY=30 # Time in seconds to wait for the database container to start
 
-if [ -z "$1" ]; then
-  echo "No argument passed, using default: $DB"
-else
-  DB="$1"
-fi
-
 function run_tests_for_version() {
     set -e  # Exit immediately if a command exits with a non-zero status
     local VERSION=$1
@@ -39,22 +33,14 @@ function run_tests_for_version() {
     echo "===== Finished testing version $VERSION ====="
 }
 
-if [ "$1" == "mysql" ]; then
-    echo "===== Testing MySQL ====="
-    # run a mysql instance in docker
-    docker run --name mysql -d -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password mysql:latest
-    # mysql takes a while to start up
-    sleep $DB_STARTUP_DELAY
-elif [ "$1" == "mariadb" ]; then
-    echo "===== Testing MariaDB ====="
-    # run a mariadb instance in docker
-    docker run --name mariadb -d -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password mariadb:10.6
-    # mariadb takes a while to start up
-    sleep $DB_STARTUP_DELAY
-fi
+echo "===== Testing MariaDB ====="
+# run a mariadb instance in docker
+docker run --name mariadb -d -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password mariadb:10.6
+# mariadb takes a while to start up
+sleep $DB_STARTUP_DELAY
 
 # List of versions to test
-VERSIONS=("0.40.0" "0.40.3" "0.41.0" "0.43.0" "0.44.1" "0.44.3" "0.45.2" "0.45.3" "0.45.4" "0.45.5" "0.45.6" "0.46.0" "0.47.0" "0.50.0" "0.51.0" "0.52.0")
+VERSIONS=("0.53.2")
 
 # Start completely fresh
 rm -rf ~/.config/zenml
@@ -69,35 +55,14 @@ do
     # Install the specific version
     pip3 install -U pip setuptools wheel
     pip3 install "zenml[templates,server]==$VERSION"
-    # handles unpinned sqlmodel dependency in older versions
-    pip3 install "sqlmodel==0.0.8" "bcrypt==4.0.1"
-
-    # Get the major and minor version of Python
-    PYTHON_VERSION=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
-
-    # Check if the Python version is 3.9 and VERSION is > 0.47.0
-    if [[ "$PYTHON_VERSION" == "3.9" ]]; then
-        case "$VERSION" in
-            "0.47.0"|"0.50.0"|"0.51.0"|"0.52.0")
-                pip3 install importlib_metadata
-                ;;
-        esac
-    fi
 
-
-    if [ "$1" == "mysql" ]; then
-        zenml connect --url mysql://127.0.0.1/zenml --username root --password password
-    elif [ "$1" == "mariadb" ]; then
-        zenml connect --url mysql://127.0.0.1/zenml --username root --password password
-    fi
+    zenml connect --url mysql://127.0.0.1/zenml --username root --password password
 
     # Run the tests for this version
     run_tests_for_version $VERSION
 
-    if [ "$1" == "mysql" ]; then
-        zenml disconnect
-        sleep 5
-    fi
+    zenml disconnect
+    sleep 5
 
     deactivate
 done
@@ -112,21 +77,11 @@ pip3 install -U pip setuptools wheel
 pip3 install -e ".[templates,server]"
 pip3 install importlib_metadata
 
-if [ "$1" == "mysql" ]; then
-    zenml connect --url mysql://127.0.0.1/zenml --username root --password password
-elif [ "$1" == "mariadb" ]; then
-    zenml connect --url mysql://127.0.0.1/zenml --username root --password password
-fi
+zenml connect --url mysql://127.0.0.1/zenml --username root --password password
 
 run_tests_for_version current_branch_mysql
 
-if [ "$1" == "mysql" ]; then
-    zenml disconnect
-    docker rm -f mysql
-elif [ "$1" == "mariadb" ]; then
-    zenml disconnect
-    docker rm -f mariadb
-fi
+zenml disconnect
+docker rm -f mariadb
 
 deactivate
-

From 68b004f645da8b06470ae00519406a61735aa6ff Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Mon, 8 Jan 2024 10:37:27 +0100
Subject: [PATCH 18/35] Remove support for MariaDB and add version 0.53.1

---
 scripts/test-migrations-mysql.sh | 15 +--------------
 1 file changed, 1 insertion(+), 14 deletions(-)

diff --git a/scripts/test-migrations-mysql.sh b/scripts/test-migrations-mysql.sh
index 8bb55be0aa5..306cef1b3ad 100755
--- a/scripts/test-migrations-mysql.sh
+++ b/scripts/test-migrations-mysql.sh
@@ -45,16 +45,10 @@ if [ "$1" == "mysql" ]; then
     docker run --name mysql -d -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password mysql:latest
     # mysql takes a while to start up
     sleep $DB_STARTUP_DELAY
-elif [ "$1" == "mariadb" ]; then
-    echo "===== Testing MariaDB ====="
-    # run a mariadb instance in docker
-    docker run --name mariadb -d -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password mariadb:10.6
-    # mariadb takes a while to start up
-    sleep $DB_STARTUP_DELAY
 fi
 
 # List of versions to test
-VERSIONS=("0.40.0" "0.40.3" "0.41.0" "0.43.0" "0.44.1" "0.44.3" "0.45.2" "0.45.3" "0.45.4" "0.45.5" "0.45.6" "0.46.0" "0.47.0" "0.50.0" "0.51.0" "0.52.0")
+VERSIONS=("0.40.0" "0.40.3" "0.41.0" "0.43.0" "0.44.1" "0.44.3" "0.45.2" "0.45.3" "0.45.4" "0.45.5" "0.45.6" "0.46.0" "0.47.0" "0.50.0" "0.51.0" "0.52.0" "0.53.0" "0.53.1")
 
 # Start completely fresh
 rm -rf ~/.config/zenml
@@ -87,8 +81,6 @@ do
 
     if [ "$1" == "mysql" ]; then
         zenml connect --url mysql://127.0.0.1/zenml --username root --password password
-    elif [ "$1" == "mariadb" ]; then
-        zenml connect --url mysql://127.0.0.1/zenml --username root --password password
     fi
 
     # Run the tests for this version
@@ -114,8 +106,6 @@ pip3 install importlib_metadata
 
 if [ "$1" == "mysql" ]; then
     zenml connect --url mysql://127.0.0.1/zenml --username root --password password
-elif [ "$1" == "mariadb" ]; then
-    zenml connect --url mysql://127.0.0.1/zenml --username root --password password
 fi
 
 run_tests_for_version current_branch_mysql
@@ -123,9 +113,6 @@ run_tests_for_version current_branch_mysql
 if [ "$1" == "mysql" ]; then
     zenml disconnect
     docker rm -f mysql
-elif [ "$1" == "mariadb" ]; then
-    zenml disconnect
-    docker rm -f mariadb
 fi
 
 deactivate

From e7a4e3e2bc4bc5cbed5a525860f50ec07bb994db Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Mon, 8 Jan 2024 10:38:44 +0100
Subject: [PATCH 19/35] Split out mariadb from mysql in migration testing

---
 .github/workflows/testing_this_now.yml | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/testing_this_now.yml b/.github/workflows/testing_this_now.yml
index 180f6bb22d2..6f2972348c8 100644
--- a/.github/workflows/testing_this_now.yml
+++ b/.github/workflows/testing_this_now.yml
@@ -15,4 +15,15 @@ jobs:
         with:
           python-version: '3.9'
       - name: Test migrations across versions
-        run: bash scripts/test-migrations.sh mariadb
+        run: bash scripts/test-migrations-mariadb.sh
+  mysql-db-migration-testing:
+    runs-on: ubuntu-dind-runners
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4.1.1
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v4.8.0
+        with:
+          python-version: '3.9'
+      - name: Test migrations across versions
+        run: bash scripts/test-migrations-mysql.sh

From 0baaed98b44bd189d8240b75279f5398dda70e2e Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Mon, 8 Jan 2024 20:09:28 +0100
Subject: [PATCH 20/35] add latest zenml version

---
 scripts/test-migrations-mariadb.sh | 2 +-
 scripts/test-migrations-mysql.sh   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/test-migrations-mariadb.sh b/scripts/test-migrations-mariadb.sh
index 0e91337cc8d..4fe147946d9 100755
--- a/scripts/test-migrations-mariadb.sh
+++ b/scripts/test-migrations-mariadb.sh
@@ -40,7 +40,7 @@ docker run --name mariadb -d -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password mariad
 sleep $DB_STARTUP_DELAY
 
 # List of versions to test
-VERSIONS=("0.53.2")
+VERSIONS=("0.54.0")
 
 # Start completely fresh
 rm -rf ~/.config/zenml
diff --git a/scripts/test-migrations-mysql.sh b/scripts/test-migrations-mysql.sh
index 306cef1b3ad..92438335b37 100755
--- a/scripts/test-migrations-mysql.sh
+++ b/scripts/test-migrations-mysql.sh
@@ -48,7 +48,7 @@ if [ "$1" == "mysql" ]; then
 fi
 
 # List of versions to test
-VERSIONS=("0.40.0" "0.40.3" "0.41.0" "0.43.0" "0.44.1" "0.44.3" "0.45.2" "0.45.3" "0.45.4" "0.45.5" "0.45.6" "0.46.0" "0.47.0" "0.50.0" "0.51.0" "0.52.0" "0.53.0" "0.53.1")
+VERSIONS=("0.40.0" "0.40.3" "0.41.0" "0.43.0" "0.44.1" "0.44.3" "0.45.2" "0.45.3" "0.45.4" "0.45.5" "0.45.6" "0.46.0" "0.47.0" "0.50.0" "0.51.0" "0.52.0" "0.53.0" "0.53.1" "0.54.0")
 
 # Start completely fresh
 rm -rf ~/.config/zenml

From 5b6d0849b73b524bf628c1a1d2e6ea345c2b02a9 Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Mon, 8 Jan 2024 20:20:52 +0100
Subject: [PATCH 21/35] Update migration testing scripts for different
 databases

---
 .github/workflows/release.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 83f4222fede..e9afd32796a 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -49,7 +49,7 @@ jobs:
         python-version: "3.9"
 
     - name: Test migrations across versions
-      run: bash scripts/test-migrations.sh mysql
+      run: bash scripts/test-migrations-mysql.sh mysql
 
   sqlite-db-migration-testing:
     runs-on: ubuntu-dind-runners
@@ -64,7 +64,7 @@ jobs:
         python-version: "3.9"
 
     - name: Test migrations across versions
-      run: bash scripts/test-migrations.sh sqlite
+      run: bash scripts/test-migrations-mysql.sh sqlite
 
   mariadb-db-migration-testing:
     runs-on: ubuntu-dind-runners
@@ -79,7 +79,7 @@ jobs:
         python-version: "3.9"
 
     - name: Test migrations across versions
-      run: bash scripts/test-migrations.sh mariadb
+      run: bash scripts/test-migrations-mariadb.sh
 
   publish-python-package:
     needs: [setup-and-test, mlstacks-compatibility-check, sqlite-db-migration-testing, mysql-db-migration-testing]

From 7472dd9cd1db1bbab987d255cb5c60be9eab4616 Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Mon, 8 Jan 2024 20:21:40 +0100
Subject: [PATCH 22/35] restore the CI to working state

---
 .github/workflows/ci.yml               | 12 +++++------
 .github/workflows/testing_this_now.yml | 29 --------------------------
 2 files changed, 6 insertions(+), 35 deletions(-)
 delete mode 100644 .github/workflows/testing_this_now.yml

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c547524adb7..a6f2c5f1c13 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -3,12 +3,12 @@ name: Build, Lint, Unit, Integration and Templates Tests
 on:
   workflow_dispatch:
   workflow_call:
-  # push:
-  #   branches: ["main"]
-  #   paths-ignore: ["docs/**", "docker/**", "*", "!pyproject.toml", "**.md"]
-  # pull_request:
-  #   types: [opened, synchronize, ready_for_review]
-  #   paths-ignore: ["docs/**", "docker/**", "*", "!pyproject.toml", "**.md"]
+  push:
+    branches: ["main"]
+    paths-ignore: ["docs/**", "docker/**", "*", "!pyproject.toml", "**.md"]
+  pull_request:
+    types: [opened, synchronize, ready_for_review]
+    paths-ignore: ["docs/**", "docker/**", "*", "!pyproject.toml", "**.md"]
 
 concurrency:
   # New commit on branch cancels running workflows of the same branch
diff --git a/.github/workflows/testing_this_now.yml b/.github/workflows/testing_this_now.yml
deleted file mode 100644
index 6f2972348c8..00000000000
--- a/.github/workflows/testing_this_now.yml
+++ /dev/null
@@ -1,29 +0,0 @@
----
-name: TEST MariaDB DB migrations
-on:
-  pull_request:
-    types: [opened, synchronize, ready_for_review]
-    paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md']
-jobs:
-  mariadb-db-migration-testing:
-    runs-on: ubuntu-dind-runners
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4.1.1
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v4.8.0
-        with:
-          python-version: '3.9'
-      - name: Test migrations across versions
-        run: bash scripts/test-migrations-mariadb.sh
-  mysql-db-migration-testing:
-    runs-on: ubuntu-dind-runners
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4.1.1
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v4.8.0
-        with:
-          python-version: '3.9'
-      - name: Test migrations across versions
-        run: bash scripts/test-migrations-mysql.sh

From 98d9e83e5221222cd40b075b520e65539f50885b Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Mon, 8 Jan 2024 20:22:23 +0100
Subject: [PATCH 23/35] formatting

---
 .../728c6369cfaa_add_name_column_to_input_artifact_pk.py      | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py b/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py
index 4d489e323b9..4055fe624a2 100644
--- a/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py
+++ b/src/zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py
@@ -49,7 +49,9 @@ def upgrade() -> None:
         )
 
         # Update the PK
-        batch_op.drop_constraint(constraint_name=constraint_name, type_="primary")
+        batch_op.drop_constraint(
+            constraint_name=constraint_name, type_="primary"
+        )
         batch_op.create_primary_key(
             constraint_name="pk_step_run_input_artifact",
             columns=["step_id", "artifact_id", "name"],

From 1f062259005d39ebd67c90415e1e08eb1248e604 Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Tue, 9 Jan 2024 15:00:56 +0100
Subject: [PATCH 24/35] use zenml init

---
 .github/workflows/ci.yml           | 12 ++++-----
 .github/workflows/release.yml      |  1 +
 .github/workflows/test_me_now.yml  | 43 ++++++++++++++++++++++++++++++
 scripts/test-migrations-mariadb.sh |  3 ++-
 scripts/test-migrations-mysql.sh   |  3 ++-
 5 files changed, 54 insertions(+), 8 deletions(-)
 create mode 100644 .github/workflows/test_me_now.yml

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 59f7f4a42dc..8e606878b90 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -3,12 +3,12 @@ name: Build, Lint, Unit, Integration and Templates Tests
 on:
   workflow_dispatch:
   workflow_call:
-  push:
-    branches: [main]
-    paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md']
-  pull_request:
-    types: [opened, synchronize, ready_for_review]
-    paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md']
+  # push:
+  #   branches: [main]
+  #   paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md']
+  # pull_request:
+  #   types: [opened, synchronize, ready_for_review]
+  #   paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md']
 concurrency:
   # New commit on branch cancels running workflows of the same branch
   group: ${{ github.workflow }}-${{ github.ref }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 8c21a6398b8..5489bf5bdad 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -67,6 +67,7 @@ jobs:
       - mlstacks-compatibility-check
       - sqlite-db-migration-testing
       - mysql-db-migration-testing
+      - mariadb-db-migration-testing
     uses: ./.github/workflows/publish_to_pypi.yml
     secrets: inherit
   wait-for-package-release:
diff --git a/.github/workflows/test_me_now.yml b/.github/workflows/test_me_now.yml
new file mode 100644
index 00000000000..917577ff693
--- /dev/null
+++ b/.github/workflows/test_me_now.yml
@@ -0,0 +1,43 @@
+---
+name: TESTING DB FIXES
+on:
+  push:
+    branches: [main]
+    paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md']
+  pull_request:
+    types: [opened, synchronize, ready_for_review]
+    paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md']
+jobs:
+  mysql-db-migration-testing:
+    runs-on: ubuntu-dind-runners
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4.1.1
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v5.0.0
+        with:
+          python-version: '3.9'
+      - name: Test migrations across versions
+        run: bash scripts/test-migrations-mysql.sh mysql
+    sqlite-db-migration-testing:
+      runs-on: ubuntu-dind-runners
+      steps:
+        - name: Checkout code
+          uses: actions/checkout@v4.1.1
+        - name: Set up Python 3.9
+          uses: actions/setup-python@v5.0.0
+          with:
+            python-version: '3.9'
+        - name: Test migrations across versions
+          run: bash scripts/test-migrations-mysql.sh sqlite
+    mariadb-db-migration-testing:
+      runs-on: ubuntu-dind-runners
+      steps:
+        - name: Checkout code
+          uses: actions/checkout@v4.1.1
+        - name: Set up Python 3.9
+          uses: actions/setup-python@v4.8.0
+          with:
+            python-version: '3.9'
+        - name: Test migrations across versions
+          run: bash scripts/test-migrations-mariadb.sh
diff --git a/scripts/test-migrations-mariadb.sh b/scripts/test-migrations-mariadb.sh
index 4fe147946d9..fa563df2763 100755
--- a/scripts/test-migrations-mariadb.sh
+++ b/scripts/test-migrations-mariadb.sh
@@ -11,7 +11,8 @@ function run_tests_for_version() {
     # Initialize zenml with the appropriate template
     # hardcoded to 0.43.0 since this is the latest template-starter repo
     # release tag
-    copier copy -l --trust -r release/0.43.0 https://github.com/zenml-io/template-starter.git test_starter
+    mkdir test_starter
+    zenml init --template starter --path test_starter --template-with-defaults
     cd test_starter
 
     export ZENML_ANALYTICS_OPT_IN=false
diff --git a/scripts/test-migrations-mysql.sh b/scripts/test-migrations-mysql.sh
index 92438335b37..502e9bc35d8 100755
--- a/scripts/test-migrations-mysql.sh
+++ b/scripts/test-migrations-mysql.sh
@@ -17,7 +17,8 @@ function run_tests_for_version() {
     # Initialize zenml with the appropriate template
     # hardcoded to 0.43.0 since this is the latest template-starter repo
     # release tag
-    copier copy -l --trust -r release/0.43.0 https://github.com/zenml-io/template-starter.git test_starter
+    mkdir test_starter
+    zenml init --template starter --path test_starter --template-with-defaults
     cd test_starter
 
     export ZENML_ANALYTICS_OPT_IN=false

From c16c65368137ed77b5a486e900347376a7521d6a Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Tue, 9 Jan 2024 15:06:37 +0100
Subject: [PATCH 25/35] reformat yml

---
 .github/workflows/test_me_now.yml | 44 +++++++++++++++----------------
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/test_me_now.yml b/.github/workflows/test_me_now.yml
index 917577ff693..c0f2b60b139 100644
--- a/.github/workflows/test_me_now.yml
+++ b/.github/workflows/test_me_now.yml
@@ -19,25 +19,25 @@ jobs:
           python-version: '3.9'
       - name: Test migrations across versions
         run: bash scripts/test-migrations-mysql.sh mysql
-    sqlite-db-migration-testing:
-      runs-on: ubuntu-dind-runners
-      steps:
-        - name: Checkout code
-          uses: actions/checkout@v4.1.1
-        - name: Set up Python 3.9
-          uses: actions/setup-python@v5.0.0
-          with:
-            python-version: '3.9'
-        - name: Test migrations across versions
-          run: bash scripts/test-migrations-mysql.sh sqlite
-    mariadb-db-migration-testing:
-      runs-on: ubuntu-dind-runners
-      steps:
-        - name: Checkout code
-          uses: actions/checkout@v4.1.1
-        - name: Set up Python 3.9
-          uses: actions/setup-python@v4.8.0
-          with:
-            python-version: '3.9'
-        - name: Test migrations across versions
-          run: bash scripts/test-migrations-mariadb.sh
+  sqlite-db-migration-testing:
+    runs-on: ubuntu-dind-runners
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4.1.1
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v5.0.0
+        with:
+          python-version: '3.9'
+      - name: Test migrations across versions
+        run: bash scripts/test-migrations-mysql.sh sqlite
+  mariadb-db-migration-testing:
+    runs-on: ubuntu-dind-runners
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4.1.1
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v4.8.0
+        with:
+          python-version: '3.9'
+      - name: Test migrations across versions
+        run: bash scripts/test-migrations-mariadb.sh

From 6d6df663ea392a209073df3eb1b499aa6b1f9ba6 Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Tue, 9 Jan 2024 15:14:04 +0100
Subject: [PATCH 26/35] revert to copier instantiation

---
 scripts/test-migrations-mariadb.sh | 3 +--
 scripts/test-migrations-mysql.sh   | 3 +--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/scripts/test-migrations-mariadb.sh b/scripts/test-migrations-mariadb.sh
index fa563df2763..4fe147946d9 100755
--- a/scripts/test-migrations-mariadb.sh
+++ b/scripts/test-migrations-mariadb.sh
@@ -11,8 +11,7 @@ function run_tests_for_version() {
     # Initialize zenml with the appropriate template
     # hardcoded to 0.43.0 since this is the latest template-starter repo
     # release tag
-    mkdir test_starter
-    zenml init --template starter --path test_starter --template-with-defaults
+    copier copy -l --trust -r release/0.43.0 https://github.com/zenml-io/template-starter.git test_starter
     cd test_starter
 
     export ZENML_ANALYTICS_OPT_IN=false
diff --git a/scripts/test-migrations-mysql.sh b/scripts/test-migrations-mysql.sh
index 502e9bc35d8..92438335b37 100755
--- a/scripts/test-migrations-mysql.sh
+++ b/scripts/test-migrations-mysql.sh
@@ -17,8 +17,7 @@ function run_tests_for_version() {
     # Initialize zenml with the appropriate template
     # hardcoded to 0.43.0 since this is the latest template-starter repo
     # release tag
-    mkdir test_starter
-    zenml init --template starter --path test_starter --template-with-defaults
+    copier copy -l --trust -r release/0.43.0 https://github.com/zenml-io/template-starter.git test_starter
     cd test_starter
 
     export ZENML_ANALYTICS_OPT_IN=false

From f24392df5dfd61f01957bdfbf1e0bea8d49c17e3 Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Tue, 9 Jan 2024 15:23:30 +0100
Subject: [PATCH 27/35] add conditional check for version >= 0.43.0

---
 scripts/test-migrations-mariadb.sh | 15 +++++++++++----
 scripts/test-migrations-mysql.sh   | 16 ++++++++++++----
 2 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/scripts/test-migrations-mariadb.sh b/scripts/test-migrations-mariadb.sh
index 4fe147946d9..0d59e47b114 100755
--- a/scripts/test-migrations-mariadb.sh
+++ b/scripts/test-migrations-mariadb.sh
@@ -6,12 +6,19 @@ DB_STARTUP_DELAY=30 # Time in seconds to wait for the database container to star
 function run_tests_for_version() {
     set -e  # Exit immediately if a command exits with a non-zero status
     local VERSION=$1
+    local MIN_VERSION="0.43.0"
 
     echo "===== Testing version $VERSION ====="
-    # Initialize zenml with the appropriate template
-    # hardcoded to 0.43.0 since this is the latest template-starter repo
-    # release tag
-    copier copy -l --trust -r release/0.43.0 https://github.com/zenml-io/template-starter.git test_starter
+
+    # Compare semantic versions
+    if [[ $(printf "%s\n%s" "$VERSION" "$MIN_VERSION" | sort -V | head -n1) = "$MIN_VERSION" ]]; then
+        copier copy -l --trust -r release/0.43.0 https://github.com/zenml-io/template-starter.git test_starter
+    else
+        mkdir test_starter
+        zenml init --template starter --path test_starter --template-with-defaults
+        cd test_starter
+    fi
+
     cd test_starter
 
     export ZENML_ANALYTICS_OPT_IN=false
diff --git a/scripts/test-migrations-mysql.sh b/scripts/test-migrations-mysql.sh
index 92438335b37..15d15ed0375 100755
--- a/scripts/test-migrations-mysql.sh
+++ b/scripts/test-migrations-mysql.sh
@@ -12,12 +12,19 @@ fi
 function run_tests_for_version() {
     set -e  # Exit immediately if a command exits with a non-zero status
     local VERSION=$1
+    local MIN_VERSION="0.43.0"
 
     echo "===== Testing version $VERSION ====="
-    # Initialize zenml with the appropriate template
-    # hardcoded to 0.43.0 since this is the latest template-starter repo
-    # release tag
-    copier copy -l --trust -r release/0.43.0 https://github.com/zenml-io/template-starter.git test_starter
+
+    # Compare semantic versions
+    if [[ $(printf "%s\n%s" "$VERSION" "$MIN_VERSION" | sort -V | head -n1) = "$MIN_VERSION" ]]; then
+        copier copy -l --trust -r release/0.43.0 https://github.com/zenml-io/template-starter.git test_starter
+    else
+        mkdir test_starter
+        zenml init --template starter --path test_starter --template-with-defaults
+        cd test_starter
+    fi
+
     cd test_starter
 
     export ZENML_ANALYTICS_OPT_IN=false
@@ -39,6 +46,7 @@ function run_tests_for_version() {
     echo "===== Finished testing version $VERSION ====="
 }
 
+
 if [ "$1" == "mysql" ]; then
     echo "===== Testing MySQL ====="
     # run a mysql instance in docker

From aba3ee0fcdc92b9ffe2847fa0cdfd1bdde51357d Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Tue, 9 Jan 2024 15:31:39 +0100
Subject: [PATCH 28/35] compare set of vals instead of complicated semantic
 check

---
 scripts/test-migrations-mariadb.sh | 6 +++---
 scripts/test-migrations-mysql.sh   | 7 ++++---
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/scripts/test-migrations-mariadb.sh b/scripts/test-migrations-mariadb.sh
index 0d59e47b114..861b1d37888 100755
--- a/scripts/test-migrations-mariadb.sh
+++ b/scripts/test-migrations-mariadb.sh
@@ -6,12 +6,12 @@ DB_STARTUP_DELAY=30 # Time in seconds to wait for the database container to star
 function run_tests_for_version() {
     set -e  # Exit immediately if a command exits with a non-zero status
     local VERSION=$1
-    local MIN_VERSION="0.43.0"
+    local PRE_TEMPLATE_VERSIONS=("0.40.0" "0.40.3" "0.41.0")
 
     echo "===== Testing version $VERSION ====="
 
-    # Compare semantic versions
-    if [[ $(printf "%s\n%s" "$VERSION" "$MIN_VERSION" | sort -V | head -n1) = "$MIN_VERSION" ]]; then
+    # Check if VERSION is in VALID_VERSIONS
+    if printf '%s\n' "${PRE_TEMPLATE_VERSIONS[@]}" | grep -q "^$VERSION$"; then
         copier copy -l --trust -r release/0.43.0 https://github.com/zenml-io/template-starter.git test_starter
     else
         mkdir test_starter
diff --git a/scripts/test-migrations-mysql.sh b/scripts/test-migrations-mysql.sh
index 15d15ed0375..ab628ccfb73 100755
--- a/scripts/test-migrations-mysql.sh
+++ b/scripts/test-migrations-mysql.sh
@@ -12,12 +12,12 @@ fi
 function run_tests_for_version() {
     set -e  # Exit immediately if a command exits with a non-zero status
     local VERSION=$1
-    local MIN_VERSION="0.43.0"
+    local PRE_TEMPLATE_VERSIONS=("0.40.0" "0.40.3" "0.41.0")
 
     echo "===== Testing version $VERSION ====="
 
-    # Compare semantic versions
-    if [[ $(printf "%s\n%s" "$VERSION" "$MIN_VERSION" | sort -V | head -n1) = "$MIN_VERSION" ]]; then
+    # Check if VERSION is in VALID_VERSIONS
+    if printf '%s\n' "${PRE_TEMPLATE_VERSIONS[@]}" | grep -q "^$VERSION$"; then
         copier copy -l --trust -r release/0.43.0 https://github.com/zenml-io/template-starter.git test_starter
     else
         mkdir test_starter
@@ -47,6 +47,7 @@ function run_tests_for_version() {
 }
 
 
+
 if [ "$1" == "mysql" ]; then
     echo "===== Testing MySQL ====="
     # run a mysql instance in docker

From 15b2f929ed71b44e1784fa4f90a58653540de2a6 Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Tue, 9 Jan 2024 15:44:14 +0100
Subject: [PATCH 29/35] remove extra cd

---
 scripts/test-migrations-mariadb.sh | 1 -
 scripts/test-migrations-mysql.sh   | 1 -
 2 files changed, 2 deletions(-)

diff --git a/scripts/test-migrations-mariadb.sh b/scripts/test-migrations-mariadb.sh
index 861b1d37888..27be4ed5664 100755
--- a/scripts/test-migrations-mariadb.sh
+++ b/scripts/test-migrations-mariadb.sh
@@ -16,7 +16,6 @@ function run_tests_for_version() {
     else
         mkdir test_starter
         zenml init --template starter --path test_starter --template-with-defaults
-        cd test_starter
     fi
 
     cd test_starter
diff --git a/scripts/test-migrations-mysql.sh b/scripts/test-migrations-mysql.sh
index ab628ccfb73..3413cb1cc5f 100755
--- a/scripts/test-migrations-mysql.sh
+++ b/scripts/test-migrations-mysql.sh
@@ -22,7 +22,6 @@ function run_tests_for_version() {
     else
         mkdir test_starter
         zenml init --template starter --path test_starter --template-with-defaults
-        cd test_starter
     fi
 
     cd test_starter

From 60ab65a7dbedd787b996ad5a22557b050439d6be Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Tue, 9 Jan 2024 16:02:56 +0100
Subject: [PATCH 30/35] update mariadb script

---
 scripts/test-migrations-mariadb.sh | 19 ++++++-------------
 scripts/test-migrations-mysql.sh   |  3 ++-
 2 files changed, 8 insertions(+), 14 deletions(-)

diff --git a/scripts/test-migrations-mariadb.sh b/scripts/test-migrations-mariadb.sh
index 27be4ed5664..5a6015b895c 100755
--- a/scripts/test-migrations-mariadb.sh
+++ b/scripts/test-migrations-mariadb.sh
@@ -6,18 +6,11 @@ DB_STARTUP_DELAY=30 # Time in seconds to wait for the database container to star
 function run_tests_for_version() {
     set -e  # Exit immediately if a command exits with a non-zero status
     local VERSION=$1
-    local PRE_TEMPLATE_VERSIONS=("0.40.0" "0.40.3" "0.41.0")
 
     echo "===== Testing version $VERSION ====="
 
-    # Check if VERSION is in VALID_VERSIONS
-    if printf '%s\n' "${PRE_TEMPLATE_VERSIONS[@]}" | grep -q "^$VERSION$"; then
-        copier copy -l --trust -r release/0.43.0 https://github.com/zenml-io/template-starter.git test_starter
-    else
-        mkdir test_starter
-        zenml init --template starter --path test_starter --template-with-defaults
-    fi
-
+    mkdir test_starter
+    zenml init --template starter --path test_starter --template-with-defaults
     cd test_starter
 
     export ZENML_ANALYTICS_OPT_IN=false
@@ -35,7 +28,7 @@ function run_tests_for_version() {
     zenml pipeline runs list
 
     cd ..
-    rm -rf test_starter template-starter
+    rm -rf test_starter
     echo "===== Finished testing version $VERSION ====="
 }
 
@@ -73,8 +66,8 @@ do
     deactivate
 done
 
-
-# Test the most recent migration with MySQL
+# Test the most recent migration with MariaDB
+echo "===== TESTING CURRENT BRANCH ====="
 set -e
 python3 -m venv ".venv-current-branch"
 source ".venv-current-branch/bin/activate"
@@ -85,7 +78,7 @@ pip3 install importlib_metadata
 
 zenml connect --url mysql://127.0.0.1/zenml --username root --password password
 
-run_tests_for_version current_branch_mysql
+run_tests_for_version current_branch_mariadb
 
 zenml disconnect
 docker rm -f mariadb
diff --git a/scripts/test-migrations-mysql.sh b/scripts/test-migrations-mysql.sh
index 3413cb1cc5f..3fa623daf01 100755
--- a/scripts/test-migrations-mysql.sh
+++ b/scripts/test-migrations-mysql.sh
@@ -16,7 +16,7 @@ function run_tests_for_version() {
 
     echo "===== Testing version $VERSION ====="
 
-    # Check if VERSION is in VALID_VERSIONS
+    # Check if VERSION is in PRE_TEMPLATE_VERSIONS
     if printf '%s\n' "${PRE_TEMPLATE_VERSIONS[@]}" | grep -q "^$VERSION$"; then
         copier copy -l --trust -r release/0.43.0 https://github.com/zenml-io/template-starter.git test_starter
     else
@@ -104,6 +104,7 @@ done
 
 
 # Test the most recent migration with MySQL
+echo "===== TESTING CURRENT BRANCH ====="
 set -e
 python3 -m venv ".venv-current-branch"
 source ".venv-current-branch/bin/activate"

From cd0d85436beef379fe8527f2bf356edb85fc3313 Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Tue, 9 Jan 2024 16:16:26 +0100
Subject: [PATCH 31/35] add test flag to init command

---
 scripts/test-migrations-mariadb.sh | 3 ++-
 scripts/test-migrations-mysql.sh   | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/scripts/test-migrations-mariadb.sh b/scripts/test-migrations-mariadb.sh
index 5a6015b895c..66d3eddb730 100755
--- a/scripts/test-migrations-mariadb.sh
+++ b/scripts/test-migrations-mariadb.sh
@@ -10,7 +10,8 @@ function run_tests_for_version() {
     echo "===== Testing version $VERSION ====="
 
     mkdir test_starter
-    zenml init --template starter --path test_starter --template-with-defaults
+    zenml config analytics opt-out
+    zenml init --template starter --path test_starter --template-with-defaults --test
     cd test_starter
 
     export ZENML_ANALYTICS_OPT_IN=false
diff --git a/scripts/test-migrations-mysql.sh b/scripts/test-migrations-mysql.sh
index 3fa623daf01..5d380e59dd1 100755
--- a/scripts/test-migrations-mysql.sh
+++ b/scripts/test-migrations-mysql.sh
@@ -21,7 +21,8 @@ function run_tests_for_version() {
         copier copy -l --trust -r release/0.43.0 https://github.com/zenml-io/template-starter.git test_starter
     else
         mkdir test_starter
-        zenml init --template starter --path test_starter --template-with-defaults
+        zenml config analytics opt-out
+        zenml init --template starter --path test_starter --template-with-defaults --test
     fi
 
     cd test_starter

From d359e4d0febc7ef4750f6592b1d02cf0dc0acd51 Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Tue, 9 Jan 2024 16:19:14 +0100
Subject: [PATCH 32/35] remove (extra) analytics opt-out

---
 scripts/test-migrations-mariadb.sh | 1 -
 scripts/test-migrations-mysql.sh   | 1 -
 2 files changed, 2 deletions(-)

diff --git a/scripts/test-migrations-mariadb.sh b/scripts/test-migrations-mariadb.sh
index 66d3eddb730..eb5234d7aa1 100755
--- a/scripts/test-migrations-mariadb.sh
+++ b/scripts/test-migrations-mariadb.sh
@@ -10,7 +10,6 @@ function run_tests_for_version() {
     echo "===== Testing version $VERSION ====="
 
     mkdir test_starter
-    zenml config analytics opt-out
     zenml init --template starter --path test_starter --template-with-defaults --test
     cd test_starter
 
diff --git a/scripts/test-migrations-mysql.sh b/scripts/test-migrations-mysql.sh
index 5d380e59dd1..2a38893508f 100755
--- a/scripts/test-migrations-mysql.sh
+++ b/scripts/test-migrations-mysql.sh
@@ -21,7 +21,6 @@ function run_tests_for_version() {
         copier copy -l --trust -r release/0.43.0 https://github.com/zenml-io/template-starter.git test_starter
     else
         mkdir test_starter
-        zenml config analytics opt-out
         zenml init --template starter --path test_starter --template-with-defaults --test
     fi
 

From 00f8b9f931374d8be8351856ab2f2c0941c7e5e7 Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Tue, 9 Jan 2024 16:42:49 +0100
Subject: [PATCH 33/35] Add additional template versions for testing migrations

---
 scripts/test-migrations-mysql.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/test-migrations-mysql.sh b/scripts/test-migrations-mysql.sh
index 2a38893508f..3c32a24f74c 100755
--- a/scripts/test-migrations-mysql.sh
+++ b/scripts/test-migrations-mysql.sh
@@ -12,7 +12,7 @@ fi
 function run_tests_for_version() {
     set -e  # Exit immediately if a command exits with a non-zero status
     local VERSION=$1
-    local PRE_TEMPLATE_VERSIONS=("0.40.0" "0.40.3" "0.41.0")
+    local PRE_TEMPLATE_VERSIONS=("0.40.0" "0.40.3" "0.41.0" "0.43.0" "0.44.1" "0.44.3" "0.45.2" "0.45.3" "0.45.4" "0.45.5" "0.45.6" "0.46.0" "0.47.0")
 
     echo "===== Testing version $VERSION ====="
 

From 1f62ba306b45957939e9e0af2ab97285503c48d2 Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Tue, 9 Jan 2024 17:37:33 +0100
Subject: [PATCH 34/35] enable the CI again

---
 .github/workflows/ci.yml          | 12 ++++-----
 .github/workflows/test_me_now.yml | 43 -------------------------------
 2 files changed, 6 insertions(+), 49 deletions(-)
 delete mode 100644 .github/workflows/test_me_now.yml

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8e606878b90..59f7f4a42dc 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -3,12 +3,12 @@ name: Build, Lint, Unit, Integration and Templates Tests
 on:
   workflow_dispatch:
   workflow_call:
-  # push:
-  #   branches: [main]
-  #   paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md']
-  # pull_request:
-  #   types: [opened, synchronize, ready_for_review]
-  #   paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md']
+  push:
+    branches: [main]
+    paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md']
+  pull_request:
+    types: [opened, synchronize, ready_for_review]
+    paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md']
 concurrency:
   # New commit on branch cancels running workflows of the same branch
   group: ${{ github.workflow }}-${{ github.ref }}
diff --git a/.github/workflows/test_me_now.yml b/.github/workflows/test_me_now.yml
deleted file mode 100644
index c0f2b60b139..00000000000
--- a/.github/workflows/test_me_now.yml
+++ /dev/null
@@ -1,43 +0,0 @@
----
-name: TESTING DB FIXES
-on:
-  push:
-    branches: [main]
-    paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md']
-  pull_request:
-    types: [opened, synchronize, ready_for_review]
-    paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md']
-jobs:
-  mysql-db-migration-testing:
-    runs-on: ubuntu-dind-runners
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4.1.1
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v5.0.0
-        with:
-          python-version: '3.9'
-      - name: Test migrations across versions
-        run: bash scripts/test-migrations-mysql.sh mysql
-  sqlite-db-migration-testing:
-    runs-on: ubuntu-dind-runners
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4.1.1
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v5.0.0
-        with:
-          python-version: '3.9'
-      - name: Test migrations across versions
-        run: bash scripts/test-migrations-mysql.sh sqlite
-  mariadb-db-migration-testing:
-    runs-on: ubuntu-dind-runners
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4.1.1
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v4.8.0
-        with:
-          python-version: '3.9'
-      - name: Test migrations across versions
-        run: bash scripts/test-migrations-mariadb.sh

From 36b8b641b7ee59e2b0d8cc90638f5151369e6fc8 Mon Sep 17 00:00:00 2001
From: Alex Strick van Linschoten <stricksubscriptions@fastmail.fm>
Date: Tue, 9 Jan 2024 17:43:36 +0100
Subject: [PATCH 35/35] Add comment explaining pre-template-versions

---
 scripts/test-migrations-mysql.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/test-migrations-mysql.sh b/scripts/test-migrations-mysql.sh
index 3c32a24f74c..7c57261dc28 100755
--- a/scripts/test-migrations-mysql.sh
+++ b/scripts/test-migrations-mysql.sh
@@ -12,6 +12,8 @@ fi
 function run_tests_for_version() {
     set -e  # Exit immediately if a command exits with a non-zero status
     local VERSION=$1
+    # versions pre-templates and pre-init test flag
+    # (zenml init --test allows for a non-interactive init)
     local PRE_TEMPLATE_VERSIONS=("0.40.0" "0.40.3" "0.41.0" "0.43.0" "0.44.1" "0.44.3" "0.45.2" "0.45.3" "0.45.4" "0.45.5" "0.45.6" "0.46.0" "0.47.0")
 
     echo "===== Testing version $VERSION ====="