ci(gitlab): use template in dynamic configuration

We make the dynamic configuration generation script leaner by taking out
literal tempate strings into template files.

Author: P403n1x87

.gitlab/templates/build-base-venvs.yml

@@ -0,0 +1,35 @@
+build_base_venvs:
+  extends: .cached_testrunner
+  stage: setup
+  needs: []
+  parallel:
+    matrix:
+      - PYTHON_VERSION: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
+  variables:
+    CMAKE_BUILD_PARALLEL_LEVEL: '12'
+    PIP_VERBOSE: '0'
+    DD_PROFILING_NATIVE_TESTS: '1'
+    DD_USE_SCCACHE: '1'
+    DD_FAST_BUILD: '1'
+  rules:
+    - if: '$CI_COMMIT_REF_NAME == "main"'
+      variables:
+        DD_FAST_BUILD: '0'
+    - when: always
+  script: |
+    set -e -o pipefail
+    apt update && apt install -y sccache
+    pip install riot==0.20.1
+    riot -P -v generate --python=$PYTHON_VERSION
+    echo "Running smoke tests"
+    riot -v run -s --python=$PYTHON_VERSION smoke_test
+  artifacts:
+    name: venv_$PYTHON_VERSION
+    paths:
+      - scripts/restore-ext-cache.sh
+      - scripts/save-ext-cache.sh
+      - .riot/venv_*
+      - ddtrace/_version.py
+      - ddtrace/**/*.so*
+      - ddtrace/internal/datadog/profiling/crashtracker/crashtracker_exe*
+      - ddtrace/internal/datadog/profiling/test/test_*

.gitlab/templates/cached-testrunner.yml

@@ -0,0 +1,37 @@
+.cached_testrunner:
+  extends: .testrunner
+  variables:
+    PIP_CACHE_DIR: '${{CI_PROJECT_DIR}}/.cache/pip'
+    SCCACHE_DIR: '${{CI_PROJECT_DIR}}/.cache/sccache'
+    DD_CMAKE_INCREMENTAL_BUILD: '1'
+    DD_SETUP_CACHE_DOWNLOADS: '1'
+    EXT_CACHE_VENV: '${{CI_PROJECT_DIR}}/.cache/ext_cache_venv'
+  before_script: |
+    set -e -o pipefail
+    if [ ! -d $EXT_CACHE_VENV ]; then
+        python$PYTHON_VERSION -m venv $EXT_CACHE_VENV
+        source $EXT_CACHE_VENV/bin/activate
+        pip install cmake setuptools_rust Cython
+    else
+        source $EXT_CACHE_VENV/bin/activate
+    fi
+    python scripts/gen_ext_cache_scripts.py
+    deactivate
+    $SHELL scripts/restore-ext-cache.sh
+  after_script: |
+    set -e -o pipefail
+    source $EXT_CACHE_VENV/bin/activate
+    python scripts/gen_ext_cache_scripts.py
+    deactivate
+    $SHELL scripts/save-ext-cache.sh
+  cache:
+    # Share pip/sccache between jobs of the same Python version
+    - key: v1-build_base_venvs-${{PYTHON_VERSION}}-cache-{current_month}
+      paths:
+        - .cache
+    - key: v1-build_base_venvs-${{PYTHON_VERSION}}-ext-{current_month}
+      paths:
+        - .ext_cache
+    - key: v1-build_base_venvs-${{PYTHON_VERSION}}-download-cache-{current_month}
+      paths:
+        - .download_cache

scripts/gen_gitlab_config.py

@@ -245,85 +245,24 @@ def check(name: str, command: str, paths: t.Set[str]) -> None:
     key: v1-precheck-pip-cache
     paths:
       - .cache
-        """
+"""
         )
 
 
-def gen_build_base_venvs() -> None:
-    """Generate the list of base jobs for building virtual environments."""
+def gen_cached_testrunner() -> None:
+    """Generate the cached testrunner job."""
+    with TESTS_GEN.open("a") as f:
+        f.write(template("cached-testrunner", current_month=datetime.datetime.now().month))
 
-    current_month = datetime.datetime.now().month
 
+def gen_build_base_venvs() -> None:
+    """Generate the list of base jobs for building virtual environments.
+
+    We need to generate this dynamically from a template because it depends
+    on the cached testrunner job, which is also generated dynamically.
+    """
     with TESTS_GEN.open("a") as f:
-        f.write(
-            f"""
-build_base_venvs:
-  extends: .testrunner
-  stage: setup
-  needs: []
-  parallel:
-    matrix:
-      - PYTHON_VERSION: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
-  variables:
-    CMAKE_BUILD_PARALLEL_LEVEL: '12'
-    PIP_VERBOSE: '0'
-    DD_PROFILING_NATIVE_TESTS: '1'
-    DD_USE_SCCACHE: '1'
-    PIP_CACHE_DIR: '${{CI_PROJECT_DIR}}/.cache/pip'
-    SCCACHE_DIR: '${{CI_PROJECT_DIR}}/.cache/sccache'
-    DD_FAST_BUILD: '1'
-    DD_CMAKE_INCREMENTAL_BUILD: '1'
-    DD_SETUP_CACHE_DOWNLOADS: '1'
-    EXT_CACHE_VENV: '${{CI_PROJECT_DIR}}/.cache/ext_cache_venv'
-  rules:
-    - if: '$CI_COMMIT_REF_NAME == "main"'
-      variables:
-        DD_FAST_BUILD: '0'
-    - when: always
-  script: |
-    set -e -o pipefail
-    apt update && apt install -y sccache
-    pip install riot==0.20.1
-    if [ ! -d $EXT_CACHE_VENV ]; then
-        python$PYTHON_VERSION -m venv $EXT_CACHE_VENV
-        source $EXT_CACHE_VENV/bin/activate
-        pip install cmake setuptools_rust Cython
-    else
-        source $EXT_CACHE_VENV/bin/activate
-    fi
-    python scripts/gen_ext_cache_scripts.py
-    deactivate
-    $SHELL scripts/restore-ext-cache.sh
-    riot -P -v generate --python=$PYTHON_VERSION
-    echo "Running smoke tests"
-    riot -v run -s --python=$PYTHON_VERSION smoke_test
-    source $EXT_CACHE_VENV/bin/activate
-    python scripts/gen_ext_cache_scripts.py
-    deactivate
-    $SHELL scripts/save-ext-cache.sh
-  cache:
-    # Share pip/sccache between jobs of the same Python version
-    - key: v1-build_base_venvs-${{PYTHON_VERSION}}-cache-{current_month}
-      paths:
-        - .cache
-    - key: v1-build_base_venvs-${{PYTHON_VERSION}}-ext-{current_month}
-      paths:
-        - .ext_cache
-    - key: v1-build_base_venvs-${{PYTHON_VERSION}}-download-cache-{current_month}
-      paths:
-        - .download_cache
-  artifacts:
-    name: venv_$PYTHON_VERSION
-    paths:
-      - scripts/restore-ext-cache.sh
-      - scripts/save-ext-cache.sh
-      - .riot/venv_*
-      - ddtrace/_version.py
-      - ddtrace/**/*.so*
-      - ddtrace/internal/datadog/profiling/crashtracker/crashtracker_exe*
-      - ddtrace/internal/datadog/profiling/test/test_*
-        """
-        )
+        f.write(template("build-base-venvs"))
 
 
 # -----------------------------------------------------------------------------
@@ -358,6 +297,14 @@ def gen_build_base_venvs() -> None:
 sys.path.append(str(ROOT / "scripts"))
 sys.path.append(str(ROOT / "tests"))
 
+
+def template(name: str, **params):
+    """Render a template file with the given parameters."""
+    if not (template_path := (GITLAB / "templates" / name).with_suffix(".yml")).exists():
+        raise FileNotFoundError(f"Template file {template_path} does not exist")
+    return "\n" + template_path.read_text().format(**params).strip() + "\n"
+
+
 has_error = False
 
 LOGGER.info("Configuration generation steps:")