self-managed release: Adapt automation to v26.0.0

ci/deploy/docker.py

@@ -13,7 +13,7 @@
 from materialize import ci_util, git, mzbuild, spawn
 from materialize.mz_version import MzVersion
 from materialize.rustc_flags import Sanitizer
-from materialize.version_list import get_all_mz_versions, get_self_managed_versions
+from materialize.version_list import get_all_mz_versions
 from materialize.xcompile import Arch
 
 
@@ -54,19 +54,7 @@ def include_image(image: mzbuild.Image) -> bool:
         for repo in repos
     ]
 
-    if ci_helm_chart_version and ci_mz_version:
-        # On tag builds, always tag the images as such.
-        mzbuild.tag_multiarch_images(
-            f"self-managed-{ci_helm_chart_version}", ci_mz_version, deps
-        )
-
-        version = MzVersion.parse_mz(ci_mz_version)
-        latest_version = max(
-            t for t in get_self_managed_versions() if t.prerelease is None
-        )
-        if version == latest_version:
-            mzbuild.tag_multiarch_images("latest-self-managed", ci_mz_version, deps)
-    elif buildkite_tag:
+    if buildkite_tag:
         # On tag builds, always tag the images as such.
         mzbuild.publish_multiarch_images(buildkite_tag, deps)
 

ci/deploy/pipeline.template.yml

@@ -11,31 +11,50 @@
 priority: 30
 
 steps:
-  - command: bin/ci-builder run nightly ci/deploy/devsite.sh
+  - id: deploy-devsite
+    label: ":rocket: Dev. site"
+    command: bin/ci-builder run nightly ci/deploy/devsite.sh
     branches: main
     timeout_in_minutes: 60
     agents:
       queue: linux-x86_64
     concurrency: 1
     concurrency_group: deploy/devsite
 
-  - command: bin/ci-builder run stable bin/pyactivate -m ci.deploy.docker
+  - id: deploy-docker
+    label: ":rocket: :docker:"
+    command: bin/ci-builder run stable bin/pyactivate -m ci.deploy.docker
     timeout_in_minutes: 30
     agents:
       queue: linux-x86_64-small
     concurrency: 1
     concurrency_group: deploy/linux
 
-  - command: bin/ci-builder run stable bin/pyactivate -m ci.deploy.pypi
+  - id: deploy-pypi
+    label: ":rocket: :pypi:"
+    command: bin/ci-builder run stable bin/pyactivate -m ci.deploy.pypi
     timeout_in_minutes: 30
-    concurrency: 1
-    concurrency_group: deploy/pypi
     agents:
       queue: linux-x86_64-small
+    concurrency: 1
+    concurrency_group: deploy/pypi
 
-  - command: bin/ci-builder run stable bin/pyactivate -m ci.deploy.npm
+  - id: deploy-npm
+    label: ":rocket: :npm:"
+    command: bin/ci-builder run stable bin/pyactivate -m ci.deploy.npm
     timeout_in_minutes: 30
+    agents:
+      queue: linux-x86_64-small
     concurrency: 1
     concurrency_group: deploy/npm
+
+  - id: deploy-helm-chart
+    label: ":rocket: :helm:"
+    command: bin/ci-builder run stable misc/helm-charts/publish.sh
+    depends_on: "deploy-docker"
+    branches: "v*.*"
+    timeout_in_minutes: 30
     agents:
-      queue: linux-x86_64-small
+      queue: linux-aarch64-small
+    concurrency: 1
+    concurrency_group: deploy/helm_chart

ci/mkpipeline.py

@@ -197,6 +197,7 @@ def fetch_hashes() -> None:
         args.sanitizer,
         lto,
     )
+    add_nightly_deploy_dependency(pipeline, args.pipeline)
     remove_dependencies_on_prs(pipeline, args.pipeline, hash_check)
     remove_mz_specific_keys(pipeline)
 
@@ -818,6 +819,29 @@ def visit(step: PipelineStep) -> None:
     ]
 
 
+def add_nightly_deploy_dependency(pipeline: Any, pipeline_name: str) -> None:
+    """In release builds orchestratord and terraform tests in Nightly require
+    the release tag to exist already, so we have to wait for the Deploy step to
+    finish before triggering Nightly. But we don't want to make the Deploy step
+    synchronous in non-release builds since it would make the test run take
+    significantly longer to finish."""
+    tag = os.environ["BUILDKITE_TAG"]
+    if pipeline_name != "test" or not tag:
+        return
+
+    previous_step: dict | None = None
+    for step in steps(pipeline):
+        if step.get("id") == "deploy":
+            step["async"] = False
+            assert previous_step and "wait" in previous_step
+            previous_step["skip"] = True
+
+        if step.get("id") == "nightly-if-release":
+            step["depends_on"] = "deploy"
+
+        previous_step = step
+
+
 def add_cargo_test_dependency(
     pipeline: Any,
     pipeline_name: str,

ci/test/pipeline.template.yml

@@ -767,19 +767,6 @@ steps:
   - wait: ~
     continue_on_failure: true
 
-  - id: coverage-pr-analyze
-    label: Analyze code coverage for PR
-    timeout_in_minutes: 20
-    command: bin/ci-builder run stable ci/test/coverage_report.sh
-    inputs: ["*"]
-    priority: 1
-    agents:
-      queue: hetzner-aarch64-8cpu-16gb
-    coverage: only
-
-  - wait: ~
-    continue_on_failure: true
-
   - id: deploy
     label: Deploy
     trigger: deploy
@@ -792,3 +779,16 @@ steps:
         BUILDKITE_TAG: "$BUILDKITE_TAG"
     coverage: skip
     sanitizer: skip
+
+  - wait: ~
+    continue_on_failure: true
+
+  - id: coverage-pr-analyze
+    label: Analyze code coverage for PR
+    timeout_in_minutes: 20
+    command: bin/ci-builder run stable ci/test/coverage_report.sh
+    inputs: ["*"]
+    priority: 1
+    agents:
+      queue: hetzner-aarch64-8cpu-16gb
+    coverage: only