orchestratord test: Fix upgrade tests and reenable authenticator kinds

ci/deploy/pipeline.template.yml

@@ -39,23 +39,3 @@ steps:
     concurrency_group: deploy/npm
     agents:
       queue: linux-x86_64-small
-
-  - label: ":bulb: Full SQL Logic Tests"
-    trigger: slt
-    async: true
-    branches: "v*.*rc*"
-    build:
-      commit: "$BUILDKITE_COMMIT"
-      branch: "$BUILDKITE_BRANCH"
-      env:
-        BUILDKITE_TAG: "$BUILDKITE_TAG"
-
-  - label: ":nightmare: Full Nightly"
-    trigger: nightly
-    async: true
-    branches: "v*.*rc*"
-    build:
-      commit: "$BUILDKITE_COMMIT"
-      branch: "$BUILDKITE_BRANCH"
-      env:
-        BUILDKITE_TAG: "$BUILDKITE_TAG"

ci/test/build.py

@@ -11,7 +11,6 @@
 
 import os
 import shutil
-import subprocess
 import sys
 from pathlib import Path
 
@@ -20,7 +19,7 @@
     DEBUGINFO_BINS,
     upload_debuginfo_to_s3,
 )
-from materialize.mzbuild import CargoBuild, Repository, ResolvedImage
+from materialize.mzbuild import CargoBuild, Repository, ResolvedImage, RustICE
 from materialize.rustc_flags import Sanitizer
 from materialize.xcompile import Arch
 
@@ -45,18 +44,17 @@ def main() -> None:
         deps.ensure(pre_build=lambda images: upload_debuginfo(repo, images))
         set_build_status("success")
         annotate_buildkite_with_tags(repo.rd.arch, deps)
-    except subprocess.CalledProcessError as e:
-        set_build_status("failed")
-        if e.returncode == 101:  # panic in Rust
-            print(
-                "--- Detected Rust ICE (https://github.com/rust-lang/rust/issues/148581), clearing cargo target directories"
-            )
-            for dir in ["target", "target-xcompile"]:
-                if os.path.exists(dir):
-                    shutil.rmtree(dir, ignore_errors=True)
+    except RustICE:
+        # We retry twice automatically, see mkpipeline.py
+        if int(os.getenv("BUILDKITE_RETRY_COUNT", "0")) >= 2:
             set_build_status("failed")
-            sys.exit(199)
-        raise
+        print(
+            "--- Detected Rust ICE (https://github.com/rust-lang/rust/issues/148581), clearing cargo target directories"
+        )
+        for dir in ["target", "target-xcompile"]:
+            if os.path.exists(dir):
+                shutil.rmtree(dir, ignore_errors=True)
+        sys.exit(199)
     except:
         set_build_status("failed")
         raise

doc/user/content/releases/v26.0.md

@@ -2,6 +2,7 @@
 title: "Materialize v26.0"
 date: 2025-11-19
 released: false
+rc: 1
 _build:
   render: never
 ---

misc/python/materialize/mz_version.py

@@ -86,7 +86,7 @@ def __str__(self) -> str:
         return f"{self.get_prefix()}{self.str_without_prefix()}"
 
     def is_dev_version(self) -> bool:
-        return self.prerelease is not None
+        return self.prerelease == "dev"
 
 
 class MzVersion(TypedVersionBase):

misc/python/materialize/mzbuild.py

@@ -24,6 +24,7 @@
 import os
 import platform
 import re
+import selectors
 import shutil
 import stat
 import subprocess
@@ -48,6 +49,78 @@
 from materialize.xcompile import Arch, target
 
 
+class RustICE(Exception):
+    pass
+
+
+def run_and_detect_rust_ice(
+    cmd: list[str], cwd: str | Path
+) -> subprocess.CompletedProcess:
+    """This function is complex since it prints out each line immediately to
+    stdout/stderr, but still records them at the same time so that we can scan
+    for the Rust ICE."""
+    stdout_result = ""
+    stderr_result = ""
+    p = subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+        bufsize=1,
+    )
+
+    sel = selectors.DefaultSelector()
+    sel.register(p.stdout, selectors.EVENT_READ)  # type: ignore
+    sel.register(p.stderr, selectors.EVENT_READ)  # type: ignore
+    assert p.stdout is not None
+    assert p.stderr is not None
+    os.set_blocking(p.stdout.fileno(), False)
+    os.set_blocking(p.stderr.fileno(), False)
+    while True:
+        for key, val in sel.select():
+            output = ""
+            while True:
+                new_output = key.fileobj.read(1024)  # type: ignore
+                if not new_output:
+                    break
+                output += new_output
+            if not output:
+                continue
+            if key.fileobj is p.stdout:
+                print(
+                    output,
+                    end="",
+                    flush=True,
+                )
+                stdout_result += output
+            else:
+                print(
+                    output,
+                    end="",
+                    file=sys.stderr,
+                    flush=True,
+                )
+                stderr_result += output
+        p.wait()
+        retcode = p.poll()
+        assert retcode is not None
+        if retcode:
+            panic_msg = (
+                "panicked at compiler/rustc_metadata/src/rmeta/def_path_hash_map.rs"
+            )
+            if panic_msg in stdout_result or panic_msg in stderr_result:
+                raise RustICE()
+
+            raise subprocess.CalledProcessError(
+                retcode, p.args, output=stdout_result, stderr=stderr_result
+            )
+        return subprocess.CompletedProcess(
+            p.args, retcode, stdout_result, stderr_result
+        )
+
+    assert False, "unreachable"
+
+
 class Fingerprint(bytes):
     """A SHA-1 hash of the inputs to an `Image`.
 
@@ -470,7 +543,7 @@ def prepare_batch(cls, cargo_builds: list["PreImage"]) -> dict[str, Any]:
 
         cargo_build = cls.generate_cargo_build_command(rd, list(bins), list(examples))
 
-        spawn.runv(cargo_build, cwd=rd.root)
+        run_and_detect_rust_ice(cargo_build, cwd=rd.root)
 
         # Re-run with JSON-formatted messages and capture the output so we can
         # later analyze the build artifacts in `run`. This should be nearly

misc/python/materialize/release/auto_cut_release.py

@@ -66,6 +66,7 @@ def main():
 title: "Materialize {args.next_version}"
 date: {args.next_date}
 released: false
+rc: 1
 _build:
   render: never
 ---

misc/python/materialize/version_list.py

@@ -587,13 +587,22 @@ def __init__(self, respect_released_tag: bool) -> None:
                 continue
 
             current_patch = metadata.get("patch", 0)
-
-            for patch in range(current_patch + 1):
-                version = MzVersion.parse_mz(f"{base}.{patch}")
-                if not respect_released_tag and version >= current_version:
-                    continue
-                if version not in INVALID_VERSIONS:
-                    self.versions.append(version)
+            current_rc = metadata.get("rc", 0)
+
+            if current_rc > 0:
+                for rc in range(current_rc + 1):
+                    version = MzVersion.parse_mz(f"{base}.{current_patch}-rc.{rc}")
+                    if not respect_released_tag and version >= current_version:
+                        continue
+                    if version not in INVALID_VERSIONS:
+                        self.versions.append(version)
+            else:
+                for patch in range(current_patch + 1):
+                    version = MzVersion.parse_mz(f"{base}.{patch}")
+                    if not respect_released_tag and version >= current_version:
+                        continue
+                    if version not in INVALID_VERSIONS:
+                        self.versions.append(version)
 
         assert len(self.versions) > 0
         self.versions.sort()