Fix the bench job's venv discovery and separate "no data" from "regression"

All six bench cells failed at "Create both environments" while every build
step, Windows included, succeeded.  The cause was

    PY=$(ls venv-$v/bin/python venv-$v/Scripts/python.exe 2>/dev/null | head -1)

GitHub runs `shell: bash` as `bash --noprofile --norc -eo pipefail`, so `set -e`
is active, and an assignment takes the exit status of its command substitution.
One of those two paths is always absent -- that is the whole point of listing
both -- so ls exits 2 and kills the step.  Replaced with an explicit -x test at
both call sites.

Also stop conflating outcomes in compare.py.  It exited 1 both when a benchmark
regressed and when no artifacts existed at all, so this run reported a
performance regression when what actually happened was that no benchmark ever
ran.  It now always exits 0 and publishes status=ok|regressed|nodata through
GITHUB_OUTPUT, with the workflow failing on each for its own reason and its own
message.  The report file is written unconditionally so the PR-comment step has
something to post either way.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Author: FrancescAlted

.github/bench-abi3/compare.py

@@ -28,6 +28,28 @@
 NOISE_FLOOR_MS = 5.0
 
 
+def emit(report, *, status, failures):
+    """Publish the report everywhere, and report `status` rather than exiting.
+
+    "no data" and "a real regression" are different outcomes and the workflow
+    needs to tell them apart, so this always exits 0 and hands the verdict to
+    the caller through GITHUB_OUTPUT.
+    """
+    print(report)
+    summary = os.environ.get("GITHUB_STEP_SUMMARY")
+    if summary:
+        with open(summary, "a") as fh:
+            fh.write(report + "\n")
+    out = os.environ.get("GITHUB_OUTPUT")
+    if out:
+        with open(out, "a") as fh:
+            fh.write(f"status={status}\n")
+            fh.write(f"failures={failures}\n")
+    # Always written, so the PR-comment step has a file even when there is no data.
+    with open("abi3-bench-report.md", "w") as fh:
+        fh.write(report + "\n")
+
+
 def load_cell(cell_dir):
     out = {}
     for build in ("base", "abi3"):
@@ -47,8 +69,10 @@ def best(runs, name):
 def main(root):
     cells = sorted(d for d in glob.glob(os.path.join(root, "bench-*")) if os.path.isdir(d))
     if not cells:
-        print(f"no result directories under {root}", file=sys.stderr)
-        return 1
+        msg = f"No benchmark results were produced under `{root}` (the bench jobs did not upload artifacts)."
+        print(msg, file=sys.stderr)
+        emit(msg + "\n", status="nodata", failures=0)
+        return 0
 
     lines = ["## abi3 vs. version-specific build\n"]
     lines.append(
@@ -107,22 +131,12 @@ def main(root):
     else:
         lines.append("\nNo regression past threshold on any platform. ✅\n")
 
-    report = "\n".join(lines)
-    print(report)
-
-    summary = os.environ.get("GITHUB_STEP_SUMMARY")
-    if summary:
-        with open(summary, "a") as fh:
-            fh.write(report + "\n")
-    out = os.environ.get("GITHUB_OUTPUT")
-    if out:
-        with open(out, "a") as fh:
-            fh.write(f"failures={len(failures)}\n")
-
-    with open("abi3-bench-report.md", "w") as fh:
-        fh.write(report + "\n")
-
-    return 1 if failures else 0
+    emit(
+        "\n".join(lines),
+        status="regressed" if failures else "ok",
+        failures=len(failures),
+    )
+    return 0
 
 
 if __name__ == "__main__":

.github/workflows/abi3-bench.yml

@@ -80,7 +80,14 @@ jobs:
         run: |
           for v in abi3 base; do
             python -m venv "venv-$v"
-            PY=$(ls venv-$v/bin/python venv-$v/Scripts/python.exe 2>/dev/null | head -1)
+            # Not `ls a b | head -1`: GitHub runs bash with `set -eo pipefail`,
+            # and ls exits non-zero when one of the two paths is absent,
+            # which is always the case -- it is one layout or the other.
+            if [ -x "venv-$v/bin/python" ]; then
+              PY="venv-$v/bin/python"
+            else
+              PY="venv-$v/Scripts/python.exe"
+            fi
             "$PY" -m pip install --quiet --upgrade pip
             "$PY" -m pip install --quiet wheels-$v/*.whl
             echo "-- $v --"
@@ -100,7 +107,11 @@ jobs:
           OUT=results/bench-${{ matrix.os }}-${{ matrix.python-version }}
           for round in 1 2 3; do
             for v in abi3 base; do
-              PY=$(ls venv-$v/bin/python venv-$v/Scripts/python.exe 2>/dev/null | head -1)
+              if [ -x "venv-$v/bin/python" ]; then
+                PY="venv-$v/bin/python"
+              else
+                PY="venv-$v/Scripts/python.exe"
+              fi
               echo "== round $round / $v =="
               "$PY" .github/bench-abi3/bench.py > "$OUT/$v-$round.json"
             done
@@ -133,7 +144,6 @@ jobs:
 
       - name: Build the report
         id: report
-        continue-on-error: true
         run: python .github/bench-abi3/compare.py results
 
       # The step summary is not reliably readable through the public API, so
@@ -152,7 +162,13 @@ jobs:
           fi
 
       - name: Fail if a benchmark regressed past threshold
-        if: steps.report.outcome == 'failure'
+        if: steps.report.outputs.status == 'regressed'
+        run: |
+          echo "${{ steps.report.outputs.failures }} benchmark(s) past threshold; see the report above."
+          exit 1
+
+      - name: Fail if no benchmark data was produced
+        if: steps.report.outputs.status == 'nodata'
         run: |
-          echo "See the report above: at least one benchmark regressed past threshold."
+          echo "The bench jobs uploaded no artifacts -- this is a workflow failure, not a regression."
           exit 1