Merge pull request #300 from haraldschilly/claude/loop-d5-slice

Let the nightly loop see d5: opt-in --aocc-extra-dims

Author: haraldschilly

.github/workflows/self_improve_nightly.yml

@@ -293,6 +293,20 @@ jobs:
               --sync-eval
               --eps-accept 0.0125
               --inactivity-min-eps-accept 0.006
+              # 2026-08-11 — widen the quick preset's dims=(2,) to
+              # (2, 5).  The loop's two sharpest measured results of
+              # 2026-08 both lived at d5 and were invisible here: the
+              # JSO d5 add (08-02) and the NLSHADE_LBC per-dim split
+              # (08-11, d2 -0.0241 vs d5 +0.0080, both CIs excluding
+              # zero).  Measured cost on this battery: 8.9s -> 11.9s
+              # (1.34x), so the nightly goes ~5.4 -> ~7.2 min against a
+              # 90-minute timeout.
+              #
+              # NOTE this moves the ledger's score LEVEL, not just its
+              # noise: d2 alone reads ~0.369, (d2, d5) reads ~0.309.
+              # Nights before and after are not on one scale — the
+              # per-iteration 'aocc_extra_dims' field records which.
+              --aocc-extra-dims 5
             )
           fi
           "${CMD[@]}"

TODO.md

@@ -2,6 +2,20 @@
 
 ## Recent Improvements (continued)
 
+### Nightly loop can see d5 — 2026-08-11 (third session)
+- [x] **`--aocc-extra-dims 5` in the nightly** — the quick preset's
+      `dims=(2,)` becomes `(2, 5)` for every measurement leg.  Closes
+      the blind spot behind the JSO d5 add (08-02) and the NLSHADE_LBC
+      per-dim split (08-11).  Cost 8.9s → 11.9s (1.34×).
+- [x] **`with_extra_dims` composes, never edits** the frozen presets;
+      name gains a `+d5` suffix.
+- [ ] **The ledger score level shifts** with this ship (d2 ~0.369 →
+      (d2,d5) ~0.309).  `aocc_extra_dims` is recorded per iteration;
+      codify-scan and `summary` should group by it before pooling.
+- [ ] **Re-measure the CMA-ES arm (GOAL §5.2) at d5** with the 12-seed
+      standard instrument — if it splits like NLSHADE_LBC did, one
+      dimension-gating mechanism ships both arms.
+
 ### Loop measurement fidelity: hold-out metric bug, seed rotation, sync-eval — 2026-08-11 (second session)
 - [x] **Hold-out leg measured `composite_score` on AOCC runs** — the
       8.5× "instance-family generalization gap" (`GOAL.md` §2 / §5.1's

panobbgo/harness_ioh.py

@@ -61,7 +61,7 @@
 import hashlib
 import json
 import time
-from dataclasses import asdict, dataclass, field
+from dataclasses import asdict, dataclass, field, replace
 from typing import Any, Dict, List, Optional, Sequence, Tuple
 
 import numpy as np
@@ -279,6 +279,39 @@ def make_quick_battery() -> IOHBatterySpec:
     )
 
 
+def with_extra_dims(battery: IOHBatterySpec, extra_dims: Sequence[int]) -> IOHBatterySpec:
+    """Return ``battery`` widened with ``extra_dims``, preserving everything else.
+
+    The battery presets are frozen contracts (``planning/GOAL.md`` §4:
+    "extend via opt-in flags, never edit"), so a caller that wants a
+    regime the preset cannot reach composes one instead of editing the
+    factory.  Dims already present are ignored and the result is sorted,
+    so the call is idempotent and order-insensitive.
+
+    The name gains a ``+d<k>`` suffix per added dim so a report or
+    ledger record cannot silently conflate a widened battery with the
+    preset it came from — the two measure different things and their
+    mean AOCC is not comparable.
+
+    This exists because the nightly loop runs the quick battery, which
+    is ``dims=(2,)``.  Two of the sharpest measured results of 2026-08
+    (the JSO d5 add on 2026-08-02 and the NLSHADE_LBC per-dim split on
+    2026-08-11, where d2 lost 0.0241 while d5 gained 0.0080) lived
+    entirely at d5 — invisible to the regime the loop actually samples.
+
+    Note the budget interaction: ``budget_for`` is
+    ``budget_multiplier * dim``, so adding dim 5 to the quick battery
+    (multiplier 100) buys 500-eval runs alongside the 200-eval ones.
+    The added dim costs more per run than the ones already there.
+    """
+    merged = tuple(sorted(set(battery.dims) | {int(d) for d in extra_dims}))
+    if merged == battery.dims:
+        return battery
+    added = [d for d in merged if d not in battery.dims]
+    suffix = "".join(f"+d{d}" for d in added)
+    return replace(battery, name=f"{battery.name}{suffix}", dims=merged)
+
+
 def make_standard_battery() -> IOHBatterySpec:
     """Mid-sized battery: a meaningful AOCC estimate without overnight runs."""
     return IOHBatterySpec(

panobbgo/self_improve.py

@@ -3198,6 +3198,30 @@ class LoopConfig:
     #: against a non-sync-eval one** — the noise floors differ, so a
     #: mixed-mode A/B reads the mode change as a spec effect.
     sync_eval: bool = False
+    #: Extra dimensions appended to the AOCC battery for *every*
+    #: measurement the loop makes — screening, confirm, guard, hold-out.
+    #:
+    #: The mode presets are frozen contracts, so this composes a widened
+    #: battery via :func:`~panobbgo.harness_ioh.with_extra_dims` instead
+    #: of editing them.  ``(5,)`` on the quick preset turns the nightly's
+    #: ``dims=(2,)`` regime into ``dims=(2, 5)``.
+    #:
+    #: Motivation: the loop samples quick-2-D, but the two sharpest
+    #: measured results of 2026-08 both lived at d5 — the JSO d5 add
+    #: (2026-08-02) and the NLSHADE_LBC per-dim split (2026-08-11, d2
+    #: −0.0241 against d5 +0.0080, both CIs excluding zero).  A regime
+    #: the loop cannot see is a regime it cannot optimise, and worse, a
+    #: change that helps there reads as noise or as a loss in the
+    #: aggregate.
+    #:
+    #: Cost scales super-linearly: ``budget_for`` is
+    #: ``budget_multiplier * dim``, so adding dim 5 to the quick battery
+    #: doubles the run count *and* the added runs are 2.5x longer.
+    #:
+    #: Empty (default) leaves every existing invocation byte-identical.
+    #: Inert under ``metric="composite"``, which reaches higher dims
+    #: through ``--extra-highdim`` / :attr:`extra_families` instead.
+    aocc_extra_dims: Tuple[int, ...] = ()
 
     def __post_init__(self) -> None:
         if self.iterations < 0:
@@ -3475,6 +3499,17 @@ class LoopIterationRecord:
     #: pooled CI narrower than either mode justifies.  Consumers that
     #: aggregate across nights should group by this field.
     sync_eval: bool = False
+    #: Dimensions appended to the mode's AOCC battery for this iteration
+    #: (:attr:`LoopConfig.aocc_extra_dims`).  Empty on legacy records and
+    #: on composite runs.
+    #:
+    #: Recorded for the same reason as :attr:`sync_eval`: widening the
+    #: battery moves the *level* of the score, not just its noise —
+    #: measured on the quick preset, d2 alone reads 0.3685 while
+    #: (d2, d5) reads ~0.31 — so a night before the widening and a night
+    #: after are not on one scale.  Cross-night consumers must group by
+    #: this field as well.
+    aocc_extra_dims: Tuple[int, ...] = ()
 
     def to_dict(self) -> Dict[str, Any]:
         d: Dict[str, Any] = {
@@ -3502,6 +3537,7 @@ def to_dict(self) -> Dict[str, Any]:
             "bandit_reward": self.bandit_reward,
             "confirmed": self.confirmed,
             "sync_eval": self.sync_eval,
+            "aocc_extra_dims": list(self.aocc_extra_dims),
         }
         return d
 
@@ -4502,6 +4538,7 @@ def _run_internal(
                 bandit_reward=bandit_reward,
                 confirmed=confirmed_flag,
                 sync_eval=bool(self.config.sync_eval),
+                aocc_extra_dims=tuple(self.config.aocc_extra_dims),
             )
             records.append(rec)
             ledger.write(rec)
@@ -4665,6 +4702,7 @@ def _measure_aocc(
             make_quick_battery,
             make_standard_battery,
             run_ioh_harness,
+            with_extra_dims,
         )
 
         battery_factories = {
@@ -4673,6 +4711,12 @@ def _measure_aocc(
             "full": make_full_battery,
         }
         battery = battery_factories[self.config.mode]()
+        if self.config.aocc_extra_dims:
+            # Widen *every* measurement identically — screening,
+            # confirm, guard and hold-out all route through here, so
+            # the loop can never compare a 2-D baseline against a
+            # (2, 5)-D candidate.
+            battery = with_extra_dims(battery, self.config.aocc_extra_dims)
         if verbose:
             print(f"[self_improve] iter={iteration} measuring {label} (AOCC, battery={battery.name})")
         # Mix the iteration into the base seed so each iteration draws
@@ -4718,6 +4762,7 @@ def _skip_record(
             effective_eps_accept=effective_eps_accept,
             iters_since_accept=iters_since_accept,
             sync_eval=bool(self.config.sync_eval),
+            aocc_extra_dims=tuple(self.config.aocc_extra_dims),
         )
 
     def _stop_requested(self) -> bool:

planning/SELF_IMPROVEMENT_LOG.md

@@ -17,6 +17,54 @@ Conventions:
 * Graduate items from "Next iteration ideas" to a dated entry when
   shipped.
 
+### 2026-08-11 (third session) — the nightly loop can finally see d5
+
+* **What** — an opt-in `--aocc-extra-dims` widens the AOCC battery for
+  *every* measurement the loop makes (screening, confirm, guard,
+  hold-out), and the nightly turns it on with `5`.  The quick preset's
+  `dims=(2,)` becomes `(2, 5)`.  Mode presets stay frozen per GOAL §4 —
+  the widened battery is *composed* by
+  :func:`~panobbgo.harness_ioh.with_extra_dims`, which appends, sorts,
+  and suffixes the name (`ioh-quick+d5`) so a report can never conflate
+  it with the preset.
+
+* **Why** — the loop has been optimising a regime that hides its own
+  best results.  Two measured cases, both outside the nightly battery:
+
+  | date | change | where the effect lived |
+  |---|---|---|
+  | 2026-08-02 | JSO `add_heuristic` | d5 |
+  | 2026-08-11 | NLSHADE_LBC `add_heuristic` | d2 −0.0241 [−0.0401,−0.0080] **vs** d5 +0.0080 [+0.0007,+0.0154] |
+
+  The second is the sharper lesson: the two dims moved in *opposite*
+  directions with both CIs excluding zero.  A 2-D-only loop reads that
+  change as a loss and rejects it; a (2, 5) loop at least sees the
+  conflict.  GOAL §4's own cadence guardrail — "if the quick-battery
+  score stalls for >1 week with evidence banked, the bottleneck is the
+  *measurement regime*" — has been triggered for five weeks.
+
+* **Cost** — measured on the quick battery, 2 specs, sync-eval on:
+  **8.9 s → 11.9 s (1.34×)**, 6 runs → 12.  Much less than the 2.5×
+  the budget rule (`budget_multiplier * dim`) suggests, because run
+  time here is not budget-bound.  The nightly goes ~5.4 → ~7.2 min
+  against a 90-minute timeout.
+
+* **Scale discontinuity, deliberate and recorded** — widening moves the
+  *level*, not just the noise: on this battery d2 alone reads 0.3685
+  while (d2, d5) reads ~0.309, because d5 is simply harder.  Nights
+  before and after the switch are therefore **not on one scale**.  The
+  per-iteration `aocc_extra_dims` field records which battery produced
+  each record, exactly as `sync_eval` records which evaluation mode did;
+  any cross-night consumer has to group by both.  Recording this is the
+  same discipline whose absence produced the hold-out metric-mismatch
+  bug fixed earlier today.
+
+* **Validation** — 1967 passed, 1 skipped; `ruff format --check` clean;
+  pyright 0 errors.  9 new tests, including that every measurement leg
+  widens identically (a 2-D baseline against a (2,5)-D candidate would
+  be a silent catastrophe) and that re-widening does not stack name
+  suffixes.
+
 ### 2026-08-11 (second session) — measurement fidelity: hold-out metric-mismatch bug fixed, confirm gate crosses a base seed, nightly seed rotation, `--sync-eval`, eps recalibration
 
 * **What** — a review of the full 34-night AOCC ledger (952 records,

scripts/self_improve.py

@@ -308,6 +308,26 @@ def _build_parser() -> argparse.ArgumentParser:
         ),
     )
     run_p.set_defaults(extra_highdim=False)
+    run_p.add_argument(
+        "--aocc-extra-dims",
+        dest="aocc_extra_dims",
+        default="",
+        metavar="D[,D...]",
+        help=(
+            "Append these dimensions to the AOCC battery for every "
+            "measurement (screening, confirm, guard, hold-out).  "
+            "'--aocc-extra-dims 5' turns the nightly's quick dims=(2,) "
+            "regime into dims=(2, 5).  The mode presets stay frozen; "
+            "the widened battery is composed, not edited, and its name "
+            "gains a '+d5' suffix so it cannot be confused with the "
+            "preset.  Motivation: the two sharpest measured results of "
+            "2026-08 (JSO d5 add, NLSHADE_LBC per-dim split) both lived "
+            "at d5, invisible to a 2-D-only loop.  Costs more than "
+            "linearly — budget is budget_multiplier*dim, so d5 runs are "
+            "2.5x longer than d2 ones.  AOCC path only; the composite "
+            "path reaches higher dims via --extra-highdim."
+        ),
+    )
     run_p.add_argument(
         "--timeout",
         type=float,
@@ -1181,13 +1201,15 @@ def _run_subprocess(cmd: Sequence[str]) -> "subprocess.CompletedProcess[Any]":
     return subprocess.run(list(cmd), check=False)
 
 
-def _parse_seed_list(raw: str) -> tuple:
-    """Parse a comma-separated seed list (e.g. ``"1234,5678,9012"``).
+def _parse_seed_list(raw: str, flag: str = "--holdout-base-seeds") -> tuple:
+    """Parse a comma-separated integer list (e.g. ``"1234,5678,9012"``).
 
     Empty / blank → empty tuple.  Whitespace around entries is tolerated
     so command-line callers can write ``"1234, 5678"`` without quoting
     surprises.  Non-integer entries raise ``ValueError`` with the
-    offending token for ergonomic error messages.
+    offending token for ergonomic error messages; ``flag`` names the
+    option in that message so the parser can be shared by more than one
+    comma-separated-int option.
     """
     s = (raw or "").strip()
     if not s:
@@ -1200,7 +1222,7 @@ def _parse_seed_list(raw: str) -> tuple:
         try:
             out.append(int(token))
         except ValueError as e:
-            raise ValueError(f"--holdout-base-seeds: invalid integer {token!r}") from e
+            raise ValueError(f"{flag}: invalid integer {token!r}") from e
     return tuple(out)
 
 
@@ -1220,6 +1242,17 @@ def _cmd_run(args: argparse.Namespace) -> int:
     except ValueError as e:
         print(f"Error: {e}", file=sys.stderr)
         return 1
+    try:
+        aocc_extra_dims = _parse_seed_list(getattr(args, "aocc_extra_dims", ""), flag="--aocc-extra-dims")
+    except ValueError as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 1
+    if any(d < 1 for d in aocc_extra_dims):
+        print(
+            f"Error: --aocc-extra-dims must be positive, got {list(aocc_extra_dims)}",
+            file=sys.stderr,
+        )
+        return 1
     try:
         extra_families = None
         if getattr(args, "extra_highdim", False):
@@ -1271,6 +1304,7 @@ def _cmd_run(args: argparse.Namespace) -> int:
             confirm_accepts=args.confirm_accepts,
             confirm_iteration_offset=args.confirm_iteration_offset,
             sync_eval=args.sync_eval,
+            aocc_extra_dims=aocc_extra_dims,
         )
     except ValueError as e:
         print(f"Error: {e}", file=sys.stderr)