RFC: Estimated-performance regression tests

.github/workflows/pytest.yml

@@ -0,0 +1,16 @@
+name: Pytest
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+jobs:
+  pytest:
+    name: Pytest
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: ./.github/actions/setup-ubuntu
+    - name: Run pytest
+      run: |
+        python3 -m pytest tests/pytest

pyproject.toml

@@ -54,6 +54,8 @@ dev = [
     "black==26.5.1",
     "pydoclint==0.8.5",
     "flake8==7.3.0",
+    "pytest==8.4.2",
+    "pyyaml==6.0.2",
 ]
 docs = [
     "sphinx==8.2.3; python_version >= '3.11'",
@@ -77,3 +79,7 @@ slothy-cli = "slothy.cli:main"
 [tool.setuptools.packages.find]
 where = ["."]
 include = ["slothy*"]
+
+[tool.pytest.ini_options]
+# Pytest covers tests/pytest; the legacy suite in tests/naive is run by test.py.
+testpaths = ["tests/pytest"]

requirements.txt

@@ -4,6 +4,8 @@ unicorn==2.1.4
 black==26.5.1
 pydoclint==0.8.5
 flake8==7.3.0
+pytest==8.4.2
+pyyaml==6.0.2
 
 # Optional dependencies for documentation (only for Python >= 3.11)
 sphinx==8.2.3 ; python_version >= "3.11"

slothy/core/heuristics.py

@@ -361,11 +361,13 @@ def periodic(body: list, logger: any, conf: any) -> any:
         :param conf: The configuration to be applied.
         :type conf: any
 
-        :return: Tuple (preamble, kernel, postamble, num_exceptional_iterations)
-            of preamble, kernel and postamble (each as a list of SourceLine
-            objects), plus the number of iterations jointly accounted for by
-            the preamble and postamble (the caller will need this to adjust the
-            loop counter).
+        :return: Tuple (preamble, kernel, postamble, num_exceptional_iterations,
+            result) of preamble, kernel and postamble (each as a list of
+            SourceLine objects), the number of iterations jointly accounted for
+            by the preamble and postamble (the caller will need this to adjust
+            the loop counter), and the :class:`Result` object for the (kernel)
+            optimization. The result is ``None`` if no single representative
+            result is available (currently the case for the halving heuristic).
         :rtype: any
         """
 
@@ -386,10 +388,13 @@ def periodic(body: list, logger: any, conf: any) -> any:
         # the heuristics for linear optimization.
         if not conf.sw_pipelining.enabled:
             res = Heuristics.linear(body, logger=logger, conf=conf)
-            return [], res.code, [], 0
+            return [], res.code, [], 0, res
 
         if conf.sw_pipelining.halving_heuristic:
-            return Heuristics._periodic_halving(body, logger, conf)
+            preamble, kernel, postamble, num_exceptional = Heuristics._periodic_halving(
+                body, logger, conf
+            )
+            return preamble, kernel, postamble, num_exceptional, None
 
         # 'Normal' software pipelining
         #
@@ -437,7 +442,7 @@ def periodic(body: list, logger: any, conf: any) -> any:
             )
             postamble = res_postamble.code
 
-        return preamble, kernel, postamble, num_exceptional_iterations
+        return preamble, kernel, postamble, num_exceptional_iterations, result
 
     @staticmethod
     def linear(body: list, logger: any, conf: any) -> any:

slothy/core/slothy.py

@@ -394,7 +394,9 @@ def optimize(
                 pre, body, post, "ORIGINAL", indentation
             )
 
-        early, core, late, num_exceptional = Heuristics.periodic(body, logger, c)
+        early, core, late, num_exceptional, result = Heuristics.periodic(
+            body, logger, c
+        )
 
         if self.config.with_llvm_mca_before is True:
             core = core + orig_stats
@@ -436,6 +438,11 @@ def indented(code):
         self.source = pre + optimized_source + post
         assert SourceLine.is_source(self.source)
 
+        # Expose the optimization result so that callers (e.g. tests) can inspect
+        # SLOTHY's performance estimate, such as result.cycles and result.stalls.
+        self.last_result = result
+        self.success = True
+
     def get_loop_input_output(
         self, loop_lbl: str, forced_loop_type: any = None
     ) -> list:
@@ -632,7 +639,7 @@ def optimize_loop(
                 early, body, late, "ORIGINAL", indentation
             )
 
-        preamble_code, kernel_code, postamble_code, num_exceptional = (
+        preamble_code, kernel_code, postamble_code, num_exceptional, result = (
             Heuristics.periodic(body, logger, c)
         )
 
@@ -746,5 +753,13 @@ def loop_lbl_iter(i):
             DFG(kernel_code, logger.getChild("dfg_kernel_deps"), dfgc).inputs
         )
 
+        # Expose SLOTHY's performance estimate for the loop kernel so that callers
+        # (e.g. tests) can inspect it. These refer to a single kernel iteration.
+        if result is not None:
+            self.last_result.cycles = result.cycles
+            self.last_result.stalls = result.stalls
+            self.last_result.codesize = result.codesize
+            self.last_result.codesize_with_bubbles = result.codesize_with_bubbles
+
         self.source = early + optimized_code + late
         self.success = True

tests/pytest/conftest.py

@@ -0,0 +1,13 @@
+#
+# Copyright (c) SLOTHY contributors
+# SPDX-License-Identifier: MIT
+#
+
+"""Pytest configuration for the SLOTHY test suite."""
+
+import sys
+from pathlib import Path
+
+# Make the slothy package importable when running `pytest` from the repo root
+# without having installed slothy (mirrors how test.py is run from the root).
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))

tests/pytest/estimated_performance/cases/aarch64_ldst.s

@@ -0,0 +1,6 @@
+
+ld3 {v2.8b, v3.8b, v4.8b}, [x0]
+st3 {v2.8b, v3.8b, v4.8b}, [x0], #24
+
+ld4 {v0.4S, v1.4S, v2.4S, v3.4S}, [x0]
+st4 {v0.4S, v1.4S, v2.4S, v3.4S}, [x0], #64

tests/pytest/estimated_performance/cases/aarch64_ldst.yml

@@ -0,0 +1,14 @@
+optimize:
+  call: optimize
+
+config:
+  variable_size: true
+  constraints.stalls_first_attempt: 32
+
+expected:
+  cortex_a55:
+    cycles: 23
+    stalls: 21
+  cortex_a72:
+    cycles: 8
+    stalls: 6

tests/pytest/estimated_performance/cases/aarch64_simple0.s

@@ -0,0 +1,24 @@
+ldr q0, [x1, #0]
+ldr q1, [x2, #0]
+
+ldr q8,[x0]
+ldr q9, [x0, #1*16]
+ldr q10,[x0, #2*16]
+ldr q11,[x0,#3*16]
+
+mul v24.8h, v9.8h, v0.h[0]
+sqrdmulh v9.8h, v9.8h, v0.h[1]
+mls v24.8h, v9.8h, v1.h[0]
+sub v9.8h,v8.8h,v24.8h
+add     v8.8h,    v8.8h, v24.8h
+
+mul v24.8h, v11.8h, v0.h[0]
+sqrdmulh v11.8h, v11.8h, v0.h[1]
+mls v24.8h, v11.8h, v1.h[0]
+sub     v11.8h,    v10.8h, v24.8h
+add     v10.8h,    v10.8h, v24.8h
+
+str q8,  [x0], #4*16
+str q9,  [x0, #-3*16]
+str q10, [x0, #-2*16]
+str q11, [x0, #-1*16]

tests/pytest/estimated_performance/cases/aarch64_simple0.yml

@@ -0,0 +1,11 @@
+optimize:
+  call: optimize
+
+config:
+  variable_size: true
+  constraints.stalls_first_attempt: 32
+
+expected:
+  cortex_a55:
+    cycles: 28
+    stalls: 18

tests/pytest/estimated_performance/test_estimated_performance.py

@@ -0,0 +1,180 @@
+#
+# Copyright (c) SLOTHY contributors
+# SPDX-License-Identifier: MIT
+#
+
+"""Estimated-performance regression tests.
+
+Each case is a pair of files in ``cases``: an assembly snippet ``<name>.s`` and a
+sidecar ``<name>.yml`` describing how to invoke SLOTHY and the cycle/stall
+estimate it is expected to report for the optimized code. These are SLOTHY's own
+estimates (``result.cycles`` / ``result.stalls``), not measured hardware numbers;
+because they are the optimum of a minimization they are deterministic, so pinning
+them turns a change in what SLOTHY believes it found into a test failure.
+
+Run the tests with::
+
+    python3 -m pytest tests/pytest
+
+Print SLOTHY's current estimate for every case (handy when authoring or
+refreshing a sidecar) with::
+
+    PYTHONPATH=. python3 tests/pytest/estimated_performance/test_estimated_performance.py
+"""
+
+import logging
+from pathlib import Path
+
+import pytest
+import yaml
+
+from slothy import Slothy
+
+import slothy.targets.arm_v7m.arch_v7m as Arch_Armv7M
+import slothy.targets.arm_v81m.arch_v81m as Arch_Armv81M
+import slothy.targets.aarch64.aarch64_neon as AArch64_Neon
+import slothy.targets.riscv.riscv as RISCV
+
+import slothy.targets.arm_v7m.cortex_m7 as Target_CortexM7
+import slothy.targets.arm_v81m.cortex_m55r1 as Target_CortexM55r1
+import slothy.targets.arm_v81m.cortex_m85r1 as Target_CortexM85r1
+import slothy.targets.aarch64.cortex_a55 as Target_CortexA55
+import slothy.targets.aarch64.cortex_a72_frontend as Target_CortexA72
+import slothy.targets.aarch64.neoverse_n1_experimental as Target_NeoverseN1
+import slothy.targets.aarch64.aarch64_big_experimental as Target_AArch64Big
+import slothy.targets.aarch64.apple_m1_firestorm_experimental as Target_AppleM1Firestorm
+import slothy.targets.aarch64.apple_m1_icestorm_experimental as Target_AppleM1Icestorm
+import slothy.targets.riscv.xuantie_c908 as Target_XuanTieC908
+
+# Maps the `target` name used in a sidecar to its (architecture, target) modules.
+# The architecture is derived from the target, so sidecars only name the target.
+TARGETS = {
+    "cortex_m7": (Arch_Armv7M, Target_CortexM7),
+    "cortex_m55r1": (Arch_Armv81M, Target_CortexM55r1),
+    "cortex_m85r1": (Arch_Armv81M, Target_CortexM85r1),
+    "cortex_a55": (AArch64_Neon, Target_CortexA55),
+    "cortex_a72": (AArch64_Neon, Target_CortexA72),
+    "neoverse_n1": (AArch64_Neon, Target_NeoverseN1),
+    "aarch64_big": (AArch64_Neon, Target_AArch64Big),
+    "apple_m1_firestorm": (AArch64_Neon, Target_AppleM1Firestorm),
+    "apple_m1_icestorm": (AArch64_Neon, Target_AppleM1Icestorm),
+    "xuantie_c908": (RISCV, Target_XuanTieC908),
+}
+
+CASES_DIR = Path(__file__).resolve().parent / "cases"
+
+
+def _load_case(path):
+    """Load a sidecar YAML file into a dict."""
+    with open(path, "r", encoding="utf8") as f:
+        return yaml.safe_load(f)
+
+
+def _apply_config(config, overrides):
+    """Apply a mapping of (possibly dotted) keys onto a slothy.config object."""
+    for dotted, value in overrides.items():
+        obj = config
+        *parents, leaf = dotted.split(".")
+        for name in parents:
+            obj = getattr(obj, name)
+        setattr(obj, leaf, value)
+
+
+def _run(case_path, target_name):
+    """Run SLOTHY for one (case, target) pair and return slothy.last_result."""
+    if target_name not in TARGETS:
+        raise KeyError(
+            f"Unknown target '{target_name}' in {case_path.name}; "
+            f"known targets: {sorted(TARGETS)}"
+        )
+    arch, target = TARGETS[target_name]
+
+    case = _load_case(case_path)
+    source = case.get("source") or f"{case_path.stem}.s"
+    source_path = case_path.parent / source
+
+    logger = logging.getLogger(f"estperf.{case_path.stem}.{target_name}")
+    logger.setLevel(logging.WARNING)
+
+    slothy = Slothy(arch, target, logger=logger)
+    slothy.load_source_from_file(str(source_path))
+
+    # variable_size lets SLOTHY minimize stalls; a case may override it.
+    cfg = {"variable_size": True}
+    cfg.update(case.get("config") or {})
+    _apply_config(slothy.config, cfg)
+
+    # On Apple M1, x18 is reserved by the platform ABI.
+    if "m1" in target_name:
+        slothy.config.reserved_regs = ["x18"]
+
+    opt = case.get("optimize") or {}
+    call = opt.get("call", "optimize")
+    if call == "optimize":
+        slothy.optimize(start=opt.get("start"), end=opt.get("end"))
+    elif call == "optimize_loop":
+        slothy.optimize_loop(opt.get("loop"))
+    else:
+        raise ValueError(f"Unknown optimize.call '{call}' in {case_path.name}")
+
+    return slothy.last_result
+
+
+def _discover():
+    """Discover (case_path, target_name) pairs from the sidecars in CASES_DIR."""
+    params, ids = [], []
+    for path in sorted(CASES_DIR.glob("*.yml")):
+        case = _load_case(path)
+        for target_name in case.get("expected") or {}:
+            params.append((path, target_name))
+            ids.append(f"{path.stem}-{target_name}")
+    return params, ids
+
+
+_PARAMS, _IDS = _discover()
+
+
+@pytest.mark.parametrize("case_path,target_name", _PARAMS, ids=_IDS)
+def test_estimated_performance(case_path, target_name):
+    """SLOTHY's estimate for the optimized code matches the pinned values."""
+    expected = _load_case(case_path)["expected"][target_name]
+    result = _run(case_path, target_name)
+    assert result is not None, "optimization did not expose a result"
+
+    label = f"{case_path.stem}[{target_name}]"
+
+    # A case may set `match: max` to assert "no worse than" instead of equality,
+    # e.g. for a large case that only solves to a bound rather than the optimum.
+    if expected.get("match") == "max":
+        assert result.cycles <= expected["cycles"], (
+            f"{label}: expected <= {expected['cycles']} cycles, "
+            f"SLOTHY estimated {result.cycles}"
+        )
+        return
+
+    assert result.cycles == expected["cycles"], (
+        f"{label}: expected {expected['cycles']} cycles, "
+        f"SLOTHY estimated {result.cycles}"
+    )
+    if "stalls" in expected:
+        assert result.stalls == expected["stalls"], (
+            f"{label}: expected {expected['stalls']} stalls, "
+            f"SLOTHY estimated {result.stalls}"
+        )
+
+
+def _print_estimates():
+    """Print SLOTHY's current estimate for every (case, target) pair."""
+    params, _ = _discover()
+    for case_path, target_name in params:
+        result = _run(case_path, target_name)
+        bound = getattr(result, "cycles_bound", None)
+        optimal = bound is not None and result.cycles == bound
+        print(
+            f"{case_path.stem}-{target_name}: "
+            f"cycles={result.cycles} stalls={result.stalls} optimal={optimal}"
+        )
+
+
+if __name__ == "__main__":
+    _print_estimates()