diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml new file mode 100644 index 000000000..d2050c0a3 --- /dev/null +++ b/.github/workflows/pytest.yml @@ -0,0 +1,16 @@ +name: Pytest +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] +jobs: + pytest: + name: Pytest + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/setup-ubuntu + - name: Run pytest + run: | + python3 -m pytest tests/pytest diff --git a/pyproject.toml b/pyproject.toml index f2c6552bd..cfbc646f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,8 @@ dev = [ "black==26.5.1", "pydoclint==0.8.5", "flake8==7.3.0", + "pytest==8.4.2", + "pyyaml==6.0.2", ] docs = [ "sphinx==8.2.3; python_version >= '3.11'", @@ -77,3 +79,7 @@ slothy-cli = "slothy.cli:main" [tool.setuptools.packages.find] where = ["."] include = ["slothy*"] + +[tool.pytest.ini_options] +# Pytest covers tests/pytest; the legacy suite in tests/naive is run by test.py. +testpaths = ["tests/pytest"] diff --git a/requirements.txt b/requirements.txt index a611d25d9..529ecdd6f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,8 @@ unicorn==2.1.4 black==26.5.1 pydoclint==0.8.5 flake8==7.3.0 +pytest==8.4.2 +pyyaml==6.0.2 # Optional dependencies for documentation (only for Python >= 3.11) sphinx==8.2.3 ; python_version >= "3.11" diff --git a/slothy/core/heuristics.py b/slothy/core/heuristics.py index 186dd22a1..44d8d5557 100644 --- a/slothy/core/heuristics.py +++ b/slothy/core/heuristics.py @@ -361,11 +361,13 @@ def periodic(body: list, logger: any, conf: any) -> any: :param conf: The configuration to be applied. :type conf: any - :return: Tuple (preamble, kernel, postamble, num_exceptional_iterations) - of preamble, kernel and postamble (each as a list of SourceLine - objects), plus the number of iterations jointly accounted for by - the preamble and postamble (the caller will need this to adjust the - loop counter). + :return: Tuple (preamble, kernel, postamble, num_exceptional_iterations, + result) of preamble, kernel and postamble (each as a list of + SourceLine objects), the number of iterations jointly accounted for + by the preamble and postamble (the caller will need this to adjust + the loop counter), and the :class:`Result` object for the (kernel) + optimization. The result is ``None`` if no single representative + result is available (currently the case for the halving heuristic). :rtype: any """ @@ -386,10 +388,13 @@ def periodic(body: list, logger: any, conf: any) -> any: # the heuristics for linear optimization. if not conf.sw_pipelining.enabled: res = Heuristics.linear(body, logger=logger, conf=conf) - return [], res.code, [], 0 + return [], res.code, [], 0, res if conf.sw_pipelining.halving_heuristic: - return Heuristics._periodic_halving(body, logger, conf) + preamble, kernel, postamble, num_exceptional = Heuristics._periodic_halving( + body, logger, conf + ) + return preamble, kernel, postamble, num_exceptional, None # 'Normal' software pipelining # @@ -437,7 +442,7 @@ def periodic(body: list, logger: any, conf: any) -> any: ) postamble = res_postamble.code - return preamble, kernel, postamble, num_exceptional_iterations + return preamble, kernel, postamble, num_exceptional_iterations, result @staticmethod def linear(body: list, logger: any, conf: any) -> any: diff --git a/slothy/core/slothy.py b/slothy/core/slothy.py index e625f66ad..0e5564350 100644 --- a/slothy/core/slothy.py +++ b/slothy/core/slothy.py @@ -394,7 +394,9 @@ def optimize( pre, body, post, "ORIGINAL", indentation ) - early, core, late, num_exceptional = Heuristics.periodic(body, logger, c) + early, core, late, num_exceptional, result = Heuristics.periodic( + body, logger, c + ) if self.config.with_llvm_mca_before is True: core = core + orig_stats @@ -436,6 +438,11 @@ def indented(code): self.source = pre + optimized_source + post assert SourceLine.is_source(self.source) + # Expose the optimization result so that callers (e.g. tests) can inspect + # SLOTHY's performance estimate, such as result.cycles and result.stalls. + self.last_result = result + self.success = True + def get_loop_input_output( self, loop_lbl: str, forced_loop_type: any = None ) -> list: @@ -632,7 +639,7 @@ def optimize_loop( early, body, late, "ORIGINAL", indentation ) - preamble_code, kernel_code, postamble_code, num_exceptional = ( + preamble_code, kernel_code, postamble_code, num_exceptional, result = ( Heuristics.periodic(body, logger, c) ) @@ -746,5 +753,13 @@ def loop_lbl_iter(i): DFG(kernel_code, logger.getChild("dfg_kernel_deps"), dfgc).inputs ) + # Expose SLOTHY's performance estimate for the loop kernel so that callers + # (e.g. tests) can inspect it. These refer to a single kernel iteration. + if result is not None: + self.last_result.cycles = result.cycles + self.last_result.stalls = result.stalls + self.last_result.codesize = result.codesize + self.last_result.codesize_with_bubbles = result.codesize_with_bubbles + self.source = early + optimized_code + late self.success = True diff --git a/tests/pytest/conftest.py b/tests/pytest/conftest.py new file mode 100644 index 000000000..06268a068 --- /dev/null +++ b/tests/pytest/conftest.py @@ -0,0 +1,13 @@ +# +# Copyright (c) SLOTHY contributors +# SPDX-License-Identifier: MIT +# + +"""Pytest configuration for the SLOTHY test suite.""" + +import sys +from pathlib import Path + +# Make the slothy package importable when running `pytest` from the repo root +# without having installed slothy (mirrors how test.py is run from the root). +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) diff --git a/tests/pytest/estimated_performance/cases/aarch64_ldst.s b/tests/pytest/estimated_performance/cases/aarch64_ldst.s new file mode 100644 index 000000000..5bd9d412f --- /dev/null +++ b/tests/pytest/estimated_performance/cases/aarch64_ldst.s @@ -0,0 +1,6 @@ + +ld3 {v2.8b, v3.8b, v4.8b}, [x0] +st3 {v2.8b, v3.8b, v4.8b}, [x0], #24 + +ld4 {v0.4S, v1.4S, v2.4S, v3.4S}, [x0] +st4 {v0.4S, v1.4S, v2.4S, v3.4S}, [x0], #64 \ No newline at end of file diff --git a/tests/pytest/estimated_performance/cases/aarch64_ldst.yml b/tests/pytest/estimated_performance/cases/aarch64_ldst.yml new file mode 100644 index 000000000..43d067c2b --- /dev/null +++ b/tests/pytest/estimated_performance/cases/aarch64_ldst.yml @@ -0,0 +1,14 @@ +optimize: + call: optimize + +config: + variable_size: true + constraints.stalls_first_attempt: 32 + +expected: + cortex_a55: + cycles: 23 + stalls: 21 + cortex_a72: + cycles: 8 + stalls: 6 diff --git a/tests/pytest/estimated_performance/cases/aarch64_simple0.s b/tests/pytest/estimated_performance/cases/aarch64_simple0.s new file mode 100644 index 000000000..990876b6f --- /dev/null +++ b/tests/pytest/estimated_performance/cases/aarch64_simple0.s @@ -0,0 +1,24 @@ +ldr q0, [x1, #0] +ldr q1, [x2, #0] + +ldr q8,[x0] +ldr q9, [x0, #1*16] +ldr q10,[x0, #2*16] +ldr q11,[x0,#3*16] + +mul v24.8h, v9.8h, v0.h[0] +sqrdmulh v9.8h, v9.8h, v0.h[1] +mls v24.8h, v9.8h, v1.h[0] +sub v9.8h,v8.8h,v24.8h +add v8.8h, v8.8h, v24.8h + +mul v24.8h, v11.8h, v0.h[0] +sqrdmulh v11.8h, v11.8h, v0.h[1] +mls v24.8h, v11.8h, v1.h[0] +sub v11.8h, v10.8h, v24.8h +add v10.8h, v10.8h, v24.8h + +str q8, [x0], #4*16 +str q9, [x0, #-3*16] +str q10, [x0, #-2*16] +str q11, [x0, #-1*16] diff --git a/tests/pytest/estimated_performance/cases/aarch64_simple0.yml b/tests/pytest/estimated_performance/cases/aarch64_simple0.yml new file mode 100644 index 000000000..6327dd504 --- /dev/null +++ b/tests/pytest/estimated_performance/cases/aarch64_simple0.yml @@ -0,0 +1,11 @@ +optimize: + call: optimize + +config: + variable_size: true + constraints.stalls_first_attempt: 32 + +expected: + cortex_a55: + cycles: 28 + stalls: 18 diff --git a/tests/pytest/estimated_performance/test_estimated_performance.py b/tests/pytest/estimated_performance/test_estimated_performance.py new file mode 100644 index 000000000..29936b98e --- /dev/null +++ b/tests/pytest/estimated_performance/test_estimated_performance.py @@ -0,0 +1,180 @@ +# +# Copyright (c) SLOTHY contributors +# SPDX-License-Identifier: MIT +# + +"""Estimated-performance regression tests. + +Each case is a pair of files in ``cases``: an assembly snippet ``.s`` and a +sidecar ``.yml`` describing how to invoke SLOTHY and the cycle/stall +estimate it is expected to report for the optimized code. These are SLOTHY's own +estimates (``result.cycles`` / ``result.stalls``), not measured hardware numbers; +because they are the optimum of a minimization they are deterministic, so pinning +them turns a change in what SLOTHY believes it found into a test failure. + +Run the tests with:: + + python3 -m pytest tests/pytest + +Print SLOTHY's current estimate for every case (handy when authoring or +refreshing a sidecar) with:: + + PYTHONPATH=. python3 tests/pytest/estimated_performance/test_estimated_performance.py +""" + +import logging +from pathlib import Path + +import pytest +import yaml + +from slothy import Slothy + +import slothy.targets.arm_v7m.arch_v7m as Arch_Armv7M +import slothy.targets.arm_v81m.arch_v81m as Arch_Armv81M +import slothy.targets.aarch64.aarch64_neon as AArch64_Neon +import slothy.targets.riscv.riscv as RISCV + +import slothy.targets.arm_v7m.cortex_m7 as Target_CortexM7 +import slothy.targets.arm_v81m.cortex_m55r1 as Target_CortexM55r1 +import slothy.targets.arm_v81m.cortex_m85r1 as Target_CortexM85r1 +import slothy.targets.aarch64.cortex_a55 as Target_CortexA55 +import slothy.targets.aarch64.cortex_a72_frontend as Target_CortexA72 +import slothy.targets.aarch64.neoverse_n1_experimental as Target_NeoverseN1 +import slothy.targets.aarch64.aarch64_big_experimental as Target_AArch64Big +import slothy.targets.aarch64.apple_m1_firestorm_experimental as Target_AppleM1Firestorm +import slothy.targets.aarch64.apple_m1_icestorm_experimental as Target_AppleM1Icestorm +import slothy.targets.riscv.xuantie_c908 as Target_XuanTieC908 + +# Maps the `target` name used in a sidecar to its (architecture, target) modules. +# The architecture is derived from the target, so sidecars only name the target. +TARGETS = { + "cortex_m7": (Arch_Armv7M, Target_CortexM7), + "cortex_m55r1": (Arch_Armv81M, Target_CortexM55r1), + "cortex_m85r1": (Arch_Armv81M, Target_CortexM85r1), + "cortex_a55": (AArch64_Neon, Target_CortexA55), + "cortex_a72": (AArch64_Neon, Target_CortexA72), + "neoverse_n1": (AArch64_Neon, Target_NeoverseN1), + "aarch64_big": (AArch64_Neon, Target_AArch64Big), + "apple_m1_firestorm": (AArch64_Neon, Target_AppleM1Firestorm), + "apple_m1_icestorm": (AArch64_Neon, Target_AppleM1Icestorm), + "xuantie_c908": (RISCV, Target_XuanTieC908), +} + +CASES_DIR = Path(__file__).resolve().parent / "cases" + + +def _load_case(path): + """Load a sidecar YAML file into a dict.""" + with open(path, "r", encoding="utf8") as f: + return yaml.safe_load(f) + + +def _apply_config(config, overrides): + """Apply a mapping of (possibly dotted) keys onto a slothy.config object.""" + for dotted, value in overrides.items(): + obj = config + *parents, leaf = dotted.split(".") + for name in parents: + obj = getattr(obj, name) + setattr(obj, leaf, value) + + +def _run(case_path, target_name): + """Run SLOTHY for one (case, target) pair and return slothy.last_result.""" + if target_name not in TARGETS: + raise KeyError( + f"Unknown target '{target_name}' in {case_path.name}; " + f"known targets: {sorted(TARGETS)}" + ) + arch, target = TARGETS[target_name] + + case = _load_case(case_path) + source = case.get("source") or f"{case_path.stem}.s" + source_path = case_path.parent / source + + logger = logging.getLogger(f"estperf.{case_path.stem}.{target_name}") + logger.setLevel(logging.WARNING) + + slothy = Slothy(arch, target, logger=logger) + slothy.load_source_from_file(str(source_path)) + + # variable_size lets SLOTHY minimize stalls; a case may override it. + cfg = {"variable_size": True} + cfg.update(case.get("config") or {}) + _apply_config(slothy.config, cfg) + + # On Apple M1, x18 is reserved by the platform ABI. + if "m1" in target_name: + slothy.config.reserved_regs = ["x18"] + + opt = case.get("optimize") or {} + call = opt.get("call", "optimize") + if call == "optimize": + slothy.optimize(start=opt.get("start"), end=opt.get("end")) + elif call == "optimize_loop": + slothy.optimize_loop(opt.get("loop")) + else: + raise ValueError(f"Unknown optimize.call '{call}' in {case_path.name}") + + return slothy.last_result + + +def _discover(): + """Discover (case_path, target_name) pairs from the sidecars in CASES_DIR.""" + params, ids = [], [] + for path in sorted(CASES_DIR.glob("*.yml")): + case = _load_case(path) + for target_name in case.get("expected") or {}: + params.append((path, target_name)) + ids.append(f"{path.stem}-{target_name}") + return params, ids + + +_PARAMS, _IDS = _discover() + + +@pytest.mark.parametrize("case_path,target_name", _PARAMS, ids=_IDS) +def test_estimated_performance(case_path, target_name): + """SLOTHY's estimate for the optimized code matches the pinned values.""" + expected = _load_case(case_path)["expected"][target_name] + result = _run(case_path, target_name) + assert result is not None, "optimization did not expose a result" + + label = f"{case_path.stem}[{target_name}]" + + # A case may set `match: max` to assert "no worse than" instead of equality, + # e.g. for a large case that only solves to a bound rather than the optimum. + if expected.get("match") == "max": + assert result.cycles <= expected["cycles"], ( + f"{label}: expected <= {expected['cycles']} cycles, " + f"SLOTHY estimated {result.cycles}" + ) + return + + assert result.cycles == expected["cycles"], ( + f"{label}: expected {expected['cycles']} cycles, " + f"SLOTHY estimated {result.cycles}" + ) + if "stalls" in expected: + assert result.stalls == expected["stalls"], ( + f"{label}: expected {expected['stalls']} stalls, " + f"SLOTHY estimated {result.stalls}" + ) + + +def _print_estimates(): + """Print SLOTHY's current estimate for every (case, target) pair.""" + params, _ = _discover() + for case_path, target_name in params: + result = _run(case_path, target_name) + bound = getattr(result, "cycles_bound", None) + optimal = bound is not None and result.cycles == bound + print( + f"{case_path.stem}-{target_name}: " + f"cycles={result.cycles} stalls={result.stalls} optimal={optimal}" + ) + + +if __name__ == "__main__": + _print_estimates()