test: add compiler benchmarks

[ss2165]

Closes #787

Follow up #795

Example of running just bench:

[codecov-commenter]

Codecov Report

All modified and coverable lines are covered by tests ✅

Project coverage is 92.96%. Comparing base (b41e0fc) to head (1bde9bf).

Additional details and impacted files

@@           Coverage Diff           @@
##             main     #796   +/-   ##
=======================================
  Coverage   92.96%   92.96%           
=======================================
  Files          71       71           
  Lines        8373     8373           
=======================================
  Hits         7784     7784           
  Misses        589      589           

☔ View full report in Codecov by Sentry.
📢 Have feedback on the report? Share it here.

[tatiana-s]

Where can you see the extra info like nodes and bytes?

test_compiler_bench.py should probably be renamed to test_big_array.py but otherwise looks good 👍

[ss2165]

Where can you see the extra info like nodes and bytes?

I've added a just bench_save command to write data out to a JSON file - you can see it there. Hopefully whatever rendering service we settle on will also be able to show it.

[ss2165]

Example of JSON output

{
    "machine_info": {
        "node": "FHWDM45WYW",
        "processor": "arm",
        "machine": "arm64",
        "python_compiler": "Clang 16.0.6 ",
        "python_implementation": "CPython",
        "python_implementation_version": "3.12.6",
        "python_version": "3.12.6",
        "python_build": [
            "main",
            "Sep  6 2024 19:03:47"
        ],
        "release": "24.3.0",
        "system": "Darwin",
        "cpu": {
            "python_version": "3.12.6.final.0 (64 bit)",
            "cpuinfo_version": [
                9,
                0,
                0
            ],
            "cpuinfo_version_string": "9.0.0",
            "arch": "ARM_8",
            "bits": 64,
            "count": 10,
            "arch_string_raw": "arm64",
            "brand_raw": "Apple M2 Pro"
        }
    },
    "commit_info": {
        "id": "1bde9bf866ff987ada0f33cb39c08cd214fed308",
        "time": "2025-02-05T11:36:29+00:00",
        "author_time": "2025-02-05T11:36:29+00:00",
        "dirty": false,
        "project": "guppylang",
        "branch": "ss/bench"
    },
    "benchmarks": [
        {
            "group": null,
            "name": "test_big_array_compile",
            "fullname": "tests/benchmarks/test_big_array.py::test_big_array_compile",
            "params": null,
            "param": null,
            "extra_info": {
                "nodes": 44064
            },
            "options": {
                "disable_gc": false,
                "timer": "perf_counter",
                "min_rounds": 5,
                "max_time": 1.0,
                "min_time": 5e-06,
                "warmup": false
            },
            "stats": {
                "min": 3.9752212910025264,
                "max": 4.248307959001977,
                "mean": 4.140934758400545,
                "stddev": 0.11329629526366841,
                "rounds": 5,
                "median": 4.156465000000026,
                "iqr": 0.18050488600238168,
                "q1": 4.059443541498695,
                "q3": 4.239948427501076,
                "iqr_outliers": 0,
                "stddev_outliers": 1,
                "outliers": "1;0",
                "ld15iqr": 3.9752212910025264,
                "hd15iqr": 4.248307959001977,
                "ops": 0.24149136809541397,
                "total": 20.704673792002723,
                "iterations": 1
            }
        },
        {
            "group": null,
            "name": "test_big_array_check",
            "fullname": "tests/benchmarks/test_big_array.py::test_big_array_check",
            "params": null,
            "param": null,
            "extra_info": {},
            "options": {
                "disable_gc": false,
                "timer": "perf_counter",
                "min_rounds": 5,
                "max_time": 1.0,
                "min_time": 5e-06,
                "warmup": false
            },
            "stats": {
                "min": 0.28164800000377,
                "max": 0.9083091669963324,
                "mean": 0.4104689084007987,
                "stddev": 0.27832299942772065,
                "rounds": 5,
                "median": 0.2876088749981136,
                "iqr": 0.16177697975217598,
                "q1": 0.28339809350109135,
                "q3": 0.44517507325326733,
                "iqr_outliers": 1,
                "stddev_outliers": 1,
                "outliers": "1;1",
                "ld15iqr": 0.28164800000377,
                "hd15iqr": 0.9083091669963324,
                "ops": 2.4362381158076873,
                "total": 2.0523445420039934,
                "iterations": 1
            }
        },
        {
            "group": null,
            "name": "test_big_array_json",
            "fullname": "tests/benchmarks/test_big_array.py::test_big_array_json",
            "params": null,
            "param": null,
            "extra_info": {
                "bytes": 21371461
            },
            "options": {
                "disable_gc": false,
                "timer": "perf_counter",
                "min_rounds": 5,
                "max_time": 1.0,
                "min_time": 5e-06,
                "warmup": false
            },
            "stats": {
                "min": 3.5713138339997386,
                "max": 4.075987416996213,
                "mean": 3.8275914418001777,
                "stddev": 0.23441430942564725,
                "rounds": 5,
                "median": 3.939101166004548,
                "iqr": 0.41590086424730544,
                "q1": 3.579795271250987,
                "q3": 3.9956961354982923,
                "iqr_outliers": 0,
                "stddev_outliers": 3,
                "outliers": "3;0",
                "ld15iqr": 3.5713138339997386,
                "hd15iqr": 4.075987416996213,
                "ops": 0.2612609039405951,
                "total": 19.137957209000888,
                "iterations": 1
            }
        },
        {
            "group": null,
            "name": "test_compile_many_ctrl_flow",
            "fullname": "tests/benchmarks/test_ctrl_flow.py::test_compile_many_ctrl_flow",
            "params": null,
            "param": null,
            "extra_info": {},
            "options": {
                "disable_gc": false,
                "timer": "perf_counter",
                "min_rounds": 5,
                "max_time": 1.0,
                "min_time": 5e-06,
                "warmup": false
            },
            "stats": {
                "min": 2.24999530473724e-06,
                "max": 8.33300146041438e-06,
                "mean": 2.7777771012754074e-06,
                "stddev": 1.4186863679222934e-06,
                "rounds": 18,
                "median": 2.3544998839497566e-06,
                "iqr": 2.5100598577409983e-07,
                "q1": 2.2909953258931637e-06,
                "q3": 2.5420013116672635e-06,
                "iqr_outliers": 2,
                "stddev_outliers": 1,
                "outliers": "1;2",
                "ld15iqr": 2.24999530473724e-06,
                "hd15iqr": 3.541994374245405e-06,
                "ops": 360000.08767472854,
                "total": 4.999998782295734e-05,
                "iterations": 1
            }
        },
        {
            "group": null,
            "name": "test_import_guppy",
            "fullname": "tests/benchmarks/test_prelude.py::test_import_guppy",
            "params": null,
            "param": null,
            "extra_info": {},
            "options": {
                "disable_gc": false,
                "timer": "perf_counter",
                "min_rounds": 5,
                "max_time": 1.0,
                "min_time": 5e-06,
                "warmup": false
            },
            "stats": {
                "min": 0.0001944169998751022,
                "max": 0.0003337079979246482,
                "mean": 0.0002006374212343787,
                "stddev": 1.0057435602810562e-05,
                "rounds": 451,
                "median": 0.00019700000120792538,
                "iqr": 5.520249032997526e-06,
                "q1": 0.00019609424998634495,
                "q3": 0.00020161449901934247,
                "iqr_outliers": 39,
                "stddev_outliers": 37,
                "outliers": "37;39",
                "ld15iqr": 0.0001944169998751022,
                "hd15iqr": 0.000210291997063905,
                "ops": 4984.11509601606,
                "total": 0.09048747697670478,
                "iterations": 1
            }
        }
    ],
    "datetime": "2025-02-05T11:38:30.799083+00:00",
    "version": "5.1.0"
}