From 783d311e7d5a493b6c8c1676dba99bee3ba657bb Mon Sep 17 00:00:00 2001 From: Jonathan Karlsen Date: Fri, 31 Jan 2025 12:00:23 +0100 Subject: [PATCH] Fix everest fixture for caching testcase results --- tests/everest/conftest.py | 16 +++++++++++----- tests/everest/test_api_snapshots.py | 6 ++++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/tests/everest/conftest.py b/tests/everest/conftest.py index 41ef76c3e75..0fa6df6621d 100644 --- a/tests/everest/conftest.py +++ b/tests/everest/conftest.py @@ -157,11 +157,17 @@ def create_evaluator_server_config(run_model): @pytest.fixture -def cached_example(pytestconfig, evaluator_server_config_generator): +def cached_example(pytestconfig: pytest.Config, evaluator_server_config_generator): cache = pytestconfig.cache def run_config(test_data_case: str): - if cache.get(f"cached_example:{test_data_case}", None) is None: + if ( + cache.get( + f"cached_example:{test_data_case}:{os.environ.get('PYTEST_XDIST_WORKER', '')}", + None, + ) + is None + ): my_tmpdir = Path(tempfile.mkdtemp()) config_path = ( Path(__file__) / f"../../../test-data/everest/{test_data_case}" @@ -187,14 +193,14 @@ def run_config(test_data_case: str): } cache.set( - f"cached_example:{test_data_case}", + f"cached_example:{test_data_case}:{os.environ.get('PYTEST_XDIST_WORKER', '')}", (str(result_path), config_file, optimal_result_json), ) result_path, config_file, optimal_result_json = cache.get( - f"cached_example:{test_data_case}", (None, None, None) + f"cached_example:{test_data_case}:{os.environ.get('PYTEST_XDIST_WORKER', '')}", + (None, None, None), ) - copied_tmpdir = tempfile.mkdtemp() shutil.copytree(result_path, Path(copied_tmpdir) / "everest") copied_path = str(Path(copied_tmpdir) / "everest") diff --git a/tests/everest/test_api_snapshots.py b/tests/everest/test_api_snapshots.py index cac5fdac177..bcafc25ba22 100644 --- a/tests/everest/test_api_snapshots.py +++ b/tests/everest/test_api_snapshots.py @@ -74,6 +74,9 @@ def test_api_snapshots(config_file, snapshot, cached_example): .strip() + "\n" ) + snapshot.snapshot_dir = ( + Path(str(snapshot.snapshot_dir).split(config_file)[0]) / config_file + ) snapshot.assert_match(snapshot_str, "snapshot.json") @@ -119,6 +122,9 @@ def test_api_summary_snapshot(config_file, snapshot, cached_example): api = EverestDataAPI(config) dicts = api.summary_values().to_dicts() + snapshot.snapshot_dir = ( + Path(str(snapshot.snapshot_dir).split(config_file)[0]) / config_file + ) snapshot.assert_match( orjson.dumps(dicts, option=orjson.OPT_INDENT_2).decode("utf-8").strip() + "\n", "snapshot.json",