From f347f04b8da9370e2c20f3c56e03b70ade0c776e Mon Sep 17 00:00:00 2001 From: James Zian Xu Date: Thu, 13 Feb 2025 16:48:50 -0500 Subject: [PATCH 1/2] Improve Pytest Parser (#21) - Add parsers for deserialization of python dict as serialized by junitxml to capture config and tags - Add test coverage for tt_torch pytest full model tests --- .../src/parsers/python_pytest_parser.py | 17 ++++++++++++- .../test/data/tt_torch_models/mnist.xml | 24 +++++++++++++++++++ .../test_tt_torch_full_model_tests_parser.py | 20 ++++++++++++++++ 3 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 .github/actions/collect_data/test/data/tt_torch_models/mnist.xml create mode 100644 .github/actions/collect_data/test/test_tt_torch_full_model_tests_parser.py diff --git a/.github/actions/collect_data/src/parsers/python_pytest_parser.py b/.github/actions/collect_data/src/parsers/python_pytest_parser.py index 6c323cb..611d88c 100644 --- a/.github/actions/collect_data/src/parsers/python_pytest_parser.py +++ b/.github/actions/collect_data/src/parsers/python_pytest_parser.py @@ -10,6 +10,8 @@ from .parser import Parser from . import junit_xml_utils from utils import parse_timestamp +import ast +import html class PythonPytestParser(Parser): @@ -102,9 +104,22 @@ def get_category_from_pytest_testcase_(testcase_): # to be populated with [] if available config = None - tags = None + try: + tag_string = properties.get("tags") + if tag_string is not None: + tags = ast.literal_eval(html.unescape(tag_string)) + except (ValueError, SyntaxError, TypeError) as e: + print(f"Error parsing tags: {e}") + + try: + config_string = properties.get("config") + if config_string is not None: + config = ast.literal_eval(html.unescape(config_string)) + except (ValueError, SyntaxError, TypeError) as e: + print(f"Error parsing config: {e}") + return Test( test_start_ts=test_start_ts, test_end_ts=test_end_ts, diff --git a/.github/actions/collect_data/test/data/tt_torch_models/mnist.xml b/.github/actions/collect_data/test/data/tt_torch_models/mnist.xml new file mode 100644 index 0000000..7a9c974 --- /dev/null +++ b/.github/actions/collect_data/test/data/tt_torch_models/mnist.xml @@ -0,0 +1,24 @@ + + + + + + + + + + /localdev/jameszianxu/tt-torch/tests/models/mnist/test_mnist.py:66: Skipped + + + + + + + + + + + + + + diff --git a/.github/actions/collect_data/test/test_tt_torch_full_model_tests_parser.py b/.github/actions/collect_data/test/test_tt_torch_full_model_tests_parser.py new file mode 100644 index 0000000..ec1be24 --- /dev/null +++ b/.github/actions/collect_data/test/test_tt_torch_full_model_tests_parser.py @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: (c) 2025 Tenstorrent AI ULC +# +# SPDX-License-Identifier: Apache-2.0 + +import pytest +from parsers.python_pytest_parser import PythonPytestParser + + +@pytest.mark.parametrize( + "tar, project, github_job_id, expected", + [ + ("mnist.xml", "tt-torch", 5, {"tests_cnt": 2}), + ], +) +def test_tt_torch_full_model_tests_parser(tar, project, github_job_id, expected): + filepath = f"test/data/tt_torch_models/{tar}" + parser = PythonPytestParser() + assert parser.can_parse(filepath) + tests = parser.parse(filepath, project=project, github_job_id=github_job_id) + assert len(tests) == expected["tests_cnt"] From a4c71978d2ffe29b1e92e7112b56aa3f29709eaf Mon Sep 17 00:00:00 2001 From: Nebojsa Sumrak Date: Fri, 21 Feb 2025 11:38:04 +0100 Subject: [PATCH 2/2] Nsumrak/add job status (#27) * job_status added * Test job_status --- .github/actions/collect_data/src/pydantic_models.py | 2 ++ .github/actions/collect_data/src/utils.py | 2 ++ .github/actions/collect_data/test/test_generate_data.py | 1 + .gitignore | 1 + 4 files changed, 6 insertions(+) diff --git a/.github/actions/collect_data/src/pydantic_models.py b/.github/actions/collect_data/src/pydantic_models.py index 5dffc68..cfce26b 100644 --- a/.github/actions/collect_data/src/pydantic_models.py +++ b/.github/actions/collect_data/src/pydantic_models.py @@ -65,6 +65,8 @@ class Job(BaseModel): "criteria. Failure mechanisms that are only descriptive of the " "job itself." ) + job_status: str = Field(description="Status of the job, e.g. success, failure, cancelled, etc.") + docker_image: Optional[str] = Field(None, description="Name of the Docker image used for the CI job.") is_build_job: bool = Field(description="Flag identifying if the job is a software build.") job_matrix_config: Optional[dict] = Field( diff --git a/.github/actions/collect_data/src/utils.py b/.github/actions/collect_data/src/utils.py index 2f1c6bf..70b4253 100644 --- a/.github/actions/collect_data/src/utils.py +++ b/.github/actions/collect_data/src/utils.py @@ -181,6 +181,7 @@ def get_job_row_from_github_job(github_job): job_end_ts = github_job["completed_at"] job_success = github_job["conclusion"] == "success" + job_status = str(github_job.get("conclusion", "unknown")) is_build_job = "build" in name or "build" in labels @@ -205,6 +206,7 @@ def get_job_row_from_github_job(github_job): "job_start_ts": job_start_ts, "job_end_ts": job_end_ts, "job_success": job_success, + "job_status": job_status, "is_build_job": is_build_job, "job_matrix_config": job_matrix_config, "docker_image": docker_image, diff --git a/.github/actions/collect_data/test/test_generate_data.py b/.github/actions/collect_data/test/test_generate_data.py index efa481a..c83276c 100644 --- a/.github/actions/collect_data/test/test_generate_data.py +++ b/.github/actions/collect_data/test/test_generate_data.py @@ -38,6 +38,7 @@ def test_create_pipeline_json(run_id, expected): expected_card_types = ["N300", "N150", "E150", None] for job in pipeline_json["jobs"]: assert job["card_type"] in expected_card_types + assert "job_status" in job # assert pipeline json has the correct number of jobs and tests assert len(pipeline_json["jobs"]) == expected["jobs_cnt"] diff --git a/.gitignore b/.gitignore index 13f3a60..9ae8fb5 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ .github/actions/collect_data/test/data/tt_torch_models/models/** .github/actions/collect_data/generated/** .env +.github/actions/collect_data/benchmark_*.json