Merge branch 'main' into add-last-updated-workflow

.github/actions/collect_data/src/parsers/python_pytest_parser.py

@@ -10,6 +10,8 @@
 from .parser import Parser
 from . import junit_xml_utils
 from utils import parse_timestamp
+import ast
+import html
 
 
 class PythonPytestParser(Parser):
@@ -102,9 +104,22 @@ def get_category_from_pytest_testcase_(testcase_):
 
     # to be populated with [] if available
     config = None
-
     tags = None
 
+    try:
+        tag_string = properties.get("tags")
+        if tag_string is not None:
+            tags = ast.literal_eval(html.unescape(tag_string))
+    except (ValueError, SyntaxError, TypeError) as e:
+        print(f"Error parsing tags: {e}")
+
+    try:
+        config_string = properties.get("config")
+        if config_string is not None:
+            config = ast.literal_eval(html.unescape(config_string))
+    except (ValueError, SyntaxError, TypeError) as e:
+        print(f"Error parsing config: {e}")
+
     return Test(
         test_start_ts=test_start_ts,
         test_end_ts=test_end_ts,

.github/actions/collect_data/src/pydantic_models.py

@@ -65,6 +65,8 @@ class Job(BaseModel):
         "criteria. Failure mechanisms that are only descriptive of the "
         "job itself."
     )
+    job_status: str = Field(description="Status of the job, e.g. success, failure, cancelled, etc.")
+
     docker_image: Optional[str] = Field(None, description="Name of the Docker image used for the CI job.")
     is_build_job: bool = Field(description="Flag identifying if the job is a software build.")
     job_matrix_config: Optional[dict] = Field(

.github/actions/collect_data/src/utils.py

@@ -181,6 +181,7 @@ def get_job_row_from_github_job(github_job):
     job_end_ts = github_job["completed_at"]
 
     job_success = github_job["conclusion"] == "success"
+    job_status = str(github_job.get("conclusion", "unknown"))
 
     is_build_job = "build" in name or "build" in labels
 
@@ -205,6 +206,7 @@ def get_job_row_from_github_job(github_job):
         "job_start_ts": job_start_ts,
         "job_end_ts": job_end_ts,
         "job_success": job_success,
+        "job_status": job_status,
         "is_build_job": is_build_job,
         "job_matrix_config": job_matrix_config,
         "docker_image": docker_image,

.github/actions/collect_data/test/data/tt_torch_models/mnist.xml

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="utf-8"?>
+<testsuites>
+	<testsuite name="pytest" errors="0" failures="0" skipped="1" tests="2" time="14.073" timestamp="2025-02-07T20:40:32.211846+00:00" hostname="yyz-lab-72-special-jameszianxu-for-reservation-19509">
+		<testcase classname="tests.models.mnist.test_mnist" name="test_mnist_train[full-train]" time="0.002">
+			<properties>
+				<property name="start_timestamp" value="2025-02-07T20:40:32.763501+00:00" />
+				<property name="end_timestamp" value="2025-02-07T20:40:32.767367+00:00" />
+			</properties>
+			<skipped type="pytest.skip" message="Skipped">
+				/localdev/jameszianxu/tt-torch/tests/models/mnist/test_mnist.py:66: Skipped
+			</skipped>
+		</testcase>
+		<testcase classname="tests.models.mnist.test_mnist" name="test_mnist_train[full-eval]" time="13.517">
+			<properties>
+				<property name="start_timestamp" value="2025-02-07T20:40:32.767729+00:00" />
+				<property name="model_name" value="Mnist" />
+				<property name="frontend" value="tt-torch" />
+				<property name="config" value="{'compiler_config': {'compile_depth': 'CompileDepth.EXECUTE', 'profile_ops': True, 'torch_mlir_module': None, 'stablehlo_mlir_module': None, 'unique_ops': {}, 'stable_hlo_ops': [], 'model_name': 'Mnist', 'results_path': 'results/models/', 'single_op_timeout': 30, 'enable_consteval': True, 'remove_embedded_constants': False, '_consteval_parameters': True, '_enable_intermediate_verification': False, '_verify_op_by_op': False}}" />
+				<property name="tags" value="{'pccs': [0.9967075672249393], 'atols': [0.015625]}" />
+				<property name="end_timestamp" value="2025-02-07T20:40:46.284660+00:00" />
+			</properties>
+		</testcase>
+	</testsuite>
+</testsuites>

.github/actions/collect_data/test/test_generate_data.py

@@ -38,6 +38,7 @@ def test_create_pipeline_json(run_id, expected):
         expected_card_types = ["N300", "N150", "E150", None]
         for job in pipeline_json["jobs"]:
             assert job["card_type"] in expected_card_types
+            assert "job_status" in job
 
         # assert pipeline json has the correct number of jobs and tests
         assert len(pipeline_json["jobs"]) == expected["jobs_cnt"]

.github/actions/collect_data/test/test_tt_torch_full_model_tests_parser.py

@@ -0,0 +1,20 @@
+# SPDX-FileCopyrightText: (c) 2025 Tenstorrent AI ULC
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import pytest
+from parsers.python_pytest_parser import PythonPytestParser
+
+
+@pytest.mark.parametrize(
+    "tar, project, github_job_id, expected",
+    [
+        ("mnist.xml", "tt-torch", 5, {"tests_cnt": 2}),
+    ],
+)
+def test_tt_torch_full_model_tests_parser(tar, project, github_job_id, expected):
+    filepath = f"test/data/tt_torch_models/{tar}"
+    parser = PythonPytestParser()
+    assert parser.can_parse(filepath)
+    tests = parser.parse(filepath, project=project, github_job_id=github_job_id)
+    assert len(tests) == expected["tests_cnt"]

.gitignore

@@ -7,3 +7,4 @@
 .env
 venv/**
 venv
+.github/actions/collect_data/benchmark_*.json