extra: ci_shared: deduplicate TestEntry in JSON via composite key

TestEntry gains a key property: '{artifact}|{board}|{sketch}|{link_mode}'.
This uniquely identifies each test; '|' cannot appear in any component.

JSON schema change:
- New top-level 'all_tests' dict: { key: TestEntry }
- TestGroup 'tests' field: [key, ...] instead of [{...full entry...}, ...]

TestGroup.to_dict() stores keys; TestGroup.from_dict(d, all_tests) looks
up TestEntry objects from the shared dict. Each test is serialised once
instead of once per group it belongs to (previously 3x).

ci_collect_data: maintains ALL_TESTS dict, included in JSON output.
ci_generate_report: reconstructs ALL_TESTS first, passes to from_dict.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: pillo79

extra/ci_collect_data.py

@@ -49,6 +49,9 @@
 # Loader build results, one per board
 BOARD_LOADERS = {}                                          # { board: LoaderEntry() }
 
+# All test entries, keyed by (artifact, board, sketch, link_mode) composite key
+ALL_TESTS = {}                                              # { key: TestEntry() }
+
 # Test results grouped by artifact, board, and artifact/sketch
 # (grouping sketch results across different artifacts is really confusing)
 ARTIFACT_TESTS = defaultdict(TestGroup)                     # { artifact: TestGroup() }
@@ -73,6 +76,7 @@ def log_test(artifact, board, sketch, link_mode, exceptions, status, issues, job
     ARTIFACT_TESTS[artifact].track(test_entry)
     BOARD_TESTS[board].track(test_entry)
     SKETCH_TESTS[artifact][sketch].track(test_entry)
+    ALL_TESTS[test_entry.key] = test_entry
 
 # ---------------------------------------------------------------------------
 # Main Logic
@@ -194,6 +198,7 @@ def log_test(artifact, board, sketch, link_mode, exceptions, status, issues, job
     'all_board_data': ALL_BOARD_DATA,
     'regions_by_soc': {soc: list(regions) for soc, regions in REGIONS_BY_SOC.items()},
     'board_loaders': {board: loader.to_dict() for board, loader in BOARD_LOADERS.items()},
+    'all_tests': {key: t.to_dict() for key, t in ALL_TESTS.items()},
     'artifact_tests': {artifact: tg.to_dict() for artifact, tg in ARTIFACT_TESTS.items()},
     'board_tests': {board: tg.to_dict() for board, tg in BOARD_TESTS.items()},
     'sketch_tests': {

extra/ci_generate_report.py

@@ -22,7 +22,7 @@
 import ci_shared
 from ci_shared import (
     TestStatus, SKIP, PASS, WARNING, EXPECTED_ERROR, INVALID_EXCEPTION, ERROR, FAILURE,
-    LoaderEntry, TestGroup,
+    LoaderEntry, TestEntry, TestGroup,
 )
 
 BASE_COLOR = 0x20               # darkest tone
@@ -372,11 +372,14 @@ def print_mem_report(artifact, artifact_boards):
     for board, loader in data['board_loaders'].items()
 }
 
+# Reconstruct all test entries first (needed by TestGroup.from_dict)
+ALL_TESTS = {key: TestEntry.from_dict(t) for key, t in data['all_tests'].items()}
+
 # Reconstruct test group data structures
-ARTIFACT_TESTS = {artifact: TestGroup.from_dict(tg) for artifact, tg in data['artifact_tests'].items()}
-BOARD_TESTS = {board: TestGroup.from_dict(tg) for board, tg in data['board_tests'].items()}
+ARTIFACT_TESTS = {artifact: TestGroup.from_dict(tg, ALL_TESTS) for artifact, tg in data['artifact_tests'].items()}
+BOARD_TESTS = {board: TestGroup.from_dict(tg, ALL_TESTS) for board, tg in data['board_tests'].items()}
 SKETCH_TESTS = {
-    artifact: {sketch: TestGroup.from_dict(tg) for sketch, tg in sketches.items()}
+    artifact: {sketch: TestGroup.from_dict(tg, ALL_TESTS) for sketch, tg in sketches.items()}
     for artifact, sketches in data['sketch_tests'].items()
 }
 

extra/ci_shared.py

@@ -159,6 +159,10 @@ def __init__(self, artifact, board, sketch, link_mode, excepted, status, issues,
                 self.group = ""
                 self.name = sketch
 
+    @property
+    def key(self):
+        return f"{self.artifact}|{self.board}|{self.sketch}|{self.link_mode}"
+
     def to_dict(self):
         return {
             'artifact': self.artifact,
@@ -225,17 +229,17 @@ def to_dict(self):
             'group_names': [list(gn) for gn in self.group_names],
             'counts': {str(int(k)): v for k, v in self.counts.items()},
             'status': int(self.status),
-            'tests': [t.to_dict() for t in self.tests],
+            'tests': [t.key for t in self.tests],
         }
 
     @classmethod
-    def from_dict(cls, d):
+    def from_dict(cls, d, all_tests):
         tg = cls.__new__(cls)
         tg.boards = set(d['boards'])
         tg.sketches = set(d['sketches'])
         tg.group_names = set(tuple(gn) for gn in d['group_names'])
         tg.counts = {TestStatus(int(k)): v for k, v in d['counts'].items()}
         tg.status = TestStatus(d['status'])
-        tg.tests = [TestEntry.from_dict(t) for t in d['tests']]
+        tg.tests = [all_tests[k] for k in d['tests']]
         tg.tests_with_issues = [t for t in tg.tests if t.issues]
         return tg