Merge pull request #2965 from cta-observatory/update_monitoring_data_format

Add data quality-related subgroup definitions to HDF5 data format

Author: maxnoe

docs/changes/2965.datamodel.rst

@@ -0,0 +1 @@
+Add data quality monitoring groups to the HDF5 data format specification.

src/ctapipe/io/datawriter.py

@@ -44,8 +44,9 @@ def _get_tel_index(event, tel_id):
 #   (meaning readers need to update scripts)
 # - increase the minor number if new columns or datasets are added
 # - increase the patch number if there is a small bugfix to the model.
-DATA_MODEL_VERSION = "v7.3.0"
+DATA_MODEL_VERSION = "v7.4.0"
 DATA_MODEL_CHANGE_HISTORY = """
+- v7.4.0: - Add new data quality and top-level monitoring groups for DL0 and DL1 data.
 - v7.3.0: - Add possibility to attach monitoring data to the event HDF5 file.
           - Add the event type to the telescope trigger container.
 - v7.2.0: - Added new monitoring groups: DL1_TEL_[OPTICAL_PSF, MUON_THROUGHPUT, ILLUMINATOR_THROUGHPUT]_GROUP

src/ctapipe/io/hdf5dataformat.py

@@ -23,6 +23,14 @@
     "SIMULATION_PARAMETERS_GROUP",
     "R0_TEL_GROUP",
     "R1_TEL_GROUP",
+    "DL0_TEL_MONITORING_GROUP",
+    "DL0_SUBARRAY_MONITORING_GROUP",
+    "DL0_TEL_QUALITY_GROUP",
+    "DL0_SUBARRAY_QUALITY_GROUP",
+    "DL1_TEL_MONITORING_GROUP",
+    "DL1_SUBARRAY_MONITORING_GROUP",
+    "DL1_SUBARRAY_QUALITY_GROUP",
+    "DL1_TEL_QUALITY_GROUP",
     "DL1_SUBARRAY_GROUP",
     "DL1_SUBARRAY_TRIGGER_TABLE",
     "DL1_TEL_GROUP",
@@ -101,6 +109,14 @@
 DL2_SUBARRAY_PARTICLETYPE_GROUP = "/dl2/event/subarray/particle_type"
 
 # Monitoring group
+DL0_TEL_MONITORING_GROUP = "/dl0/monitoring/telescope"
+DL0_SUBARRAY_MONITORING_GROUP = "/dl0/monitoring/subarray"
+DL0_TEL_QUALITY_GROUP = "/dl0/monitoring/telescope/quality"
+DL0_SUBARRAY_QUALITY_GROUP = "/dl0/monitoring/subarray/quality"
+DL1_TEL_MONITORING_GROUP = "/dl1/monitoring/telescope"
+DL1_SUBARRAY_MONITORING_GROUP = "/dl1/monitoring/subarray"
+DL1_TEL_QUALITY_GROUP = "/dl1/monitoring/telescope/quality"
+DL1_SUBARRAY_QUALITY_GROUP = "/dl1/monitoring/subarray/quality"
 DL0_TEL_POINTING_GROUP = "/dl0/monitoring/telescope/pointing"
 DL1_SUBARRAY_POINTING_GROUP = "/dl1/monitoring/subarray/pointing"
 DL1_TEL_POINTING_GROUP = "/dl1/monitoring/telescope/pointing"

src/ctapipe/io/hdf5eventsource.py

@@ -106,6 +106,7 @@
     "v7.1.0",
     "v7.2.0",
     "v7.3.0",
+    "v7.4.0",
 ]
 
 

src/ctapipe/io/hdf5merger.py

@@ -14,12 +14,15 @@
 from ..utils.arrays import recarray_drop_columns
 from . import metadata
 from .hdf5dataformat import (
+    DL0_SUBARRAY_QUALITY_GROUP,
     DL0_TEL_POINTING_GROUP,
+    DL0_TEL_QUALITY_GROUP,
     DL1_CAMERA_COEFFICIENTS_GROUP,
     DL1_COLUMN_NAMES,
     DL1_IMAGE_STATISTICS_TABLE,
     DL1_PIXEL_STATISTICS_GROUP,
     DL1_SUBARRAY_POINTING_GROUP,
+    DL1_SUBARRAY_QUALITY_GROUP,
     DL1_SUBARRAY_TRIGGER_TABLE,
     DL1_TEL_CALIBRATION_GROUP,
     DL1_TEL_ILLUMINATOR_THROUGHPUT_GROUP,
@@ -29,6 +32,7 @@
     DL1_TEL_OPTICAL_PSF_GROUP,
     DL1_TEL_PARAMETERS_GROUP,
     DL1_TEL_POINTING_GROUP,
+    DL1_TEL_QUALITY_GROUP,
     DL1_TEL_TRIGGER_TABLE,
     DL2_EVENT_STATISTICS_GROUP,
     DL2_SUBARRAY_CROSS_CALIBRATION_GROUP,
@@ -53,6 +57,7 @@
 COMPATIBLE_DATA_MODEL_VERSIONS = [
     "v7.2.0",
     "v7.3.0",
+    "v7.4.0",
 ]
 
 
@@ -475,9 +480,11 @@ def _append_monitoring_data(self, other):
         """Append monitoring data (pointing, calibration, throughput, pixel statistics)."""
         self._append_monitoring_subarray_groups(other)
         self._append_monitoring_dl2_groups(other)
+        self._append_quality_subarray_groups(other)
         if self.telescope_events:
             self._append_monitoring_telescope_groups(other)
             self._append_pixel_statistics(other)
+            self._append_quality_telescope_groups(other)
 
     def _append_monitoring_subarray_groups(self, other):
         """Append monitoring subarray groups."""
@@ -523,6 +530,32 @@ def _append_pixel_statistics(self, other):
                         other, other.root[key], once=self.single_ob
                     )
 
+    def _append_quality_telescope_groups(self, other):
+        """Append quality telescope groups.
+
+        Traverses ``DL0_TEL_QUALITY_GROUP`` and ``DL1_TEL_QUALITY_GROUP`` and
+        appends all subgroups, which have the structure
+        ``{quality_group}/{metric}/{tel_XXX}``.
+        """
+        for root_group in (DL0_TEL_QUALITY_GROUP, DL1_TEL_QUALITY_GROUP):
+            if root_group not in other.root:
+                continue
+            for metric_group in other.root[root_group]._f_iter_nodes("Group"):
+                self._append_table_group(other, metric_group, once=self.single_ob)
+
+    def _append_quality_subarray_groups(self, other):
+        """Append quality subarray groups.
+
+        Traverses ``DL0_SUBARRAY_QUALITY_GROUP`` and ``DL1_SUBARRAY_QUALITY_GROUP``
+        and appends all subgroups, which have the structure
+        ``{quality_group}/{array_element}/{metric}``.
+        """
+        for root_group in (DL0_SUBARRAY_QUALITY_GROUP, DL1_SUBARRAY_QUALITY_GROUP):
+            if root_group not in other.root:
+                continue
+            for elem_group in other.root[root_group]._f_iter_nodes("Group"):
+                self._append_table_group(other, elem_group, once=self.single_ob)
+
     def _append_statistics_data(self, other):
         """Append processing statistics data."""
         # quality query statistics

src/ctapipe/io/tests/test_hdf5eventsource.py

@@ -31,7 +31,7 @@ def test_is_compatible(compatible_file, request):
 def test_metadata(dl1_file):
     with HDF5EventSource(input_url=dl1_file) as source:
         assert source.is_simulation
-        assert source.datamodel_version == (7, 3, 0)
+        assert source.datamodel_version == (7, 4, 0)
         assert set(source.datalevels) == {
             DataLevel.DL1_IMAGES,
             DataLevel.DL1_PARAMETERS,