New NestedColumn to better represent groups of hierarchical fields

Author: FrancescAlted

doc/reference/classes.rst

@@ -55,6 +55,7 @@ Columnar table containers, column views, indexes, and CTable schema helpers.
 
     CTable
     Column
+    NestedColumn
     Index
     NullPolicy
 

doc/reference/ctable.rst

@@ -615,6 +615,54 @@ Null sentinel values are automatically excluded from all aggregates.
 .. automethod:: Column.all
 
 
+----
+
+.. _NestedColumn:
+
+NestedColumn
+============
+
+A read-only accessor for a nested (dotted) group of CTable columns, returned by
+attribute access on a :class:`CTable` (or on another ``NestedColumn``) when the
+name refers to an internal node of the dotted column tree rather than a leaf.
+
+For a table flattened from a ``struct`` / ``list<struct>`` schema (see
+:ref:`Nested fields <NestedFields>`), ``t.trip`` is a ``NestedColumn`` grouping
+every leaf under the ``trip.`` prefix, while a leaf such as ``t.trip.sec`` or
+``t.trip.begin.lon`` is a :class:`Column`.  Drilling into an intermediate node
+yields another ``NestedColumn``::
+
+    t.trip                     # <NestedColumn 'trip'>
+    t.trip.col_names           # ['sec', 'km', 'begin.lon', 'begin.lat', ...]
+    t.trip.begin               # <NestedColumn 'trip.begin'>
+    t.trip.begin.lon           # Column
+    print(t.trip.info)         # aggregate metadata over the group
+
+Users do not instantiate ``NestedColumn`` directly.
+
+.. autoclass:: NestedColumn
+
+.. rubric:: Attributes
+
+.. autosummary::
+
+    NestedColumn.col_names
+    NestedColumn.nrows
+    NestedColumn.ncols
+    NestedColumn.nbytes
+    NestedColumn.cbytes
+    NestedColumn.cratio
+    NestedColumn.info
+
+.. autoproperty:: NestedColumn.col_names
+.. autoproperty:: NestedColumn.nrows
+.. autoproperty:: NestedColumn.ncols
+.. autoproperty:: NestedColumn.nbytes
+.. autoproperty:: NestedColumn.cbytes
+.. autoproperty:: NestedColumn.cratio
+.. autoproperty:: NestedColumn.info
+
+
 ----
 
 .. _SchemaSpecs:
@@ -760,6 +808,8 @@ to a typed representation.  They are not used as an implicit fallback during
 Parquet import; unsupported Arrow/Parquet types still raise unless explicitly
 imported through :meth:`CTable.from_arrow` with ``object_fallback=True``.
 
+.. _NestedFields:
+
 Nested fields
 -------------
 
@@ -803,6 +853,11 @@ attribute proxies::
     t["trip.begin.lon"].mean()      # Column object (fast path)
     t.trip.begin.lon.max()          # attribute proxy, same column
 
+Accessing an intermediate prefix such as ``t.trip`` or ``t.trip.begin`` returns
+a :class:`~blosc2.NestedColumn` that groups all descendant leaves and reports
+aggregate metadata via :attr:`~blosc2.NestedColumn.info`; a leaf such as
+``t.trip.begin.lon`` returns a :class:`Column`.
+
 A literal ``.``, ``/``, or ``\\`` inside an Arrow field name is escaped with a
 backslash in the logical column name.  For example, path segments
 ``("trip.info", "begin/point", "lon.deg")`` become::

src/blosc2/__init__.py

@@ -632,6 +632,7 @@ def _raise(exc):
     DEFAULT_NULL_POLICY,
     Column,
     CTable,
+    NestedColumn,
     NullPolicy,
     RowTransformer,
     get_null_policy,
@@ -827,8 +828,11 @@ def _raise(exc):
     "group_reduce",
     # Classes
     "C2Array",
+    "Column",
     "CParams",
+    "CTable",
     "CTableGroupBy",
+    "NestedColumn",
     "RowTransformer",
     "Batch",
     "BatchArray",

src/blosc2/ctable.py

@@ -2486,11 +2486,29 @@ def __iter__(self):
             yield self._row_value_at_logical(i)
 
 
-class _NestedColumnNamespace:
-    """Attribute proxy for dotted nested column paths.
+class NestedColumn:
+    """A read-only accessor for a nested (dotted) group of CTable columns.
+
+    Returned by attribute access on a :class:`CTable` (or on another
+    ``NestedColumn``) when the name refers to an internal node of the dotted
+    column tree rather than a leaf.  For a table flattened from a
+    ``struct``/``list<struct>`` schema, ``t.trip`` is a ``NestedColumn``
+    grouping every leaf under the ``trip.`` prefix, while a leaf such as
+    ``t.trip.sec`` (or ``t.trip.begin.lon``) is a :class:`Column`.  Drilling
+    into an intermediate node (e.g. ``t.trip.begin``) yields another
+    ``NestedColumn``.
+
+    Exposes aggregate metadata over its descendant leaf columns
+    (:attr:`col_names`, :attr:`nrows`, :attr:`ncols`, :attr:`nbytes`,
+    :attr:`cbytes`, :attr:`cratio`) and an :attr:`info` report.
 
-    Allows `t.trip.begin.lon` when the physical leaf column is named
-    `"trip.begin.lon"`.
+    Examples
+    --------
+    >>> t.trip                      # doctest: +SKIP
+    <NestedColumn 'trip'>
+    >>> t.trip.col_names            # doctest: +SKIP
+    ['sec', 'km', 'begin.lon', ...]
+    >>> t.trip.sec                  # a leaf -> Column            # doctest: +SKIP
     """
 
     def __init__(self, table: CTable, prefix: str):
@@ -2600,11 +2618,11 @@ def __getattr__(self, name: str):
         for col_name in self._table.col_names:
             parts = split_field_path(col_name)
             if parts[: len(path_parts)] == path_parts and len(parts) > len(path_parts):
-                return _NestedColumnNamespace(self._table, path)
+                return NestedColumn(self._table, path)
         raise AttributeError(path)
 
     def __repr__(self) -> str:
-        return f"<NestedColumnNamespace {self._prefix!r}>"
+        return f"<NestedColumn {self._prefix!r}>"
 
 
 class _LazyColumnDict(dict):
@@ -8536,7 +8554,7 @@ def _nested_namespace(self, prefix: str):
         for name in self.col_names:
             parts = split_field_path(name)
             if parts[: len(prefix_parts)] == prefix_parts and len(parts) > len(prefix_parts):
-                return _NestedColumnNamespace(self, prefix)
+                return NestedColumn(self, prefix)
         return None
 
     def __getattr__(self, s: str):

src/blosc2/ndarray.py

@@ -3935,23 +3935,16 @@ def info(self) -> InfoReporter:
         chunks  : (10,)
         blocks  : (10,)
         dtype   : int64
-        cratio  : 0.73x
-        cparams : {'blocksize': 80,
-        'clevel': 1,
-        'codec': <Codec.ZSTD: 5>,
-        'codec_meta': 0,
-        'filters': [<Filter.NOFILTER: 0>,
-                <Filter.NOFILTER: 0>,
-                <Filter.NOFILTER: 0>,
-                <Filter.NOFILTER: 0>,
-                <Filter.NOFILTER: 0>,
-                <Filter.SHUFFLE: 1>],
-        'filters_meta': [0, 0, 0, 0, 0, 0],
-        'nthreads': 4,
-        'splitmode': <SplitMode.ALWAYS_SPLIT: 1>,
-        'typesize': 8,
-        'use_dict': 0}
-        dparams : {'nthreads': 4}
+        nbytes  : 80 (80 B)
+        cbytes  : 98 (98 B)
+        cratio  : 0.82x
+        cparams : CParams(codec=<Codec.ZSTD: 5>, codec_meta=0, clevel=5, use_dict=False, typesize=8,
+                : nthreads=8, blocksize=80, splitmode=<SplitMode.AUTO_SPLIT: 3>,
+                : filters=[<Filter.NOFILTER: 0>, <Filter.NOFILTER: 0>, <Filter.NOFILTER: 0>,
+                : <Filter.NOFILTER: 0>, <Filter.NOFILTER: 0>, <Filter.SHUFFLE: 1>], filters_meta=[0, 0,
+                : 0, 0, 0, 0], tuner=<Tuner.STUNE: 0>)
+        dparams : DParams(nthreads=8)
+        <BLANKLINE>
         """
         return InfoReporter(self)
 

src/blosc2/schunk.py

@@ -533,15 +533,16 @@ def info(self) -> InfoReporter:
         chunksize : 24000
         blocksize : 0
         typesize  : 1
-        nbytes    : 24000
-        cbytes    : 82
+        nbytes    : 24000 (23.44 KiB)
+        cbytes    : 82 (82 B)
         cratio    : 292.68x
-        cparams   : CParams(codec=<Codec.ZSTD: 5>, codec_meta=0, clevel=1, use_dict=False, typesize=1,
+        cparams   : CParams(codec=<Codec.ZSTD: 5>, codec_meta=0, clevel=5, use_dict=False, typesize=1,
                   : nthreads=8, blocksize=0, splitmode=<SplitMode.AUTO_SPLIT: 3>,
                   : filters=[<Filter.NOFILTER: 0>, <Filter.NOFILTER: 0>, <Filter.NOFILTER: 0>,
                   : <Filter.NOFILTER: 0>, <Filter.NOFILTER: 0>, <Filter.SHUFFLE: 1>], filters_meta=[0,
                   : 0, 0, 0, 0, 0], tuner=<Tuner.STUNE: 0>)
         dparams   : DParams(nthreads=8)
+        <BLANKLINE>
         """
         return InfoReporter(self)
 

tests/ctable/test_ctable_dataclass_schema.py

@@ -348,7 +348,7 @@ class NestedRow:
     assert t.trip.col_names == ["begin.lon", "begin.lat"]
 
     text = repr(info)
-    assert "NestedColumnNamespace" in text
+    assert "NestedColumn" in text
     assert "storage" in text
     assert "schema" in text
     assert "begin.lon" in text