Fix inline CTable TreeStore edge cases:

     - TreeStore.values() now collapses object roots.
     - Parent subtree deletion removes nested inline CTable objects.
     - Missing object registry fallback now hides/protects inline CTable internals.
     - Inline CTable indexes work in .b2d and .b2z.

Author: FrancescAlted

src/blosc2/ctable_storage.py

@@ -504,13 +504,12 @@ def discard(self) -> None:
     @staticmethod
     def _walk_descriptor_paths(descriptor: dict):
         """Yield (obj, key) for every string value that looks like a file path."""
-        _PATH_KEYS = {"path", "values_path", "positions_path", "l1_path", "l2_path"}
         stack = [descriptor]
         while stack:
             obj = stack.pop()
             if isinstance(obj, dict):
                 for k, v in obj.items():
-                    if k in _PATH_KEYS and isinstance(v, str):
+                    if (k == "path" or k.endswith("_path")) and isinstance(v, str):
                         yield obj, k
                     elif isinstance(v, (dict, list)):
                         stack.append(v)
@@ -521,22 +520,22 @@ def _walk_descriptor_paths(descriptor: dict):
 
     @staticmethod
     def _relativize_descriptor(descriptor: dict, working_dir: str) -> dict:
-        """Replace absolute paths inside *working_dir* with ``_indexes/…`` relative paths."""
-        prefix = working_dir.rstrip("/") + "/"
+        """Replace absolute paths inside *working_dir* with working-dir relative paths."""
+        prefix = working_dir.rstrip(os.sep) + os.sep
         d = copy.deepcopy(descriptor)
         for obj, key in FileTableStorage._walk_descriptor_paths(d):
             v = obj[key]
-            if v.startswith(prefix):
-                obj[key] = v[len(prefix) :]
+            if os.path.isabs(v) and v.startswith(prefix):
+                obj[key] = v[len(prefix) :].replace(os.sep, "/")
         return d
 
     @staticmethod
     def _absolutize_descriptor(descriptor: dict, working_dir: str) -> dict:
-        """Expand ``_indexes/…`` relative paths back to absolute using *working_dir*."""
+        """Expand working-dir relative paths back to absolute paths."""
         d = copy.deepcopy(descriptor)
         for obj, key in FileTableStorage._walk_descriptor_paths(d):
             v = obj[key]
-            if v.startswith(_INDEXES_DIR + "/") or v.startswith(_INDEXES_DIR + os.sep):
+            if not os.path.isabs(v):
                 obj[key] = os.path.join(working_dir, v)
         return d
 

src/blosc2/indexing.py

@@ -5889,7 +5889,12 @@ def _bucket_batch_result_dtype(where_x) -> np.dtype:
 
 def _bucket_worker_source(where_x):
     if _supports_block_reads(where_x) and getattr(where_x, "urlpath", None) is not None:
-        return blosc2.open(str(where_x.urlpath), mode="r", mmap_mode=_INDEX_MMAP_MODE)
+        urlpath = str(where_x.urlpath)
+        # Arrays opened from a b2z TreeStore/CTable are offset-backed leaves whose
+        # urlpath points at the outer bundle, not at a standalone .b2nd file.
+        # Reopening that path would materialize the whole TreeStore/CTable.
+        if not urlpath.endswith(".b2z"):
+            return blosc2.open(urlpath, mode="r", mmap_mode=_INDEX_MMAP_MODE)
     return where_x
 
 

src/blosc2/tree_store.py

@@ -229,9 +229,29 @@ def _object_roots(self) -> set:
         """Return all registered object-root full keys."""
         return set(self._objects_registry().keys())
 
+    def _probed_object_roots(self) -> set:
+        """Return object roots discovered from physical CTable manifests."""
+        roots = set()
+        candidates = set(self.map_tree.keys()) | set(self._estore.keys())
+        for key in candidates:
+            if not key.endswith("/_meta"):
+                continue
+            root = key[: -len("/_meta")]
+            if not root:
+                # A manifest at /_meta marks the TreeStore itself as a CTable
+                # backing store; it is not an inline object root to collapse.
+                continue
+            if self._probe_object_info(root) is not None:
+                roots.add(root)
+        return roots
+
+    def _known_object_roots(self) -> set:
+        """Return registered plus physically probed object-root full keys."""
+        return self._object_roots() | self._probed_object_roots()
+
     def _effective_object_roots(self) -> set:
         """Object root keys relative to the current view (subtree or root)."""
-        all_roots = self._object_roots()
+        all_roots = self._known_object_roots()
         if not self.subtree_path:
             return all_roots
         result = set()
@@ -386,7 +406,7 @@ def __setitem__(
 
         # Block overwriting an existing object root with a plain value
         full_key = self._translate_key_to_full(key)
-        if self._object_info(full_key) is not None:
+        if (self._object_info(full_key) or self._probe_object_info(full_key)) is not None:
             raise ValueError(
                 f"'{key}' is an object root (e.g. CTable). "
                 f"Delete it first with `del ts['{key}']` before assigning a new value."
@@ -421,7 +441,7 @@ def _set_ctable_object(self, key: str, value: blosc2.CTable) -> None:
         full_key = self._translate_key_to_full(key)
 
         # Raise if already exists as object root (no silent replace)
-        if self._object_info(full_key) is not None:
+        if (self._object_info(full_key) or self._probe_object_info(full_key)) is not None:
             raise ValueError(
                 f"'{key}' already exists as an object root. Delete it first with `del ts['{key}']`."
             )
@@ -531,7 +551,7 @@ def __delitem__(self, key: str) -> None:
         full_key = self._translate_key_to_full(key)
 
         # --- Object root deletion ---
-        if self._object_info(full_key) is not None:
+        if (self._object_info(full_key) or self._probe_object_info(full_key)) is not None:
             self._delete_object_subtree(full_key)
             return
 
@@ -545,8 +565,14 @@ def __delitem__(self, key: str) -> None:
         # Regular node / subtree deletion
         key_exists_as_data = super().__contains__(full_key)
         descendants = self.get_descendants(key)
-
-        if not key_exists_as_data and not descendants:
+        prefix = full_key + "/" if full_key != "/" else "/"
+        object_roots_to_delete = sorted(
+            [root for root in self._known_object_roots() if root.startswith(prefix)],
+            key=len,
+            reverse=True,
+        )
+
+        if not key_exists_as_data and not descendants and not object_roots_to_delete:
             raise KeyError(f"Key '{key}' not found")
 
         keys_to_delete = []
@@ -557,8 +583,18 @@ def __delitem__(self, key: str) -> None:
             if super().__contains__(full_desc):
                 keys_to_delete.append(descendant)
 
+        for object_root in object_roots_to_delete:
+            self._delete_object_subtree(object_root)
+
         for k in keys_to_delete:
-            super().__delitem__(self._translate_key_to_full(k))
+            full_desc = self._translate_key_to_full(k)
+            if super().__contains__(full_desc):
+                super().__delitem__(full_desc)
+
+        # Remove stale registry entries for any nested objects that were deleted as plain descendants.
+        for root in list(self._object_roots()):
+            if root.startswith(prefix):
+                self._unregister_object(root)
 
     def _delete_object_subtree(self, full_key: str) -> None:
         """Delete all physical leaves under *full_key* and unregister it."""
@@ -614,7 +650,11 @@ def __contains__(self, key: str) -> bool:
             if self._is_object_internal_key(key):
                 return False
             full_key = self._translate_key_to_full(key)
-            return super().__contains__(full_key) or self._object_info(full_key) is not None
+            return (
+                super().__contains__(full_key)
+                or self._object_info(full_key) is not None
+                or self._probe_object_info(full_key) is not None
+            )
         except ValueError:
             return False
 
@@ -679,6 +719,15 @@ def items(self) -> Iterator[tuple[str, NDArray | C2Array | SChunk | TreeStore]]:
         for key in self.keys():
             yield key, self[key]
 
+    def values(
+        self,
+    ) -> Iterator[
+        NDArray | C2Array | SChunk | blosc2.ObjectArray | blosc2.BatchArray | blosc2.CTable | TreeStore
+    ]:
+        """Return values in the current subtree view, with object roots collapsed."""
+        for key in self.keys():
+            yield self[key]
+
     def get_children(self, path: str) -> list[str]:
         """Get direct children of a given path.
 
@@ -815,7 +864,11 @@ def walk(self, path: str = "/", topdown: bool = True) -> Iterator[tuple[str, lis
             child_rel_path = path + "/" + name if path != "/" else "/" + name
             # Translate to full key in the backing store and verify it's a data node or object root
             full_key = self._translate_key_to_full(child_rel_path)
-            if super().__contains__(full_key) or self._object_info(full_key) is not None:
+            if (
+                super().__contains__(full_key)
+                or self._object_info(full_key) is not None
+                or self._probe_object_info(full_key) is not None
+            ):
                 valid_leaf_nodes.append(name)
         leaf_nodes = valid_leaf_nodes
 
@@ -865,7 +918,7 @@ def get_subtree(self, path: str) -> TreeStore:
         full_path = self._translate_key_to_full(path)
 
         # Object roots cannot be navigated as subtrees
-        if self._object_info(full_path) is not None:
+        if (self._object_info(full_path) or self._probe_object_info(full_path)) is not None:
             raise ValueError(
                 f"'{path}' is an object root (e.g. CTable), not a TreeStore subtree. "
                 f"Use ts['{path}'] to access the object."

tests/test_tree_store.py

@@ -1432,3 +1432,65 @@ def test_ctable_context_manager_auto_close(tmp_path, storage_type):
         t2 = ts["/table"]
         assert len(t2) == 3
         assert list(t2["x"][:])[-1] == 100
+
+
+@pytest.mark.parametrize("storage_type", ["b2d", "b2z"])
+def test_ctable_values_collapses_object_roots(tmp_path, storage_type):
+    """values() yields the CTable object, not its internal leaves."""
+    path = str(tmp_path / f"bundle.{storage_type}")
+    with blosc2.TreeStore(path, mode="w") as ts:
+        ts["/table"] = _make_ctable(n=2)
+        values = list(ts.values())
+
+    assert len(values) == 1
+    assert isinstance(values[0], blosc2.CTable)
+
+
+@pytest.mark.parametrize("storage_type", ["b2d", "b2z"])
+def test_ctable_delete_parent_subtree_removes_nested_object(tmp_path, storage_type):
+    """Deleting a normal subtree also deletes nested object roots and physical leaves."""
+    path = str(tmp_path / f"bundle.{storage_type}")
+    with blosc2.TreeStore(path, mode="w") as ts:
+        ts["/grp/table"] = _make_ctable(n=2)
+        del ts["/grp"]
+        assert sorted(ts.keys()) == []
+
+    with blosc2.open(path, mode="r") as ts:
+        assert sorted(ts.keys()) == []
+        assert "/grp/table" not in ts
+
+
+@pytest.mark.parametrize("storage_type", ["b2d", "b2z"])
+def test_ctable_inline_index_roundtrip(tmp_path, storage_type):
+    """Index catalogs and sidecars work for inline CTable objects."""
+    path = str(tmp_path / f"bundle.{storage_type}")
+    with blosc2.TreeStore(path, mode="w") as ts:
+        ts["/table"] = _make_ctable(n=100)
+
+    with blosc2.TreeStore(path, mode="a") as ts:
+        table = ts["/table"]
+        table.create_index("x")
+        np.testing.assert_array_equal(list(table.where(table["x"] > 95)["x"][:]), [96, 97, 98, 99])
+
+    with blosc2.open(path, mode="r") as ts:
+        table = ts["/table"]
+        assert len(table.indexes) == 1
+        np.testing.assert_array_equal(list(table.where(table["x"] > 95)["x"][:]), [96, 97, 98, 99])
+
+
+@pytest.mark.parametrize("storage_type", ["b2d", "b2z"])
+def test_ctable_registry_missing_fallback_hides_and_protects_internals(tmp_path, storage_type):
+    """Physical CTable manifests are enough to detect object roots if registry is missing."""
+    path = str(tmp_path / f"bundle.{storage_type}")
+    with blosc2.TreeStore(path, mode="w") as ts:
+        ts["/table"] = _make_ctable(n=2)
+
+    with blosc2.TreeStore(path, mode="a") as ts:
+        del ts._estore._store.vlmeta["_object_registry"]
+
+    with blosc2.open(path, mode="r") as ts:
+        assert sorted(ts.keys()) == ["/table"]
+        assert isinstance(ts["/table"], blosc2.CTable)
+        assert "/table/_meta" not in ts
+        with pytest.raises(ValueError, match="object root"):
+            ts.get_subtree("/table")