Fix six defects found reviewing the frame reader

All reproduced first; the first two make lazy=True unusable for whole
classes of ordinary arrays.

The frame header was unpacked with raw=False.  Its flags field is a msgpack
*string* holding four raw bytes, and clevel rides in the high nibble of one
of them, so from clevel=8 up it is not valid UTF-8 and every lazy open died
with UnicodeDecodeError.  Unpacked raw now.

_special_chunk rebuilt run-length chunks with compress2 and no blocksize,
which makes blosc2 take the whole chunk.  Whenever blocks != chunks -- the
default for a large chunk -- the cache rejected the chunk with "Error while
getting the buffer", and with cache_storage= the bad chunk was written to
disk.  It now passes the container's blocksize.

Structured dtypes are stored as their repr, so np.dtype() on the metalayer
string raised TypeError; added the ast.literal_eval fallback blosc2_ext
already uses.  _reopen_cache dereferenced cached.shape before it could
report a kind mismatch, raising AttributeError instead of the intended
ValueError.  normalize_urlpath dropped the drive letter for file://C:/x,
where urlparse puts it in netloc.  And open()'s Notes listed .b2z among the
formats a plain URL read handles: it is a zip archive, not a cframe, so it
needs cache_storage like the directory formats.

Tests now parametrise over clevel, over blocks != chunks and over a
structured dtype, since every one of these hid behind default parameters.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Author: FrancescAlted

plans/fsspec-support.md

@@ -103,6 +103,16 @@ The minimum that is genuinely useful.
 - `.b2d` raises `NotImplementedError`; sparse frames are not detected up front
   and fail on the `from_cframe` instead. Both messages now point at phase 2's
   `cache_storage=`, which is the actual fix.
+- What the frame parser got wrong, found in review rather than by tests, because
+  every test had used default parameters: the header must be unpacked with
+  `raw=True` (the flags field is a msgpack *string* of raw bytes, and `clevel`
+  rides in the high nibble of one of them, so from `clevel=8` up it is not valid
+  UTF-8 and `lazy=True` raised `UnicodeDecodeError`); structured dtypes are
+  stored as their `repr` and need the same `ast.literal_eval` fallback
+  `blosc2_ext` uses; and a rebuilt run-length chunk must carry the container's
+  blocksize, since `compress2` left to itself takes the whole chunk and the
+  cache then rejects the chunk. Parametrising the tests over `clevel`, over
+  `blocks != chunks` and over a structured dtype is what pins these.
 - Tests: `tests/test_fsspec.py`, 12 tests over `memory://` plus one chained
   `zip://…::file://` URL, in the default suite behind `importorskip("fsspec")`.
   No tier-2 network test, per the open question below.

src/blosc2/core.py

@@ -625,7 +625,10 @@ def normalize_urlpath(urlpath: object) -> object:
     container format, which only works if the scheme is stripped first.
     """
     if isinstance(urlpath, str) and urlpath.startswith("file://"):
-        return urllib.request.url2pathname(urllib.parse.urlparse(urlpath).path)
+        parsed = urllib.parse.urlparse(urlpath)
+        # A Windows drive lands in netloc for the two-slash form, `file://C:/x`
+        prefix = parsed.netloc if parsed.netloc.lower() not in ("", "localhost") else ""
+        return urllib.request.url2pathname(prefix + parsed.path)
     return urlpath
 
 

src/blosc2/proxy.py

@@ -325,6 +325,13 @@ def _reopen_cache(self, urlpath: str):
         # Chunk *numbers* are the currency between cache and source, so the
         # partitioning has to match, not just the logical shape: fetch() would
         # otherwise ask the source for chunk n meaning something else entirely
+        # A cache of the other kind is a mismatch in itself, and asking it for a
+        # shape it does not have would raise AttributeError instead of saying so
+        if hasattr(self.src, "shape") != hasattr(cached, "shape"):
+            raise ValueError(
+                f"the cache at {urlpath} is a {type(cached).__name__}, which does not fit a "
+                f"{type(self.src).__name__} source"
+            )
         if hasattr(self.src, "shape"):
             here = (tuple(cached.shape), cached.dtype, tuple(cached.chunks), tuple(cached.blocks))
             there = (
@@ -680,7 +687,10 @@ def _read_frame_index(f) -> tuple[bytes, list, np.ndarray]:
     header_len = struct.unpack(">i", prefix[11:15])[0]
     f.seek(0)
     raw = f.read(header_len)
-    header = msgpack.unpackb(raw, raw=False, strict_map_key=False)
+    # raw=True because the flags field is a msgpack *string* holding four raw
+    # bytes, and codec_flags packs clevel into its high nibble: from clevel 8 up
+    # that byte is not valid UTF-8 and decoding the header blows up
+    header = msgpack.unpackb(raw, raw=True, strict_map_key=False)
 
     # The offsets live in a Blosc2 chunk of their own, right after the data ones
     index_pos = header[1] + header[5]
@@ -694,7 +704,7 @@ def _read_frame_index(f) -> tuple[bytes, list, np.ndarray]:
 
 def _frame_metalayer(raw: bytes, header: list, name: str):
     """Decode the *name* metalayer out of an already-read frame header."""
-    offset = header[13][1][name]  # KeyError if the frame has no such metalayer
+    offset = header[13][1][name.encode()]  # KeyError if there is no such metalayer
     nbytes = struct.unpack(">I", raw[offset + 1 : offset + 5])[0]  # msgpack bin32
     import msgpack
 
@@ -779,7 +789,11 @@ def __init__(self, urlpath: str, max_concurrency: int = REMOTE_MAX_CONCURRENCY):
         if dtype_format != 0:
             raise NotImplementedError(f"unsupported dtype format {dtype_format} in {urlpath}")
         self._shape, self._chunks, self._blocks = tuple(shape), tuple(chunks), tuple(blocks)
-        self._dtype = np.dtype(dtype)
+        try:
+            self._dtype = np.dtype(dtype)
+        except TypeError:
+            # Structured dtypes are stored as their repr, as blosc2_ext does too
+            self._dtype = np.dtype(ast.literal_eval(dtype))
 
     @property
     def shape(self) -> tuple:
@@ -835,7 +849,13 @@ def _special_chunk(self, offset: int) -> bytes:
             # A run of zeros (1); uninitialized chunks (4) have no defined
             # content, and zeros is what reading them locally hands back too
             data = np.zeros(nitems, dtype=self._dtype)
-        return blosc2.compress2(data, typesize=self._dtype.itemsize)
+        # The blocksize has to be the container's: left to choose, blosc2 takes
+        # the whole chunk, and the cache then rejects the chunk we hand it
+        return blosc2.compress2(
+            data,
+            typesize=self._dtype.itemsize,
+            blocksize=int(np.prod(self._blocks)) * self._dtype.itemsize,
+        )
 
 
 class ProxyNDField(blosc2.Operand):

src/blosc2/schunk.py

@@ -2124,9 +2124,11 @@ def open(
       the driver for the protocol (``s3fs``, ``gcsfs``...), which fsspec asks for
       by name when it is missing; credentials are configured there, not here.
       ``mode != 'r'`` always raises, as object stores have no rename and no locks.
-      A plain URL read holds the whole object in memory, so it covers single-file
-      containers (``.b2nd``, ``.b2f``, ``.b2e``, ``.b2z``) only; ``cache_storage``
-      and ``lazy`` above lift that, each in its own way.
+      A plain URL read rebuilds the object from a cframe held in memory, so it
+      covers ``.b2nd``, ``.b2f`` and ``.b2e`` only -- a ``.b2z`` store is a zip
+      archive rather than a cframe, and needs ``cache_storage`` like the
+      directory formats do.  ``cache_storage`` and ``lazy`` above lift that, each
+      in its own way.
 
     * Persistent data handling follows a strict no-hidden-writes rule:
 

tests/ndarray/test_proxy.py

@@ -146,6 +146,20 @@ def test_reuse_cache_across_runs(tmp_path):
     np.testing.assert_array_equal(proxy[:], data)
 
 
+def test_reuse_cache_rejects_other_kind(tmp_path):
+    proxy_path = str(tmp_path / "proxy.b2f")
+
+    class Source(blosc2.ProxySource):
+        nbytes, chunksize, typesize = 1000, 100, 1
+
+        def get_chunk(self, nchunk):
+            raise NotImplementedError
+
+    blosc2.Proxy(Source(), urlpath=proxy_path, mode="a")
+    with pytest.raises(ValueError, match="does not fit"):
+        blosc2.Proxy(blosc2.asarray(np.arange(1000, dtype=np.int32)), urlpath=proxy_path, mode="a")
+
+
 def test_reuse_cache_rejects_foreign_container(tmp_path):
     path = str(tmp_path / "plain.b2nd")
     blosc2.arange(0, 120, dtype=np.int32, shape=(12, 10), urlpath=path, mode="w")

tests/test_fsspec.py

@@ -247,13 +247,38 @@ def test_lazy_multidim():
     [
         blosc2.zeros((1000,), dtype="f8", chunks=(100,)),
         blosc2.full((1000,), np.nan, dtype="f8", chunks=(100,)),
+        # blocks != chunks: a rebuilt run-length chunk must carry the container's
+        # blocksize, not whatever blosc2 picks when left to choose
+        blosc2.zeros((1000,), dtype="f8", chunks=(100,), blocks=(10,)),
+        blosc2.zeros((4_000_000,), dtype="f8", chunks=(1_000_000,)),
+        blosc2.uninit((1000,), dtype="i4", chunks=(100,), blocks=(10,)),
     ],
-    ids=["zeros", "nan"],
+    ids=["zeros", "nan", "small-blocks", "auto-blocks", "uninit"],
 )
 def test_lazy_special_chunks(arr):
     # Run-length chunks live in the offset itself, with no bytes in the file
     p = blosc2.open(_put("special.b2nd", arr), lazy=True)
-    assert np.allclose(p[:], arr[:], equal_nan=True)
+    assert p[:].shape == arr.shape
+    if arr.dtype.kind == "f":
+        assert np.allclose(p[:], arr[:], equal_nan=True)
+
+
+@pytest.mark.parametrize("clevel", [1, 5, 8, 9])
+def test_lazy_any_clevel(clevel):
+    # The frame header's flags are a msgpack *string* of raw bytes, and clevel
+    # rides in the high nibble of one of them: from 8 up it is not valid UTF-8
+    a = blosc2.arange(0, 10000, dtype="i4", chunks=(1000,), cparams={"clevel": clevel})
+    p = blosc2.open(_put(f"clevel{clevel}.b2nd", a), lazy=True)
+    assert np.array_equal(p[:], a[:])
+
+
+def test_lazy_structured_dtype():
+    data = np.zeros(1000, dtype=[("a", "<i4"), ("b", "<f8")])
+    data["a"] = np.arange(1000)
+    a = blosc2.asarray(data, chunks=(100,), blocks=(10,))
+    p = blosc2.open(_put("struct.b2nd", a), lazy=True)
+    assert p.dtype == data.dtype
+    assert np.array_equal(p[150:250], data[150:250])
 
 
 def test_lazy_one_request_per_chunk(monkeypatch):
@@ -480,6 +505,33 @@ def test_http_does_not_reach_fsspec():
         blosc2.open("http://localhost:1/foo.b2nd")
 
 
+def test_zip_store_needs_cache(tmp_path):
+    # A .b2z store is a zip archive, not a cframe, so there is nothing for the
+    # in-memory read to rebuild
+    localpath = str(tmp_path / "t.b2z")
+    with blosc2.TreeStore(localpath, mode="w") as tstore:
+        tstore["/a"] = blosc2.arange(10, dtype="i4")
+    fsspec.filesystem("memory").pipe_file("/t.b2z", pathlib.Path(localpath).read_bytes())
+
+    with pytest.raises(RuntimeError):
+        blosc2.open("memory://t.b2z")
+    with blosc2.open("memory://t.b2z", cache_storage=tmp_path / "cache") as tstore:
+        assert np.array_equal(tstore["/a"][:], np.arange(10, dtype="i4"))
+
+
+@pytest.mark.parametrize(
+    ("url", "expected"),
+    [
+        ("file:///tmp/a.b2nd", "/tmp/a.b2nd"),
+        ("file://localhost/tmp/a.b2nd", "/tmp/a.b2nd"),
+        # A Windows drive lands in the netloc for the two-slash form
+        ("file://C:/data/a.b2nd", "C:"),
+    ],
+)
+def test_normalize_file_url(url, expected):
+    assert expected in blosc2.core.normalize_urlpath(url)
+
+
 def test_file_url_uses_the_local_path(tmp_path):
     # file:// is kept off the fsspec branch so it can use mmap and every
     # container format, which only works if the scheme is stripped first