Skip to content

Commit bf87015

Browse files
FrancescAltedclaude
andcommitted
Check the source identity when a Proxy adopts a cache
Same shape, dtype and partitioning is not the same bytes: a replaced remote frame keeps its layout while every cached chunk, and every offset it was fetched by, goes stale. blosc2.open() already refetched in that case, but the hand-built form the FsspecNDSource docstring recommends went straight to the Proxy and skipped the check. So the Proxy stamps its cache with whatever identity the source can name itself by, and refuses one built against other bytes. Sources that have no identity to give are still adopted on geometry alone, as documented. This also takes over the stamping _lazy_fsspec_proxy() was doing by hand. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
1 parent 5aec265 commit bf87015

3 files changed

Lines changed: 37 additions & 1 deletion

File tree

src/blosc2/proxy.py

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -307,6 +307,11 @@ def __init__(
307307
self._schunk_cache = getattr(self._cache, "schunk", self._cache)
308308
if self.urlpath is None:
309309
self.urlpath = getattr(self._schunk_cache, "urlpath", None)
310+
# Geometry alone cannot tell a replaced source from the one the cache was
311+
# filled from, so record whatever identity the source can name itself by
312+
stamp = getattr(self.src, "stamp", None)
313+
if stamp is not None:
314+
self._schunk_cache.vlmeta["fsspec-stamp"] = stamp
310315
if vlmeta:
311316
for key in vlmeta:
312317
self._schunk_cache.vlmeta[key] = vlmeta[key]
@@ -357,6 +362,16 @@ def _reopen_cache(self, urlpath: str):
357362
f"the cache at {urlpath} was built for a different source: it holds {here}, "
358363
f"the source is {there} ({fields})"
359364
)
365+
# Same geometry is not the same bytes: a replaced remote frame keeps its
366+
# layout while every cached chunk, and every offset it was fetched by,
367+
# goes stale. Only for sources that can name themselves; the rest are
368+
# adopted on geometry alone, as documented.
369+
stamp = getattr(self.src, "stamp", None)
370+
if stamp is not None and schunk.vlmeta.get("fsspec-stamp") != stamp:
371+
raise ValueError(
372+
f"the cache at {urlpath} was built against different remote bytes; "
373+
f"pass mode='w' to fetch them anew"
374+
)
360375
return cached
361376

362377
def __exit__(self, exc_type, exc_val, exc_tb) -> bool:

src/blosc2/schunk.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1964,7 +1964,9 @@ def _lazy_fsspec_proxy(
19641964
# The remote frame was replaced, which makes every cached chunk -- and
19651965
# every offset they were fetched by -- meaningless
19661966
blosc2.remove_urlpath(path)
1967-
return blosc2.Proxy(src, urlpath=path, mode="a", vlmeta={"fsspec-stamp": src.stamp})
1967+
# Proxy stamps the cache with src.stamp itself, and refuses one built against
1968+
# other bytes; removing it above is what turns that refusal into a refetch
1969+
return blosc2.Proxy(src, urlpath=path, mode="a")
19681970

19691971

19701972
def _cache_stamp(path: str):

tests/test_fsspec.py

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -660,3 +660,22 @@ def test_lazy_open_never_opens_a_handle(monkeypatch):
660660

661661
b = blosc2.open("memory://ranges.b2nd", lazy=True)
662662
assert np.array_equal(b[100:200], a[100:200])
663+
664+
665+
def test_handbuilt_proxy_rejects_a_stale_cache(tmp_path):
666+
# The FsspecNDSource docstring recommends wrapping it in a Proxy by hand, which
667+
# bypasses the refetch blosc2.open() does; same geometry is not the same bytes
668+
memfs = fsspec.filesystem("memory")
669+
cache = str(tmp_path / "hand.b2nd")
670+
memfs.pipe_file("/hand.b2nd", blosc2.arange(100, dtype="i4", chunks=(10,)).to_cframe())
671+
p = blosc2.Proxy(blosc2.FsspecNDSource("memory://hand.b2nd"), urlpath=cache, mode="a")
672+
assert np.array_equal(p[:10], np.arange(10, dtype="i4"))
673+
del p
674+
675+
other = blosc2.arange(100, 200, dtype="i4", chunks=(10,))
676+
memfs.pipe_file("/hand.b2nd", other.to_cframe())
677+
with pytest.raises(ValueError, match="different remote bytes"):
678+
blosc2.Proxy(blosc2.FsspecNDSource("memory://hand.b2nd"), urlpath=cache, mode="a")
679+
680+
p = blosc2.Proxy(blosc2.FsspecNDSource("memory://hand.b2nd"), urlpath=cache, mode="w")
681+
assert np.array_equal(p[:], other[:])

0 commit comments

Comments
 (0)