Skip to content

Commit 82a971a

Browse files
committed
Trim the fsspec proxy plumbing
One _mark_fetched helper for the three bitmap set-bits, the partial- failure rationale lives in _save_fetched's docstring instead of two comments, and get_slice_nchunks is already unique and ordered so the set-and-sort in _missing_chunks goes.
1 parent 2b3a112 commit 82a971a

2 files changed

Lines changed: 14 additions & 9 deletions

File tree

src/blosc2/core.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
import urllib.parse
2626
import urllib.request
2727
from dataclasses import asdict
28-
from functools import lru_cache
28+
from functools import cache, lru_cache
2929
from typing import TYPE_CHECKING, ClassVar
3030

3131
import numpy as np
@@ -687,7 +687,7 @@ def fsspec_cache_path(urlpath: str, cache_storage: str | pathlib.Path, suffix: s
687687
return os.path.join(str(cache_storage), name + suffix)
688688

689689

690-
@lru_cache(maxsize=1)
690+
@cache
691691
def _suffixed_cache_mapper():
692692
"""fsspec's cache naming, plus the extension `blosc2.open()` dispatches on.
693693

src/blosc2/proxy.py

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -341,20 +341,27 @@ def _load_fetched(self, nchunks: int) -> bytearray:
341341
# A cache filled before the bitmap existed, or one handed over through
342342
# `_cache=`: everything that is not a special chunk was surely fetched
343343
fetched = bytearray((nchunks + 7) // 8)
344+
self._fetched = fetched
344345
for info in self._schunk_cache.iterchunks_info():
345346
if info.special == blosc2.SpecialValue.NOT_SPECIAL:
346-
fetched[info.nchunk // 8] |= 1 << (info.nchunk % 8)
347+
self._mark_fetched(info.nchunk)
347348
return fetched
348349

350+
def _mark_fetched(self, nchunk: int) -> None:
351+
self._fetched[nchunk // 8] |= 1 << (nchunk % 8)
352+
349353
def _missing_chunks(self, item) -> list[int]:
350354
"""The chunks *item* touches, minus those already in the cache."""
351355
nchunks = self._schunk_cache.nchunks
352356
# Full realization when item is (), else only the chunks it intersects
353-
wanted = range(nchunks) if item == () else sorted(set(blosc2.get_slice_nchunks(self._cache, item)))
357+
wanted = range(nchunks) if item == () else list(blosc2.get_slice_nchunks(self._cache, item))
354358
return [int(n) for n in wanted if not self._fetched[n // 8] >> (n % 8) & 1]
355359

356360
def _save_fetched(self) -> None:
357-
"""Persist the bitmap, so a later run does not fetch these chunks again."""
361+
"""Persist the bitmap, so a later run does not fetch these chunks again.
362+
363+
Called even when a fetch failed partway: whatever did arrive is kept.
364+
"""
358365
self._schunk_cache.vlmeta["proxy-fetched"] = bytes(self._fetched)
359366

360367
def _reopen_cache(self, urlpath: str):
@@ -459,9 +466,8 @@ def fetch(
459466
try:
460467
for nchunk, chunk in self._get_chunks(missing, max_concurrency):
461468
self._schunk_cache.update_chunk(nchunk, chunk)
462-
self._fetched[nchunk // 8] |= 1 << (nchunk % 8)
469+
self._mark_fetched(nchunk)
463470
finally:
464-
# Keep hold of whatever did arrive, even if a later chunk blew up
465471
if missing:
466472
self._save_fetched()
467473

@@ -586,13 +592,12 @@ async def _fetch_one(nchunk):
586592
chunk = await self.src.aget_chunk(nchunk)
587593
# Runs to completion between awaits, so concurrent writers can't interleave.
588594
self._schunk_cache.update_chunk(nchunk, chunk)
589-
self._fetched[nchunk // 8] |= 1 << (nchunk % 8)
595+
self._mark_fetched(nchunk)
590596

591597
if to_fetch:
592598
try:
593599
await asyncio.gather(*(_fetch_one(nchunk) for nchunk in to_fetch))
594600
finally:
595-
# Keep hold of whatever did arrive, even if a later chunk blew up
596601
self._save_fetched()
597602

598603
return self._cache

0 commit comments

Comments
 (0)