@@ -341,20 +341,27 @@ def _load_fetched(self, nchunks: int) -> bytearray:
341341 # A cache filled before the bitmap existed, or one handed over through
342342 # `_cache=`: everything that is not a special chunk was surely fetched
343343 fetched = bytearray ((nchunks + 7 ) // 8 )
344+ self ._fetched = fetched
344345 for info in self ._schunk_cache .iterchunks_info ():
345346 if info .special == blosc2 .SpecialValue .NOT_SPECIAL :
346- fetched [ info . nchunk // 8 ] |= 1 << (info .nchunk % 8 )
347+ self . _mark_fetched (info .nchunk )
347348 return fetched
348349
350+ def _mark_fetched (self , nchunk : int ) -> None :
351+ self ._fetched [nchunk // 8 ] |= 1 << (nchunk % 8 )
352+
349353 def _missing_chunks (self , item ) -> list [int ]:
350354 """The chunks *item* touches, minus those already in the cache."""
351355 nchunks = self ._schunk_cache .nchunks
352356 # Full realization when item is (), else only the chunks it intersects
353- wanted = range (nchunks ) if item == () else sorted ( set ( blosc2 .get_slice_nchunks (self ._cache , item ) ))
357+ wanted = range (nchunks ) if item == () else list ( blosc2 .get_slice_nchunks (self ._cache , item ))
354358 return [int (n ) for n in wanted if not self ._fetched [n // 8 ] >> (n % 8 ) & 1 ]
355359
356360 def _save_fetched (self ) -> None :
357- """Persist the bitmap, so a later run does not fetch these chunks again."""
361+ """Persist the bitmap, so a later run does not fetch these chunks again.
362+
363+ Called even when a fetch failed partway: whatever did arrive is kept.
364+ """
358365 self ._schunk_cache .vlmeta ["proxy-fetched" ] = bytes (self ._fetched )
359366
360367 def _reopen_cache (self , urlpath : str ):
@@ -459,9 +466,8 @@ def fetch(
459466 try :
460467 for nchunk , chunk in self ._get_chunks (missing , max_concurrency ):
461468 self ._schunk_cache .update_chunk (nchunk , chunk )
462- self ._fetched [ nchunk // 8 ] |= 1 << (nchunk % 8 )
469+ self ._mark_fetched (nchunk )
463470 finally :
464- # Keep hold of whatever did arrive, even if a later chunk blew up
465471 if missing :
466472 self ._save_fetched ()
467473
@@ -586,13 +592,12 @@ async def _fetch_one(nchunk):
586592 chunk = await self .src .aget_chunk (nchunk )
587593 # Runs to completion between awaits, so concurrent writers can't interleave.
588594 self ._schunk_cache .update_chunk (nchunk , chunk )
589- self ._fetched [ nchunk // 8 ] |= 1 << (nchunk % 8 )
595+ self ._mark_fetched (nchunk )
590596
591597 if to_fetch :
592598 try :
593599 await asyncio .gather (* (_fetch_one (nchunk ) for nchunk in to_fetch ))
594600 finally :
595- # Keep hold of whatever did arrive, even if a later chunk blew up
596601 self ._save_fetched ()
597602
598603 return self ._cache
0 commit comments