Merge branch 'main' into fill_value_default

Author: d-v-b

changes/2778.bugfix.rst

@@ -0,0 +1 @@
+Use removeprefix rather than replace when removing filename prefixes in `FsspecStore.list`

changes/2801.bugfix.rst

@@ -0,0 +1 @@
+Ensure utf8 compliant strings are used to construct numpy arrays in property-based tests

src/zarr/storage/_fsspec.py

@@ -341,7 +341,7 @@ async def set_partial_values(
     async def list(self) -> AsyncIterator[str]:
         # docstring inherited
         allfiles = await self.fs._find(self.path, detail=False, withdirs=False)
-        for onefile in (a.replace(self.path + "/", "") for a in allfiles):
+        for onefile in (a.removeprefix(self.path + "/") for a in allfiles):
             yield onefile
 
     async def list_dir(self, prefix: str) -> AsyncIterator[str]:

src/zarr/testing/store.py

@@ -400,6 +400,37 @@ async def test_list_prefix(self, store: S) -> None:
             expected = tuple(sorted(expected))
             assert observed == expected
 
+    async def test_list_empty_path(self, store: S) -> None:
+        """
+        Verify that list and list_prefix work correctly when path is an empty string,
+        i.e. no unwanted replacement occurs.
+        """
+        data = self.buffer_cls.from_bytes(b"")
+        store_dict = {
+            "foo/bar/zarr.json": data,
+            "foo/bar/c/1": data,
+            "foo/baz/c/0": data,
+        }
+        await store._set_many(store_dict.items())
+
+        # Test list()
+        observed_list = await _collect_aiterator(store.list())
+        observed_list_sorted = sorted(observed_list)
+        expected_list_sorted = sorted(store_dict.keys())
+        assert observed_list_sorted == expected_list_sorted
+
+        # Test list_prefix() with an empty prefix
+        observed_prefix_empty = await _collect_aiterator(store.list_prefix(""))
+        observed_prefix_empty_sorted = sorted(observed_prefix_empty)
+        expected_prefix_empty_sorted = sorted(store_dict.keys())
+        assert observed_prefix_empty_sorted == expected_prefix_empty_sorted
+
+        # Test list_prefix() with a non-empty prefix
+        observed_prefix = await _collect_aiterator(store.list_prefix("foo/bar/"))
+        observed_prefix_sorted = sorted(observed_prefix)
+        expected_prefix_sorted = sorted(k for k in store_dict if k.startswith("foo/bar/"))
+        assert observed_prefix_sorted == expected_prefix_sorted
+
     async def test_list_dir(self, store: S) -> None:
         root = "foo"
         store_dict = {

src/zarr/testing/strategies.py

@@ -51,6 +51,21 @@ def v2_dtypes() -> st.SearchStrategy[np.dtype]:
     )
 
 
+def safe_unicode_for_dtype(dtype: np.dtype[np.str_]) -> st.SearchStrategy[str]:
+    """Generate UTF-8-safe text constrained to max_len of dtype."""
+    # account for utf-32 encoding (i.e. 4 bytes/character)
+    max_len = max(1, dtype.itemsize // 4)
+
+    return st.text(
+        alphabet=st.characters(
+            blacklist_categories=["Cs"],  # Avoid *technically allowed* surrogates
+            min_codepoint=32,
+        ),
+        min_size=1,
+        max_size=max_len,
+    )
+
+
 # From https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html#node-names
 # 1. must not be the empty string ("")
 # 2. must not include the character "/"
@@ -86,7 +101,12 @@ def numpy_arrays(
     Generate numpy arrays that can be saved in the provided Zarr format.
     """
     zarr_format = draw(zarr_formats)
-    return draw(npst.arrays(dtype=v3_dtypes() if zarr_format == 3 else v2_dtypes(), shape=shapes))
+    dtype = draw(v3_dtypes() if zarr_format == 3 else v2_dtypes())
+    if np.issubdtype(dtype, np.str_):
+        safe_unicode_strings = safe_unicode_for_dtype(dtype)
+        return draw(npst.arrays(dtype=dtype, shape=shapes, elements=safe_unicode_strings))
+
+    return draw(npst.arrays(dtype=dtype, shape=shapes))
 
 
 @st.composite  # type: ignore[misc]