Add tests for arrays of strings

Author: lshaw8317

bench/ndarray/stringops_bench.py

@@ -18,12 +18,12 @@
 # nparr = nparr.view('S40').astype('U10')
 
 N = int(1e5)
-nparr_ = np.array(['josé', 'pepe', 'francisco'])
-nparr = np.repeat(nparr_, N)
+nparr = np.repeat(np.array(['josé', 'pepe', 'francisco']), N)
 cparams = blosc2.cparams_dflts
+cparams["filters"][-1] = blosc2.Filter.SHUFFLE
+cparams["filters_meta"][-1] = 0 # use default (typesize)
 arr1 = blosc2.asarray(nparr, cparams=cparams)
 print(f"cratio without filter: {arr1.cratio}")
-cparams["filters"][-1] = blosc2.Filter.SHUFFLE
 cparams["filters_meta"][-1] = 4
 arr1 = blosc2.asarray(nparr, cparams=cparams)
 print(f"cratio with filter: {arr1.cratio}")

tests/ndarray/test_full.py

@@ -38,6 +38,17 @@
             "full.b2nd",
             True,
         ),
+        (
+            (23, 34),
+            (20, 20),
+            (10, 10),
+            "josé",
+            None,
+            blosc2.CParams(codec=blosc2.Codec.LZ4HC, clevel=8, use_dict=False, nthreads=2),
+            {"nthreads": 2},
+            "full.b2nd",
+            True,
+        ),
         (
             (80, 51, 60),
             (20, 10, 33),
@@ -95,6 +106,7 @@ def test_full(shape, chunks, blocks, fill_value, cparams, dparams, dtype, urlpat
     [
         ((100, 1230), b"0123", None),
         ((23, 34), b"sun", None),
+        ((23, 34), "josé", None),
         ((80, 51, 60), 3.14, "f8"),
         ((13, 13), 123456789, None),
     ],
@@ -161,6 +173,7 @@ def test_complex_datatype():
             ("f_019", "<f4", (27, 363)),
             ("f_020", "S1000"),
             ("f_021", "S1000"),
+            ("f_022", "<U1000"),
         ]
     )
     a = np.zeros((256,), dtype=dtype)

tests/ndarray/test_lazyexpr.py

@@ -540,41 +540,22 @@ def test_stringops(values):
     if value1 == "NDArray":
         a1 = np.array(["abc", "def", "aterr", "oot", "zu", "ab c"])
         a1_blosc = blosc2.asarray(a1)
-        if value2 == "str":  # ("NDArray", "str")
-            value2 = "a"
-            expr_lazy = blosc2.startswith(a1_blosc, value2)
-            assert expr_lazy.shape == a1_blosc.shape
-            res_numexpr = np.char.startswith(a1, value2)
-            np.testing.assert_array_equal(expr_lazy[:], res_numexpr)
-            value2 = "c"
-            expr_lazy = blosc2.endswith(a1_blosc, value2)
-            res_numexpr = np.char.endswith(a1, value2)
-            np.testing.assert_array_equal(expr_lazy[:], res_numexpr)
-        else:  # ("NDArray", "NDArray")
-            a2 = np.array(["ba", "ef", "rr", "o ", "\tz", "c h"])
-            a2_blosc = blosc2.asarray(a2)
-            expr_lazy = blosc2.startswith(a1_blosc, a2_blosc)
-            assert expr_lazy.shape == a1_blosc.shape
-            res_numexpr = np.char.startswith(a1, a2)
-            np.testing.assert_array_equal(expr_lazy[:], res_numexpr)
-            a2 = np.array(["ab", "d", " ath", "oo", "\tabc", " c"])
-            a2_blosc = blosc2.asarray(a2)
-            expr_lazy = blosc2.endswith(a1, a2_blosc)
-            res_numexpr = np.char.endswith(a1, a2)
-            np.testing.assert_array_equal(expr_lazy[:], res_numexpr)
-    else:  # ("str", "NDArray")
-        value1 = "abc"
-        a2 = np.array(["ab", "def", "a", "oot", "zu", "ab "])
+    else:  # ("str", _)
+        a1 = a1_blosc = "abc"
+
+    if value2 == "str":  # (_, "str")
+        a2 = a2_blosc = "a"
+    else:  # (_, "NDArray")
+        a2 = np.array(["ba", "ef", "rr", "o ", "\tz", "c h"])
         a2_blosc = blosc2.asarray(a2)
-        expr_lazy = blosc2.startswith(value1, a2_blosc)
-        assert expr_lazy.shape == a2_blosc.shape
-        assert expr_lazy.dtype == blosc2.bool_
-        res_numexpr = np.char.startswith(value1, a2)
-        np.testing.assert_array_equal(expr_lazy[:], res_numexpr)
 
-        value1 = "b"
-        expr_lazy = blosc2.endswith(value1, a2_blosc)
-        res_numexpr = np.char.endswith(value1, a2)
+    for func, npfunc in zip(
+        (blosc2.startswith, blosc2.endswith), (np.char.startswith, np.char.endswith), strict=True
+    ):
+        expr_lazy = func(a1_blosc, a2_blosc)
+        res_numexpr = npfunc(a1, a2)
+        assert expr_lazy.shape == res_numexpr.shape
+        assert expr_lazy.dtype == blosc2.bool_
         np.testing.assert_array_equal(expr_lazy[:], res_numexpr)
 
 

tests/ndarray/test_stringarrays.py

@@ -0,0 +1,171 @@
+import numpy as np
+import pytest
+
+import blosc2
+
+# ----------------------------------
+# Helpers
+# ----------------------------------
+
+UNICODE_VALUES = [
+    "café",
+    "β",
+    "こんにちは",
+    "mañana",
+    "добрый",
+    "数据",
+]
+
+
+def make_unicode_array(shape, maxlen=16):
+    """
+    Create a NumPy Unicode array with non-ASCII content.
+    dtype='U' uses fixed-width UTF-32 internally.
+    """
+    total = np.prod(shape)
+    data = [UNICODE_VALUES[i % len(UNICODE_VALUES)] for i in range(total)]
+    return np.array(data, dtype=f"U{maxlen}").reshape(shape)
+
+
+# ----------------------------------
+# Parameter grids
+# ----------------------------------
+
+SHAPES = [
+    (12,),
+    (12, 6),
+    (3, 4, 5),
+]
+
+CHUNKS = [
+    None,
+    (4,),
+    (7, 4),
+    (2, 2, 5),
+]
+
+BLOCKS = [
+    None,
+    (3,),
+    (3, 3),
+    (1, 2, 5),
+]
+
+
+# ----------------------------------
+# In-memory tests
+# ----------------------------------
+
+
+@pytest.mark.parametrize("shape", SHAPES)
+@pytest.mark.parametrize("chunks", CHUNKS)
+@pytest.mark.parametrize("blocks", BLOCKS)
+def test_unicode_roundtrip_in_memory(shape, chunks, blocks):
+    arr = make_unicode_array(shape)
+
+    b2 = blosc2.asarray(
+        arr,
+        chunks=chunks if chunks and len(chunks) == arr.ndim else None,
+        blocks=blocks if blocks and len(blocks) == arr.ndim else None,
+    )
+
+    assert b2.dtype == arr.dtype
+    assert np.array_equal(b2, arr)
+
+
+def test_unicode_indexing_and_slicing():
+    arr = make_unicode_array((10,))
+    b2 = blosc2.asarray(arr, chunks=(6,), blocks=(4,))
+
+    assert b2[0] == arr[0]
+    assert b2[5] == arr[5]
+    assert np.array_equal(b2[2:8], arr[2:8])
+    assert np.array_equal(b2[::2], arr[::2])
+
+
+def test_unicode_multidimensional_slice():
+    arr = make_unicode_array((6, 8))
+    b2 = blosc2.asarray(arr, chunks=(3, 4), blocks=(1, 4))
+
+    assert np.array_equal(
+        b2[1:5, 2:7],
+        arr[1:5, 2:7],
+    )
+
+
+def test_unicode_partial_assignment():
+    arr = make_unicode_array((10,))
+    b2 = blosc2.asarray(arr)
+
+    new_vals = np.array(["Ω", "λ", "plo"], dtype=arr.dtype)
+    b2[3:6] = new_vals
+    arr[3:6] = new_vals
+
+    assert np.array_equal(b2, arr)
+
+
+# ----------------------------------
+# On-disk tests
+# ----------------------------------
+
+
+@pytest.mark.parametrize("shape", SHAPES)
+def test_unicode_roundtrip_on_disk(tmp_path, shape):
+    arr = make_unicode_array(shape)
+
+    path = tmp_path / "unicode_array.b2nd"
+
+    b2 = blosc2.asarray(
+        arr,
+        urlpath=path,
+        mode="w",
+        chunks=tuple(max(1, s // 2) for s in shape),
+        blocks=tuple(1 for _ in shape),
+    )
+
+    # Re-open from disk
+    out = blosc2.open(path)
+
+    assert out.dtype == arr.dtype
+    assert np.array_equal(out, arr)
+
+
+def test_unicode_on_disk_partial_io(tmp_path):
+    arr = make_unicode_array((20,))
+    path = tmp_path / "partial_unicode.b2nd"
+
+    b2 = blosc2.asarray(
+        arr,
+        urlpath=path,
+        mode="w",
+        chunks=(5,),
+        blocks=(2,),
+    )
+
+    # Partial read
+    assert np.array_equal(b2[4:12], arr[4:12])
+
+    # Partial write
+    replacement = np.array(
+        ["python", "is", "good", "!"],
+        dtype=arr.dtype,
+    )
+    b2[6:10] = replacement
+    arr[6:10] = replacement
+
+    reopened = blosc2.open(path)
+    assert np.array_equal(reopened, arr)
+
+
+def test_unicode_on_disk_persistence(tmp_path):
+    path = tmp_path / "persistent_unicode.b2nd"
+
+    arr1 = make_unicode_array((8,))
+    blosc2.asarray(arr1, urlpath=path, mode="w")
+
+    arr2 = make_unicode_array((8,))
+    b2 = blosc2.open(path, mode="a")
+    b2[:] = arr2
+
+    reopened = blosc2.open(path)
+    assert np.array_equal(reopened, arr2)