Add optimised compression and tests for constructors

Author: lshaw8317

bench/ndarray/stringops_bench.py

@@ -22,7 +22,7 @@
 cparams = blosc2.cparams_dflts
 cparams["filters"][-1] = blosc2.Filter.SHUFFLE
 cparams["filters_meta"][-1] = 0 # use default (typesize)
-arr1 = blosc2.asarray(nparr, cparams=cparams)
+arr1 = blosc2.asarray(nparr)
 print(f"cratio without filter: {arr1.cratio}")
 cparams["filters_meta"][-1] = 4
 arr1 = blosc2.asarray(nparr, cparams=cparams)

src/blosc2/blosc2_ext.pyx

@@ -3110,7 +3110,11 @@ cdef b2nd_context_t* create_b2nd_context(shape, chunks, blocks, dtype, kwargs):
     if 'cparams' in kwargs:
         kwargs['cparams']['typesize'] = typesize
     else:
-        kwargs['cparams'] = {'typesize': typesize}
+        kwargs['cparams'] = {} # last filter is shuffle
+        if isinstance(dtype, np.dtypes.StrDType) or dtype == np.str_:
+            kwargs['cparams'] = {'filters': [blosc2.Filter.NOFILTER] * 5 + [blosc2.Filter.SHUFFLE],
+                                'filters_meta': [0] * 5 + [4]} # unicode char bytesize
+        kwargs['cparams']['typesize'] = typesize
     if dtype.kind == 'V':
         str_dtype = str(dtype)
     else:

src/blosc2/ndarray.py

@@ -4996,6 +4996,8 @@ def _check_dtype(dtype):
     dtype = np.dtype(dtype)
     if dtype.itemsize > blosc2.MAX_TYPESIZE:
         raise ValueError(f"dtype itemsize {dtype.itemsize} is too large (>{blosc2.MAX_TYPESIZE})!")
+    if dtype == np.str_:  # itemsize is 0
+        dtype = np.dtype("<U1")  # default to 1 char strings
     return dtype
 
 

tests/ndarray/test_stringarrays.py

@@ -169,3 +169,61 @@ def test_unicode_on_disk_persistence(tmp_path):
 
     reopened = blosc2.open(path)
     assert np.array_equal(reopened, arr2)
+
+
+@pytest.mark.parametrize(
+    "constructor",
+    [
+        "zeros",
+        "ones",
+        "empty",
+        "full",
+        # "full_like", these all call ones/empty/full/zeros anyway
+        # "zeros_like",
+        # "ones_like",
+        # "empty_like",
+        "eye",
+    ],
+)
+@pytest.mark.parametrize("shape", SHAPES)
+def test_constructors(constructor, shape):
+    if constructor == "full":
+        arr = getattr(blosc2, constructor)(fill_value="pepe", shape=shape, dtype=np.str_)
+        nparr = getattr(np, constructor)(fill_value="pepe", shape=shape, dtype=np.str_)
+    elif constructor == "eye":
+        arr = getattr(blosc2, constructor)(shape[0], dtype=np.str_)
+        nparr = getattr(np, constructor)(shape[0], dtype=np.str_)
+    else:
+        arr = getattr(blosc2, constructor)(shape=shape, dtype=np.str_)
+        nparr = getattr(np, constructor)(shape=shape, dtype=np.str_)
+    np.testing.assert_array_equal(arr[()], nparr)
+
+
+def test_optimised_string_comp():
+    N = int(1e5)
+    nparr = np.repeat(np.array(["josé", "pepe", "francisco"]), N)
+    cparams = blosc2.cparams_dflts
+    arr1 = blosc2.asarray(nparr, cparams=cparams)
+    cratio_subopt = arr1.cratio
+    # when not providing cparams, blosc2_ext passes an optimised pipeline for string dtypes
+    arr1 = blosc2.asarray(nparr)
+    assert arr1.cratio > cratio_subopt
+
+
+@pytest.mark.parametrize("shape", SHAPES)
+def test_frombuffer(shape):
+    chunks = tuple(max(c // 4, 1) for c in shape)
+    dtype = np.dtype("<U8")
+    typesize = dtype.itemsize
+    # Create a buffer
+    nparr = make_unicode_array(shape, maxlen=8)
+    buffer = bytes(nparr)
+    # Create a NDArray from a buffer with default blocks
+    arr = blosc2.frombuffer(buffer, shape, chunks=chunks, dtype=dtype)
+    np.testing.assert_array_equal(arr[()], nparr)
+
+
+def test_fromiter():
+    nparr = np.arange(0, 20).astype(np.str_)
+    arr = blosc2.fromiter(range(20), shape=(20,), dtype=nparr.dtype)
+    np.testing.assert_array_equal(arr[()], nparr)

tests/ndarray/test_zeros.py

@@ -56,6 +56,16 @@
             False,
             None,
         ),
+        (
+            (2**31 - 1,),
+            (2**30,),
+            None,
+            np.str_,
+            {"codec": blosc2.Codec.LZ4, "clevel": 5, "nthreads": 2},
+            None,
+            False,
+            None,
+        ),
     ],
 )
 def test_zeros(shape, chunks, blocks, dtype, cparams, urlpath, contiguous, meta):