Fixes

Author: dstansby

numcodecs/blosc.py

@@ -95,7 +95,14 @@ def _check_buffer_size(buf, max_buffer_size):
         raise ValueError(msg)
 
 
-def compress(source, cname: str, clevel: int, shuffle: int = SHUFFLE, blocksize=AUTOBLOCKS):
+def compress(
+    source,
+    cname: str,
+    clevel: int,
+    shuffle: int = SHUFFLE,
+    blocksize=AUTOBLOCKS,
+    typesize: int = 8,
+):
     """
     Compress data.
 
@@ -127,19 +134,28 @@ def compress(source, cname: str, clevel: int, shuffle: int = SHUFFLE, blocksize=
             shuffle = BITSHUFFLE
         else:
             shuffle = SHUFFLE
+
     blosc.set_blocksize(blocksize)
     if isinstance(source, np.ndarray):
+        print('hi')
+        if typesize is None:
+            typesize = source.dtype.itemsize
         _check_not_object_array(source)
         result = blosc.compress_ptr(
             source.ctypes.data,
             source.size,
-            source.dtype.itemsize,
+            typesize,
             cname=cname,
             clevel=clevel,
             shuffle=shuffle,
         )
     else:
-        result = blosc.compress(source, cname=cname, clevel=clevel, shuffle=shuffle)
+        if typesize is None:
+            # Same default as blosc
+            typesize = 8
+        result = blosc.compress(
+            source, cname=cname, clevel=clevel, shuffle=shuffle, typesize=typesize
+        )
     blosc.set_blocksize(AUTOBLOCKS)
     return result
 
@@ -205,7 +221,16 @@ class Blosc(Codec):
     AUTOSHUFFLE = AUTOSHUFFLE
     max_buffer_size = 2**31 - 1
 
-    def __init__(self, cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=AUTOBLOCKS):
+    def __init__(
+        self,
+        cname='lz4',
+        clevel=5,
+        shuffle=SHUFFLE,
+        blocksize=AUTOBLOCKS,
+        typesize: int | None = None,
+    ):
+        if isinstance(typesize, int) and typesize < 1:
+            raise ValueError(f"Cannot use typesize {typesize} less than 1.")
         self.cname = cname
         if isinstance(cname, str):
             self._cname_bytes = cname.encode('ascii')
@@ -214,11 +239,17 @@ def __init__(self, cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=AUTOBLOCKS)
         self.clevel = clevel
         self.shuffle = shuffle
         self.blocksize = blocksize
+        self.typesize = typesize
 
     def encode(self, buf):
         _check_buffer_size(buf, self.max_buffer_size)
         return compress(
-            buf, self.cname, clevel=self.clevel, shuffle=self.shuffle, blocksize=self.blocksize
+            buf,
+            self.cname,
+            clevel=self.clevel,
+            shuffle=self.shuffle,
+            blocksize=self.blocksize,
+            typesize=self.typesize,
         )
 
     def decode(self, buf, out=None):

numcodecs/tests/test_blosc.py

@@ -103,33 +103,6 @@ def test_eq():
     assert Blosc(cname='lz4') != 'foo'
 
 
-def test_compress_blocksize_default(use_threads):
-    arr = np.arange(1000, dtype='i4')
-
-    blosc.use_threads = use_threads
-
-    # default blocksize
-    enc = blosc.compress(arr, b'lz4', 1, Blosc.NOSHUFFLE)
-    _, _, blocksize = blosc._cbuffer_sizes(enc)
-    assert blocksize > 0
-
-    # explicit default blocksize
-    enc = blosc.compress(arr, b'lz4', 1, Blosc.NOSHUFFLE, 0)
-    _, _, blocksize = blosc._cbuffer_sizes(enc)
-    assert blocksize > 0
-
-
-@pytest.mark.parametrize('bs', [2**7, 2**8])
-def test_compress_blocksize(use_threads, bs):
-    arr = np.arange(1000, dtype='i4')
-
-    blosc.use_threads = use_threads
-
-    enc = blosc.compress(arr, b'lz4', 1, Blosc.NOSHUFFLE, bs)
-    _, _, blocksize = blosc._cbuffer_sizes(enc)
-    assert blocksize == bs
-
-
 def test_compress_complib(use_threads):
     arr = np.arange(1000, dtype='i4')
     expected_complibs = {
@@ -141,43 +114,16 @@ def test_compress_complib(use_threads):
     }
     blosc.use_threads = use_threads
     for cname in blosc.list_compressors():
-        enc = blosc.compress(arr, cname.encode(), 1, Blosc.NOSHUFFLE)
+        enc = blosc.compress(arr, cname, 1, Blosc.NOSHUFFLE)
         complib = blosc.cbuffer_complib(enc)
         expected_complib = expected_complibs[cname]
         assert complib == expected_complib
     with pytest.raises(ValueError):
         # capitalized cname
-        blosc.compress(arr, b'LZ4', 1)
+        blosc.compress(arr, 'LZ4', 1)
     with pytest.raises(ValueError):
         # bad cname
-        blosc.compress(arr, b'foo', 1)
-
-
-@pytest.mark.parametrize('dtype', ['i1', 'i2', 'i4', 'i8'])
-def test_compress_metainfo(dtype, use_threads):
-    arr = np.arange(1000, dtype=dtype)
-    for shuffle in Blosc.NOSHUFFLE, Blosc.SHUFFLE, Blosc.BITSHUFFLE:
-        blosc.use_threads = use_threads
-        for cname in blosc.list_compressors():
-            enc = blosc.compress(arr, cname.encode(), 1, shuffle)
-            typesize, did_shuffle, _ = blosc._cbuffer_metainfo(enc)
-            assert typesize == arr.dtype.itemsize
-            assert did_shuffle == shuffle
-
-
-def test_compress_autoshuffle(use_threads):
-    arr = np.arange(8000)
-    for dtype in 'i1', 'i2', 'i4', 'i8', 'f2', 'f4', 'f8', 'bool', 'S10':
-        varr = arr.view(dtype)
-        blosc.use_threads = use_threads
-        for cname in blosc.list_compressors():
-            enc = blosc.compress(varr, cname.encode(), 1, Blosc.AUTOSHUFFLE)
-            typesize, did_shuffle, _ = blosc._cbuffer_metainfo(enc)
-            assert typesize == varr.dtype.itemsize
-            if typesize == 1:
-                assert did_shuffle == Blosc.BITSHUFFLE
-            else:
-                assert did_shuffle == Blosc.SHUFFLE
+        blosc.compress(arr, 'foo', 1)
 
 
 def test_config_blocksize():
@@ -269,16 +215,16 @@ def test_typesize_explicit():
     encoded_without_itemsize = codec_no_type_size.encode(arr.tobytes())
     encoded_with_itemsize = codec_itemsize.encode(arr.tobytes())
     # third byte encodes the `typesize`
-    assert encoded_without_itemsize[3] == 1  # inferred from bytes i.e., 1
+    assert encoded_without_itemsize[3] == 8  # default blosc itemsize
     assert encoded_with_itemsize[3] == itemsize  # given as a constructor argument
 
 
 def test_typesize_less_than_1():
-    with pytest.raises(ValueError, match=r"Cannot use typesize"):
+    with pytest.raises(ValueError, match=r"Cannot use typesize 0 less than 1"):
         Blosc(shuffle=Blosc.SHUFFLE, typesize=0)
     compressor = Blosc(shuffle=Blosc.SHUFFLE)
     # not really something that should be done in practice, but good for testing.
     compressor.typesize = 0
     arr = np.arange(100)
-    with pytest.raises(ValueError, match=r"Cannot use typesize"):
+    with pytest.raises(ValueError, match=r"typesize can only be in the 1-255 range"):
         compressor.encode(arr.tobytes())