Skip to content

Support unsafe casting in Delta filter #657

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion numcodecs/delta.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ def encode(self, buf):
if arr.dtype == bool:
np.not_equal(arr[1:], arr[:-1], out=enc[1:])
else:
np.subtract(arr[1:], arr[:-1], out=enc[1:])
np.subtract(arr[1:], arr[:-1], out=enc[1:], casting='unsafe')

return enc

Expand Down
55 changes: 46 additions & 9 deletions numcodecs/tests/test_delta.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,19 +13,24 @@
# mix of dtypes: integer, float
# mix of shapes: 1D, 2D, 3D
# mix of orders: C, F
# mix of encoding types: All available types for each arrays
arrays = [
np.random.randint(0, 1, size=110, dtype='?').reshape(10, 11),
np.arange(1000, dtype='<i4'),
np.linspace(1000, 1001, 1000, dtype='<f4').reshape(100, 10),
np.random.normal(loc=1000, scale=1, size=(10, 10, 10)).astype('<f8'),
np.random.randint(0, 200, size=1000, dtype='u2').astype('<u2').reshape(100, 10, order='F'),
(np.random.randint(0, 1, size=110, dtype='?').reshape(10, 11), ('?', '<u1', '<i1')),
(np.arange(1000, dtype='<i4'), ('<i4', '<i2', '<u4', 'u2')),
(np.linspace(1000, 1001, 1000, dtype='<f4').reshape(100, 10), ('<f4',)),
(np.random.normal(loc=1000, scale=1, size=(10, 10, 10)).astype('<f8'), ('<f8',)),
(
np.random.randint(0, 200, size=1000, dtype='u2').astype('<u2').reshape(100, 10, order='F'),
('<i2',),
),
]


def test_encode_decode():
for arr in arrays:
codec = Delta(dtype=arr.dtype)
check_encode_decode(arr, codec)
for arr, encoding_types in arrays:
for astype in encoding_types:
codec = Delta(dtype=arr.dtype, astype=astype)
check_encode_decode(arr, codec)


def test_encode():
Expand All @@ -49,7 +54,7 @@ def test_repr():


def test_backwards_compatibility():
for arr in arrays:
for arr, _ in arrays:
codec = Delta(dtype=arr.dtype)
check_backwards_compatibility(Delta.codec_id, [arr], [codec], prefix=str(arr.dtype))

Expand All @@ -59,3 +64,35 @@ def test_errors():
Delta(dtype=object)
with pytest.raises(ValueError):
Delta(dtype='i8', astype=object)


# overflow tests
# Note: Before implementing similar test for integer -> integer types, check numpy/numpy#8987.
oveflow_proned_float_float_pairs = [
('f4', 'f2'),
('f8', 'f4'),
]


def test_oveflow_proned_float_float_encode():
for dtype, astype in oveflow_proned_float_float_pairs:
codec = Delta(dtype=dtype, astype=astype)
arr = np.array([0, np.finfo(astype).max.astype(dtype) * 2], dtype=dtype)
with pytest.warns(RuntimeWarning, match=r"overflow encountered"):
codec.encode(arr)


overflow_proned_integer_float_paris = [
('i4', 'f2'),
('i8', 'f2'),
('u4', 'f2'),
('u8', 'f2'),
]


def test_oveflow_proned_integer_float_encode():
for dtype, astype in overflow_proned_integer_float_paris:
codec = Delta(dtype=dtype, astype=astype)
arr = np.array([0, int(np.rint(np.finfo(astype).max)) * 2], dtype=dtype)
with pytest.warns(RuntimeWarning, match=r"overflow encountered"):
codec.encode(arr)
Loading