Skip to content

Commit 815b4db

Browse files
FrancescAltedclaude
andcommitted
Align DSL operand grids, so mixed-dtype NDArray inputs work
Blocks are sized in bytes, so same-shaped operands of different itemsize get different chunks/blocks (float32 vs int64 over 1M elements: blocks of 31250 vs 15625). validate_inputs then refuses the miniexpr fast path, and a DSL kernel has no slow path to fall back on, so evaluation died with a misleading "slicing a DSL computation is not supported". Whether the grids diverge depends on array size and on the platform's cache detection, which is why CI caught it only on Windows (where platform.machine() is "AMD64", so compute_chunks_blocks' x86_64 branch never runs and blocks stay L1-sized) and WASM: their 10,007-element operands already disagree, while elsewhere both fit a single block. At 1M elements it failed everywhere, macOS included -- hence the regression test sits at that size. LazyUDF now copies mismatched NDArray operands onto the grid of the widest dtype (fewest elements per block, so every operand still fits the cache budget the heuristic aimed at), once at construction rather than per evaluation, and only for DSL kernels. Also trace the JS bridge under BLOSC_ME_JIT_TRACE. It bypasses miniexpr entirely, so it used to report nothing at all, which is what turned the WASM-only mandel dispatch test into a confusing empty-output failure rather than an obvious one. Both engines now trace, and that test asserts on both platforms instead of skipping. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
1 parent 13cf972 commit 815b4db

3 files changed

Lines changed: 63 additions & 2 deletions

File tree

src/blosc2/lazyexpr.py

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1495,6 +1495,15 @@ def _js_dtypes_ok(operands, kwargs) -> bool:
14951495
)
14961496

14971497

1498+
def _trace_js_backend(expression):
1499+
"""BLOSC_ME_JIT_TRACE counterpart for the JS bridge, which never reaches
1500+
miniexpr's trace point in `fast_eval` (see there for the message format)."""
1501+
if os.environ.get("BLOSC_ME_JIT_TRACE", "").lower() in ("1", "true", "on"):
1502+
source = getattr(expression, "dsl_source", None) or expression
1503+
expr_short = str(source)[:120].replace("\n", " ")
1504+
print(f"[blosc2] engine=js expr={expr_short}", flush=True)
1505+
1506+
14981507
def _maybe_js_backend(expression, jit, jit_backend, reduce_args, operands, kwargs, shape=None):
14991508
"""Resolve the JS backend for a DSL kernel.
15001509
@@ -1524,7 +1533,9 @@ def _maybe_js_backend(expression, jit, jit_backend, reduce_args, operands, kwarg
15241533
'jit_backend="js" requires a floating-point output dtype '
15251534
f"(got {np.dtype(out_dtype)}); drop jit_backend to use miniexpr"
15261535
)
1527-
return _as_js_udf(expression, shape), None, None
1536+
bridge = _as_js_udf(expression, shape)
1537+
_trace_js_backend(expression)
1538+
return bridge, None, None
15281539
prefer_js = (
15291540
jit is not False # jit=True/None prefer the best JIT (js); only jit=False forces interpreter
15301541
and jit_backend is None
@@ -1541,6 +1552,7 @@ def _maybe_js_backend(expression, jit, jit_backend, reduce_args, operands, kwarg
15411552
bridge = _as_js_udf(expression, shape) # transpiles; raises on any unsupported construct
15421553
except Exception:
15431554
return expression, jit, jit_backend # fall back to miniexpr, no regression
1555+
_trace_js_backend(expression)
15441556
return bridge, None, None
15451557

15461558

@@ -4576,12 +4588,45 @@ def _new_expr(cls, expression, operands, guess, out=None, where=None, ne_args=No
45764588
return new_expr
45774589

45784590

4591+
def _align_dsl_operand_grids(inputs):
4592+
"""Put every NDArray operand of a DSL kernel on a single chunks/blocks grid.
4593+
4594+
Blocks are sized in bytes, so same-shaped operands of different itemsize get
4595+
different grids by default (float32 vs int64 over 1M elements: blocks of
4596+
31250 vs 15625 elements). miniexpr needs one common grid, and a DSL kernel
4597+
has no slow path to fall back on, so evaluation would fail outright with a
4598+
confusing "slicing is not supported" error. Copy the odd operands onto the
4599+
grid of the widest dtype: its blocks hold the fewest elements, so every
4600+
operand still fits within the cache budget the heuristic aimed at.
4601+
4602+
The copies happen once, at construction, and only when the grids actually
4603+
disagree -- whether they do depends on the array size and on the platform's
4604+
cache detection, which is why this used to fail only on some CI runners.
4605+
"""
4606+
nd = [x for x in inputs if isinstance(x, blosc2.NDArray) and x.ndim > 0]
4607+
if len(nd) < 2 or len({(x.shape, x.chunks, x.blocks) for x in nd}) < 2:
4608+
return inputs
4609+
ref = max(nd, key=lambda x: x.dtype.itemsize)
4610+
aligned = []
4611+
for x in inputs:
4612+
misaligned = (
4613+
isinstance(x, blosc2.NDArray)
4614+
and x.ndim > 0
4615+
and x.shape == ref.shape
4616+
and (x.chunks, x.blocks) != (ref.chunks, ref.blocks)
4617+
)
4618+
aligned.append(x.copy(chunks=ref.chunks, blocks=ref.blocks) if misaligned else x)
4619+
return aligned
4620+
4621+
45794622
class LazyUDF(LazyArray):
45804623
def __init__(
45814624
self, func, inputs, dtype, shape=None, chunked_eval=True, jit=None, jit_backend=None, **kwargs
45824625
):
45834626
# After this, all the inputs should be np.ndarray or NDArray objects
45844627
self.inputs = convert_inputs(inputs)
4628+
if isinstance(func, DSLKernel):
4629+
self.inputs = _align_dsl_operand_grids(self.inputs)
45854630
# Get res shape
45864631
if shape is None:
45874632
self._shape = compute_broadcast_shape(self.inputs)

tests/ndarray/test_dsl_kernels.py

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1196,6 +1196,20 @@ def test_dsl_kernel_numpy_operands_mixed_dtype_promotes_output():
11961196
np.testing.assert_array_equal(res, ref)
11971197

11981198

1199+
def test_dsl_kernel_ndarray_operands_with_different_itemsize():
1200+
# Blocks are sized in bytes, so a float32 and an int64 operand get different
1201+
# chunks/blocks by default; the DSL path has no slow fallback, so it used to
1202+
# raise "slicing is not supported" whenever the grids diverged (which depends
1203+
# on array size and on the platform's cache detection).
1204+
n = 1_000_000
1205+
a = (np.arange(n) % 7).astype(np.float32)
1206+
b = (np.arange(n) % 5).astype(np.int64)
1207+
A, B = blosc2.asarray(a), blosc2.asarray(b)
1208+
assert (A.chunks, A.blocks) != (B.chunks, B.blocks)
1209+
res = blosc2.lazyudf(_numpy_operand_kernel, (A, B), dtype=None)[()]
1210+
np.testing.assert_array_equal(res, a * 2.0 + b)
1211+
1212+
11991213
def test_dsl_kernel_mixed_ndarray_and_numpy_operand():
12001214
shape = (20, 10)
12011215
a = np.arange(np.prod(shape), dtype=np.float64).reshape(shape)

tests/ndarray/test_jit_dsl_dispatch.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,9 @@ def mandel(cr, ci, max_iter):
5353
monkeypatch.setenv("BLOSC_ME_JIT_TRACE", "1")
5454
res = mandel(cr, ci, 30)
5555
captured = capsys.readouterr()
56-
assert "engine=miniexpr" in captured.out
56+
# Under WebAssembly this kernel is transpiled to the JS bridge instead of
57+
# going through miniexpr (see _maybe_js_backend); both engines trace.
58+
assert f"engine={'js' if blosc2.IS_WASM else 'miniexpr'}" in captured.out
5759
assert "def mandel" in captured.out
5860
np.testing.assert_array_equal(res, _mandel_numpy(cr, ci, 30))
5961

0 commit comments

Comments
 (0)