From cabb0937da5dd2a77ea4b4cce10d4d2b3657ee10 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Thu, 3 Apr 2025 19:31:34 +0100 Subject: [PATCH 1/9] Fix specifying memory order in v2 arrays --- src/zarr/api/asynchronous.py | 17 ++++++++--------- src/zarr/core/array.py | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 6059893920..9d89167ef2 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -1035,15 +1035,13 @@ async def create( ) warnings.warn(UserWarning(msg), stacklevel=1) config_dict["write_empty_chunks"] = write_empty_chunks - if order is not None: - if config is not None: - msg = ( - "Both order and config keyword arguments are set. " - "This is redundant. When both are set, order will be ignored and " - "config will be used." - ) - warnings.warn(UserWarning(msg), stacklevel=1) - config_dict["order"] = order + if order is not None and config is not None: + msg = ( + "Both order and config keyword arguments are set. " + "This is redundant. When both are set, order will be ignored and " + "config will be used." + ) + warnings.warn(UserWarning(msg), stacklevel=1) config_parsed = ArrayConfig.from_dict(config_dict) @@ -1057,6 +1055,7 @@ async def create( overwrite=overwrite, filters=filters, dimension_separator=dimension_separator, + order=order, zarr_format=zarr_format, chunk_shape=chunk_shape, chunk_key_encoding=chunk_key_encoding, diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 0e03cbcabb..f8251a0b36 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -1042,7 +1042,7 @@ def order(self) -> MemoryOrder: bool Memory order of the array """ - return self._config.order + return self.metadata.order @property def attrs(self) -> dict[str, JSON]: From b89600857bc789807d8441fbd743a597454c0eff Mon Sep 17 00:00:00 2001 From: David Stansby Date: Thu, 3 Apr 2025 19:42:08 +0100 Subject: [PATCH 2/9] Re-work test_order --- tests/test_array.py | 41 ++++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/tests/test_array.py b/tests/test_array.py index efcf8a6bf9..cb5cd09227 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -1241,9 +1241,11 @@ async def test_data_ignored_params(store: Store) -> None: await create_array(store, data=data, shape=None, dtype=data.dtype, overwrite=True) @staticmethod - @pytest.mark.parametrize("order_config", ["C", "F", None]) + @pytest.mark.parametrize("order", ["C", "F", None]) + @pytest.mark.parametrize("with_config", [True, False]) def test_order( - order_config: MemoryOrder | None, + order: MemoryOrder | None, + with_config: bool, zarr_format: ZarrFormat, store: MemoryStore, ) -> None: @@ -1252,28 +1254,29 @@ def test_order( value, and that for zarr v2 arrays, the ``order`` field in the array metadata is set correctly. """ config: ArrayConfigLike = {} - if order_config is None: + if order is None: config = {} expected = zarr.config.get("array.order") else: - config = {"order": order_config} - expected = order_config + config = {"order": order} + expected = order + + if not with_config: + # Test without passing config parameter + config = None + + arr = zarr.create_array( + store=store, + shape=(2, 2), + zarr_format=zarr_format, + dtype="i4", + order=order, + config=config, + ) if zarr_format == 2: - arr = zarr.create_array( - store=store, - shape=(2, 2), - zarr_format=zarr_format, - dtype="i4", - order=expected, - config=config, - ) - # guard for type checking - assert arr.metadata.zarr_format == 2 assert arr.metadata.order == expected - else: - arr = zarr.create_array( - store=store, shape=(2, 2), zarr_format=zarr_format, dtype="i4", config=config - ) + assert arr.order == expected + vals = np.asarray(arr) if expected == "C": assert vals.flags.c_contiguous From 1c3c3f6652157c7312c5553329b77208db8f3fbe Mon Sep 17 00:00:00 2001 From: David Stansby Date: Thu, 3 Apr 2025 20:57:27 +0100 Subject: [PATCH 3/9] Fix getting array order in v3 --- src/zarr/core/array.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index f8251a0b36..093dd060a0 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -1042,7 +1042,10 @@ def order(self) -> MemoryOrder: bool Memory order of the array """ - return self.metadata.order + if self.metadata.zarr_format == 2: + return self.metadata.order + else: + return self._config.order @property def attrs(self) -> dict[str, JSON]: From cc5ee3f8ab974d27c95bdf4d247cbd22f8f1d25c Mon Sep 17 00:00:00 2001 From: David Stansby Date: Thu, 3 Apr 2025 20:59:38 +0100 Subject: [PATCH 4/9] Fix order of arrays for v2 --- src/zarr/core/array.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 093dd060a0..d460086683 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -1277,14 +1277,14 @@ async def _get_selection( out_buffer = prototype.nd_buffer.create( shape=indexer.shape, dtype=out_dtype, - order=self._config.order, + order=self.order, fill_value=self.metadata.fill_value, ) if product(indexer.shape) > 0: # need to use the order from the metadata for v2 _config = self._config if self.metadata.zarr_format == 2: - _config = replace(_config, order=self.metadata.order) + _config = replace(_config, order=self.order) # reading chunks and decoding them await self.codec_pipeline.read( From 13a65a5d66bdac9d3f4e0b6f3c8e8abadf145923 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Thu, 3 Apr 2025 21:06:47 +0100 Subject: [PATCH 5/9] Fix order with V3 arrays --- src/zarr/core/array.py | 6 ++++++ tests/test_array.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index d460086683..37f16ac2d6 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -3989,6 +3989,12 @@ async def init_array( chunks_out = chunk_shape_parsed codecs_out = sub_codecs + if order is not None: + if config is None: + config = {} + if "order" not in config: + config["order"] = order + meta = AsyncArray._create_metadata_v3( shape=shape_parsed, dtype=dtype_parsed, diff --git a/tests/test_array.py b/tests/test_array.py index cb5cd09227..4965971ad0 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -1273,9 +1273,9 @@ def test_order( order=order, config=config, ) + assert arr.order == expected if zarr_format == 2: assert arr.metadata.order == expected - assert arr.order == expected vals = np.asarray(arr) if expected == "C": From 0e32d41d6e17b2c0ecca1e1d95223e22f93c7373 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Thu, 3 Apr 2025 21:21:04 +0100 Subject: [PATCH 6/9] Fix mypy --- src/zarr/core/array.py | 10 +++++----- tests/test_array.py | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 37f16ac2d6..751d5d2af7 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -3989,12 +3989,12 @@ async def init_array( chunks_out = chunk_shape_parsed codecs_out = sub_codecs - if order is not None: - if config is None: - config = {} - if "order" not in config: - config["order"] = order + if config is None: + config = {} + if order is not None and isinstance(config, dict): + config["order"] = config.get("order", order) + print(config) meta = AsyncArray._create_metadata_v3( shape=shape_parsed, dtype=dtype_parsed, diff --git a/tests/test_array.py b/tests/test_array.py index 4965971ad0..e10942fe35 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -1253,7 +1253,7 @@ def test_order( Test that the arrays generated by array indexing have a memory order defined by the config order value, and that for zarr v2 arrays, the ``order`` field in the array metadata is set correctly. """ - config: ArrayConfigLike = {} + config: ArrayConfigLike | None = {} if order is None: config = {} expected = zarr.config.get("array.order") @@ -1275,6 +1275,7 @@ def test_order( ) assert arr.order == expected if zarr_format == 2: + assert arr.metadata.zarr_format == 2 assert arr.metadata.order == expected vals = np.asarray(arr) From 2738f2d6cf410705a0cb338dae597a57e2c68c75 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Thu, 3 Apr 2025 21:24:05 +0100 Subject: [PATCH 7/9] Remove errant print() --- src/zarr/core/array.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 751d5d2af7..77a32575d5 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -3994,7 +3994,6 @@ async def init_array( if order is not None and isinstance(config, dict): config["order"] = config.get("order", order) - print(config) meta = AsyncArray._create_metadata_v3( shape=shape_parsed, dtype=dtype_parsed, From a022d77b83958f088ebf0fb481fd2f018b6afadc Mon Sep 17 00:00:00 2001 From: David Stansby Date: Thu, 3 Apr 2025 22:32:07 +0100 Subject: [PATCH 8/9] Fix order with v3 arrays --- src/zarr/core/array.py | 1 + tests/test_api.py | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 77a32575d5..8e329f6d2c 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -607,6 +607,7 @@ async def _create( if order is not None: _warn_order_kwarg() + config_parsed = replace(config_parsed, order=order) result = await cls._create_v3( store_path, diff --git a/tests/test_api.py b/tests/test_api.py index 94140ac784..c848c7a0b8 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -317,13 +317,12 @@ def test_array_order(zarr_format: ZarrFormat) -> None: def test_array_order_warns(order: MemoryOrder | None, zarr_format: ZarrFormat) -> None: with pytest.warns(RuntimeWarning, match="The `order` keyword argument .*"): arr = zarr.ones(shape=(2, 2), order=order, zarr_format=zarr_format) - expected = order or zarr.config.get("array.order") - assert arr.order == expected + assert arr.order == order vals = np.asarray(arr) - if expected == "C": + if order == "C": assert vals.flags.c_contiguous - elif expected == "F": + elif order == "F": assert vals.flags.f_contiguous else: raise AssertionError From 36a1966d7ceae8bf93ee1fd6297c6a184c992f77 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Thu, 3 Apr 2025 22:44:33 +0100 Subject: [PATCH 9/9] Fix v2 test --- tests/test_v2.py | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/tests/test_v2.py b/tests/test_v2.py index 0a4487cfcc..bf42b97378 100644 --- a/tests/test_v2.py +++ b/tests/test_v2.py @@ -194,12 +194,8 @@ def test_create_array_defaults(store: Store): ) -@pytest.mark.parametrize("array_order", ["C", "F"]) -@pytest.mark.parametrize("data_order", ["C", "F"]) -@pytest.mark.parametrize("memory_order", ["C", "F"]) -def test_v2_non_contiguous( - array_order: Literal["C", "F"], data_order: Literal["C", "F"], memory_order: Literal["C", "F"] -) -> None: +@pytest.mark.parametrize("order", ["C", "F"]) +def test_v2_non_contiguous(order: Literal["C", "F"]) -> None: store = MemoryStore() arr = zarr.create_array( store, @@ -211,12 +207,11 @@ def test_v2_non_contiguous( filters=None, compressors=None, overwrite=True, - order=array_order, - config={"order": memory_order}, + order=order, ) # Non-contiguous write - a = np.arange(arr.shape[0] * arr.shape[1]).reshape(arr.shape, order=data_order) + a = np.arange(arr.shape[0] * arr.shape[1]).reshape(arr.shape, order=order) arr[6:9, 3:6] = a[6:9, 3:6] # The slice on the RHS is important np.testing.assert_array_equal(arr[6:9, 3:6], a[6:9, 3:6]) @@ -224,9 +219,9 @@ def test_v2_non_contiguous( a[6:9, 3:6], np.frombuffer( sync(store.get("2.1", default_buffer_prototype())).to_bytes(), dtype="float64" - ).reshape((3, 3), order=array_order), + ).reshape((3, 3), order=order), ) - if memory_order == "F": + if order == "F": assert (arr[6:9, 3:6]).flags.f_contiguous else: assert (arr[6:9, 3:6]).flags.c_contiguous @@ -242,13 +237,12 @@ def test_v2_non_contiguous( compressors=None, filters=None, overwrite=True, - order=array_order, - config={"order": memory_order}, + order=order, ) # Contiguous write - a = np.arange(9).reshape((3, 3), order=data_order) - if data_order == "F": + a = np.arange(9).reshape((3, 3), order=order) + if order == "F": assert a.flags.f_contiguous else: assert a.flags.c_contiguous