From 02c0e05fafcf07e25a5f9a42d08a370afe038cc2 Mon Sep 17 00:00:00 2001 From: sameeul Date: Tue, 6 Aug 2024 17:50:25 -0400 Subject: [PATCH 1/6] Make JavaReader pickle-able --- src/bfio/bfio.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/bfio/bfio.py b/src/bfio/bfio.py index 4651618..56e325c 100644 --- a/src/bfio/bfio.py +++ b/src/bfio/bfio.py @@ -43,16 +43,15 @@ class BioReader(BioBase): logger = logging.getLogger("bfio.bfio.BioReader") _STATE_DICT = [ + "level", "_metadata", "_DIMS", "_file_path", - "max_workers", + "_max_workers", "_backend_name", "clean_metadata", "_read_only", - "_backend", - "level", - "append", + "_backend" ] def __init__( @@ -234,7 +233,15 @@ def set_backend(self, backend: typing.Optional[str] = None) -> None: self._backend_name = backend.lower() def __getstate__(self) -> typing.Dict: - state_dict = {n: getattr(self, n) for n in self._STATE_DICT} + if self._backend_name == "bioformats": + state_dict = {} + for n in self._STATE_DICT: + if n == "_backend": + state_dict[n] = "JavaReaderDummy" + else: + state_dict[n] = getattr(self, n) + else: + state_dict = {n: getattr(self, n) for n in self._STATE_DICT} return state_dict @@ -243,7 +250,10 @@ def __setstate__(self, state) -> None: assert all(n in state.keys() for n in self._STATE_DICT) for k, v in state.items(): - setattr(self, k, v) + if k == "_backend" and v == "JavaReaderDummy": + self._backend = backends.JavaReader(self) + else: + setattr(self, k, v) self._backend.frontend = self From 0360ba56a3e693b7e2144b25c8e385ae2c7e3e5c Mon Sep 17 00:00:00 2001 From: sameeul Date: Wed, 7 Aug 2024 16:18:23 -0400 Subject: [PATCH 2/6] Make PythonReader pickle-able --- src/bfio/backends.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/bfio/backends.py b/src/bfio/backends.py index a82f062..c08df65 100644 --- a/src/bfio/backends.py +++ b/src/bfio/backends.py @@ -122,18 +122,26 @@ def __init__(self, frontend): def __getstate__(self) -> Dict: state_dict = {n: getattr(self, n) for n in self._STATE_DICT} state_dict.update({"file_path": self.frontend._file_path}) + state_dict.update({"level":self.frontend.level}) return state_dict def __setstate__(self, state) -> None: for k, v in state.items(): - if k == "file_path": - self._lock = threading.Lock() - self._rdr = tifffile.TiffFile(v) - self._rdr.filehandle.close() + if k == "file_path" or k == "level": + pass else: setattr(self, k, v) + self._lock = threading.Lock() + self._rdr = tifffile.TiffFile(state["file_path"]) + self._rdr_pages = self._rdr.pages + if state["level"] is not None: + if len(self._rdr.series) != 0: + series = self._rdr.series[0] + self._rdr_pages = series.levels[state["level"]] + self._rdr.filehandle.close() + def read_metadata(self): self.logger.debug("read_metadata(): Reading metadata...") From 06359a74e3765aa5e39d3ce50b8fc3067c022c95 Mon Sep 17 00:00:00 2001 From: sameeul Date: Wed, 7 Aug 2024 16:25:51 -0400 Subject: [PATCH 3/6] Add tests --- tests/test_read.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tests/test_read.py b/tests/test_read.py index 85c8928..9d8e3cc 100644 --- a/tests/test_read.py +++ b/tests/test_read.py @@ -3,6 +3,7 @@ import requests, pathlib, shutil, logging, sys import bfio import numpy as np +import pickle import random import zarr from ome_zarr.utils import download as zarr_download @@ -420,3 +421,44 @@ def test_set_metadata(self): logger.info(br.cnames) logger.info(br.ps_x) self.assertEqual(br.cnames[0], cname[0]) + +class TestBioReaderPickle(unittest.TestCase): + def test_python_backend_pickle(self): + # create a 2D numpy array filled with random integer form 0-255 + img_height = 1024 + img_width = 1024 + source_data = np.random.randint( + 0, 256, (img_height, img_width), dtype=np.uint16 + ) + + with bfio.BioWriter( + str(TEST_DIR.joinpath("radnom_image.ome.tiff")), + X=img_width, + Y=img_height, + dtype=np.uint16, + ) as bw: + bw[0:img_height, 0:img_width, 0, 0, 0] = source_data + + br = bfio.BioReader(str(TEST_DIR.joinpath("radnom_image.ome.tiff"))) + br.close() + pickled_rdr = pickle.dumps(br) + unpickled_rdr = pickle.loads(pickled_rdr) + + assert unpickled_rdr.X == img_width + assert unpickled_rdr.Y == img_height + assert unpickled_rdr[:].sum() == source_data.sum() + + def test_bioformats_backend_pickle(self): + + br = bfio.BioReader(str(TEST_DIR.joinpath("img_r001_c001.ome.tif"))) + img_height = br.Y + img_width = br.X + data_sum = br[:].sum() + br.close() + + pickled_rdr = pickle.dumps(br) + unpickled_rdr = pickle.loads(pickled_rdr) + + assert unpickled_rdr.X == img_width + assert unpickled_rdr.Y == img_height + assert unpickled_rdr[:].sum() == data_sum From 24b5f7c0f13a7279788b12830b94ea0b52cd8779 Mon Sep 17 00:00:00 2001 From: sameeul Date: Wed, 7 Aug 2024 16:27:20 -0400 Subject: [PATCH 4/6] Fix formatting --- src/bfio/backends.py | 4 ++-- src/bfio/bfio.py | 4 ++-- tests/test_read.py | 11 ++++++----- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/bfio/backends.py b/src/bfio/backends.py index c08df65..78f2359 100644 --- a/src/bfio/backends.py +++ b/src/bfio/backends.py @@ -122,7 +122,7 @@ def __init__(self, frontend): def __getstate__(self) -> Dict: state_dict = {n: getattr(self, n) for n in self._STATE_DICT} state_dict.update({"file_path": self.frontend._file_path}) - state_dict.update({"level":self.frontend.level}) + state_dict.update({"level": self.frontend.level}) return state_dict @@ -138,7 +138,7 @@ def __setstate__(self, state) -> None: self._rdr_pages = self._rdr.pages if state["level"] is not None: if len(self._rdr.series) != 0: - series = self._rdr.series[0] + series = self._rdr.series[0] self._rdr_pages = series.levels[state["level"]] self._rdr.filehandle.close() diff --git a/src/bfio/bfio.py b/src/bfio/bfio.py index 56e325c..3e6c225 100644 --- a/src/bfio/bfio.py +++ b/src/bfio/bfio.py @@ -51,7 +51,7 @@ class BioReader(BioBase): "_backend_name", "clean_metadata", "_read_only", - "_backend" + "_backend", ] def __init__( @@ -239,7 +239,7 @@ def __getstate__(self) -> typing.Dict: if n == "_backend": state_dict[n] = "JavaReaderDummy" else: - state_dict[n] = getattr(self, n) + state_dict[n] = getattr(self, n) else: state_dict = {n: getattr(self, n) for n in self._STATE_DICT} diff --git a/tests/test_read.py b/tests/test_read.py index 9d8e3cc..0576e5b 100644 --- a/tests/test_read.py +++ b/tests/test_read.py @@ -422,6 +422,7 @@ def test_set_metadata(self): logger.info(br.ps_x) self.assertEqual(br.cnames[0], cname[0]) + class TestBioReaderPickle(unittest.TestCase): def test_python_backend_pickle(self): # create a 2D numpy array filled with random integer form 0-255 @@ -430,7 +431,7 @@ def test_python_backend_pickle(self): source_data = np.random.randint( 0, 256, (img_height, img_width), dtype=np.uint16 ) - + with bfio.BioWriter( str(TEST_DIR.joinpath("radnom_image.ome.tiff")), X=img_width, @@ -438,7 +439,7 @@ def test_python_backend_pickle(self): dtype=np.uint16, ) as bw: bw[0:img_height, 0:img_width, 0, 0, 0] = source_data - + br = bfio.BioReader(str(TEST_DIR.joinpath("radnom_image.ome.tiff"))) br.close() pickled_rdr = pickle.dumps(br) @@ -451,11 +452,11 @@ def test_python_backend_pickle(self): def test_bioformats_backend_pickle(self): br = bfio.BioReader(str(TEST_DIR.joinpath("img_r001_c001.ome.tif"))) - img_height = br.Y - img_width = br.X + img_height = br.Y + img_width = br.X data_sum = br[:].sum() br.close() - + pickled_rdr = pickle.dumps(br) unpickled_rdr = pickle.loads(pickled_rdr) From 8362c63c2191910d875dbea16ec33b7933527691 Mon Sep 17 00:00:00 2001 From: sameeul Date: Wed, 7 Aug 2024 19:26:00 -0400 Subject: [PATCH 5/6] Make TensorstoreReader pickle-able --- src/bfio/ts_backends.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/src/bfio/ts_backends.py b/src/bfio/ts_backends.py index a56964f..fe5e3c9 100644 --- a/src/bfio/ts_backends.py +++ b/src/bfio/ts_backends.py @@ -23,7 +23,19 @@ class TensorstoreReader(bfio.base_classes.TSAbstractReader): _rdr = None _offsets_bytes = None - _STATE_DICT = ["_metadata", "frontend"] + _STATE_DICT = [ + "_metadata", + "frontend", + "X", + "Y", + "Z", + "C", + "T", + "data_type", + "_file_path", + "_file_type", + "_axes_list", + ] def __init__(self, frontend): super().__init__(frontend) @@ -33,7 +45,8 @@ def __init__(self, frontend): if extension.endswith(".ome.tif") or extension.endswith(".ome.tiff"): # # check if it satisfies all the condition for python backend self._file_type = FileType.OmeTiff - self._rdr = TSReader(str(self.frontend._file_path), FileType.OmeTiff, "") + self._file_path = str(self.frontend._file_path.resolve()) + self._axes_list = "" elif extension.endswith(".zarr"): # if path exists, make sure it is a directory if not Path.is_dir(self.frontend._file_path): @@ -41,10 +54,10 @@ def __init__(self, frontend): "this filetype is not supported by tensorstore backend" ) else: - zarr_path, axes_list = self.get_zarr_array_info() + self._file_path, self._axes_list = self.get_zarr_array_info() self._file_type = FileType.OmeZarr - self._rdr = TSReader(zarr_path, FileType.OmeZarr, axes_list) + self._rdr = TSReader(self._file_path, self._file_type, self._axes_list) self.X = self._rdr._X self.Y = self._rdr._Y self.Z = self._rdr._Z @@ -141,16 +154,13 @@ def get_zarr_array_info(self): def __getstate__(self) -> Dict: state_dict = {n: getattr(self, n) for n in self._STATE_DICT} - state_dict.update({"file_path": self.frontend._file_path}) return state_dict def __setstate__(self, state) -> None: for k, v in state.items(): - if k == "file_path": - pass - else: - setattr(self, k, v) + setattr(self, k, v) + self._rdr = TSReader(self._file_path, self._file_type, self._axes_list) def read_metadata(self): From db6a06622c93f2e8c4dbfcd0120885c1f4ff8f67 Mon Sep 17 00:00:00 2001 From: sameeul Date: Thu, 8 Aug 2024 11:08:39 -0400 Subject: [PATCH 6/6] Update tests --- src/bfio/bfio.py | 8 +++- tests/test_read.py | 108 +++++++++++++++++++++------------------------ 2 files changed, 57 insertions(+), 59 deletions(-) diff --git a/src/bfio/bfio.py b/src/bfio/bfio.py index 3e6c225..9abe4a6 100644 --- a/src/bfio/bfio.py +++ b/src/bfio/bfio.py @@ -249,13 +249,17 @@ def __setstate__(self, state) -> None: assert all(n in self._STATE_DICT for n in state.keys()) assert all(n in state.keys() for n in self._STATE_DICT) + bioformats_backend = False for k, v in state.items(): if k == "_backend" and v == "JavaReaderDummy": - self._backend = backends.JavaReader(self) + bioformats_backend = True else: setattr(self, k, v) - self._backend.frontend = self + if bioformats_backend: + self._backend = backends.JavaReader(self) + else: + self._backend.frontend = self def __getitem__(self, keys: typing.Union[tuple, slice]) -> numpy.ndarray: """Image loading using numpy-like indexing. diff --git a/tests/test_read.py b/tests/test_read.py index 0576e5b..3b50b27 100644 --- a/tests/test_read.py +++ b/tests/test_read.py @@ -65,6 +65,19 @@ def setUpModule(): for z in range(br.Z): zf[t, c, z, :, :] = br[:, :, z, c, t] + # create a 2D numpy array filled with random integer form 0-255 + img_height = 8000 + img_width = 7500 + source_data = np.random.randint(0, 256, (img_height, img_width), dtype=np.uint16) + np.save(TEST_DIR.joinpath("random_image.npy"), source_data) + with bfio.BioWriter( + str(TEST_DIR.joinpath("random_image.ome.tiff")), + X=img_width, + Y=img_height, + dtype=np.uint16, + ) as bw: + bw[0:img_height, 0:img_width, 0, 0, 0] = source_data + def tearDownModule(): """Remove test images""" @@ -142,24 +155,10 @@ def test_read_tif_strip_python(self): I = br[:] def test_read_unaligned_tile_boundary_python(self): - # create a 2D numpy array filled with random integer form 0-255 - img_height = 8000 - img_width = 7500 - source_data = np.random.randint( - 0, 256, (img_height, img_width), dtype=np.uint16 - ) - with bfio.BioWriter( - str(TEST_DIR.joinpath("test_output.ome.tiff")), - X=img_width, - Y=img_height, - dtype=np.uint16, - ) as bw: - bw[0:img_height, 0:img_width, 0, 0, 0] = source_data - + source_data = np.load(TEST_DIR.joinpath("random_image.npy")) x_max = source_data.shape[0] y_max = source_data.shape[1] - - with bfio.BioReader(str(TEST_DIR.joinpath("test_output.ome.tiff"))) as test_br: + with bfio.BioReader(str(TEST_DIR.joinpath("random_image.ome.tiff"))) as br: for i in range(100): x_start = random.randint(0, x_max) y_start = random.randint(0, y_max) @@ -168,39 +167,25 @@ def test_read_unaligned_tile_boundary_python(self): x_end = x_start + x_step y_end = y_start + y_step - if x_end > x_max: x_end = x_max if y_end > y_max: y_end = y_max - test_data = test_br[x_start:x_end, y_start:y_end, ...] + test_data = br[x_start:x_end, y_start:y_end, ...] assert ( np.sum(source_data[x_start:x_end, y_start:y_end] - test_data) == 0 ) def test_read_unaligned_tile_boundary_tensorstore(self): - # create a 2D numpy array filled with random integer form 0-255 - img_height = 8000 - img_width = 7500 - source_data = np.random.randint( - 0, 256, (img_height, img_width), dtype=np.uint16 - ) - with bfio.BioWriter( - str(TEST_DIR.joinpath("test_output.ome.tiff")), - X=img_width, - Y=img_height, - dtype=np.uint16, - ) as bw: - bw[0:img_height, 0:img_width, 0, 0, 0] = source_data + source_data = np.load(TEST_DIR.joinpath("random_image.npy")) x_max = source_data.shape[0] y_max = source_data.shape[1] - with bfio.BioReader( - str(TEST_DIR.joinpath("test_output.ome.tiff")), backend="tensorstore" - ) as test_br: + str(TEST_DIR.joinpath("random_image.ome.tiff")), backend="tensorstore" + ) as br: for i in range(100): x_start = random.randint(0, x_max) y_start = random.randint(0, y_max) @@ -209,14 +194,13 @@ def test_read_unaligned_tile_boundary_tensorstore(self): x_end = x_start + x_step y_end = y_start + y_step - if x_end > x_max: x_end = x_max if y_end > y_max: y_end = y_max - test_data = test_br[x_start:x_end, y_start:y_end, ...] + test_data = br[x_start:x_end, y_start:y_end, ...] assert ( np.sum(source_data[x_start:x_end, y_start:y_end] - test_data) == 0 ) @@ -306,6 +290,21 @@ def test_sub_resolution_read(self): get_dims(br) self.assertEqual(br.shape, (1167, 404, 1, 3)) + def test_bioformats_backend_pickle(self): + + br = bfio.BioReader(str(TEST_DIR.joinpath("img_r001_c001.ome.tif"))) + img_height = br.Y + img_width = br.X + data_sum = br[:].sum() + pickled_rdr = pickle.dumps(br) + br.close() + unpickled_rdr = pickle.loads(pickled_rdr) + + assert unpickled_rdr.X == img_width + assert unpickled_rdr.Y == img_height + assert unpickled_rdr[:].sum() == data_sum + unpickled_rdr.close() + class TestZarrReader(unittest.TestCase): def test_get_dims(self): @@ -424,42 +423,37 @@ def test_set_metadata(self): class TestBioReaderPickle(unittest.TestCase): - def test_python_backend_pickle(self): - # create a 2D numpy array filled with random integer form 0-255 - img_height = 1024 - img_width = 1024 - source_data = np.random.randint( - 0, 256, (img_height, img_width), dtype=np.uint16 - ) - with bfio.BioWriter( - str(TEST_DIR.joinpath("radnom_image.ome.tiff")), - X=img_width, - Y=img_height, - dtype=np.uint16, - ) as bw: - bw[0:img_height, 0:img_width, 0, 0, 0] = source_data + def test_python_backend_pickle(self): - br = bfio.BioReader(str(TEST_DIR.joinpath("radnom_image.ome.tiff"))) - br.close() + br = bfio.BioReader( + str(TEST_DIR.joinpath("random_image.ome.tiff")), backend="python" + ) + img_height = br.Y + img_width = br.X + data_sum = br[:].sum() pickled_rdr = pickle.dumps(br) + br.close() unpickled_rdr = pickle.loads(pickled_rdr) assert unpickled_rdr.X == img_width assert unpickled_rdr.Y == img_height - assert unpickled_rdr[:].sum() == source_data.sum() + assert unpickled_rdr[:].sum() == data_sum + unpickled_rdr.close() - def test_bioformats_backend_pickle(self): + def test_tensorstore_backend_pickle(self): - br = bfio.BioReader(str(TEST_DIR.joinpath("img_r001_c001.ome.tif"))) + br = bfio.BioReader( + str(TEST_DIR.joinpath("random_image.ome.tiff")), backend="tensorstore" + ) img_height = br.Y img_width = br.X data_sum = br[:].sum() - br.close() - pickled_rdr = pickle.dumps(br) + br.close() unpickled_rdr = pickle.loads(pickled_rdr) assert unpickled_rdr.X == img_width assert unpickled_rdr.Y == img_height assert unpickled_rdr[:].sum() == data_sum + unpickled_rdr.close()