From 8aa687c82d17d9ea4e6a023105cc2f77ef440b8c Mon Sep 17 00:00:00 2001 From: rhoadesScholar Date: Wed, 29 Jan 2025 14:20:54 -0500 Subject: [PATCH 01/15] Force script variable definitions to usable type. --- cellmap_flow/inferencer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cellmap_flow/inferencer.py b/cellmap_flow/inferencer.py index 1d24c9c..80ebf0a 100644 --- a/cellmap_flow/inferencer.py +++ b/cellmap_flow/inferencer.py @@ -166,10 +166,10 @@ def load_bio_model(self, bio_model_name): def load_script_model(self, model_config: ScriptModelConfig): config = model_config.config self.model = config.model - self.read_shape = config.read_shape - self.write_shape = config.write_shape - self.output_voxel_size = config.output_voxel_size - self.context = (self.read_shape - self.write_shape) / 2 + self.read_shape = np.array(config.read_shape) + self.write_shape = np.array(config.write_shape) + self.output_voxel_size = np.array(config.output_voxel_size) + self.context = (self.read_shape - self.write_shape) // 2 # %% From 8d7ffb5d6bb08db4482d941fe169e6b3264fd4b9 Mon Sep 17 00:00:00 2001 From: rhoadesScholar Date: Wed, 29 Jan 2025 15:40:09 -0500 Subject: [PATCH 02/15] Change log level from error to info when opening dataset files --- cellmap_flow/image_data_interface.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cellmap_flow/image_data_interface.py b/cellmap_flow/image_data_interface.py index 436e7e5..8d938ce 100644 --- a/cellmap_flow/image_data_interface.py +++ b/cellmap_flow/image_data_interface.py @@ -271,7 +271,7 @@ def get_ds_info(path): roi = Roi([0] * len(shape), Coordinate(shape) * voxel_size) else: path, filename = split_dataset_path(path) - logger.error(f"Opening {path} {filename}") + logger.info(f"Opening {path} {filename}") ds = open_ds(path, filename) voxel_size = ds.voxel_size chunk_shape = ds.chunk_shape @@ -327,5 +327,3 @@ def to_ndarray_ts(self, roi=None): ) self.ts = None return res - - From 16d7b2395a69394f2b154fbda219a39448240df5 Mon Sep 17 00:00:00 2001 From: rhoadesScholar Date: Wed, 29 Jan 2025 15:41:33 -0500 Subject: [PATCH 03/15] Refactor server_check function to accept script path and dataset as arguments; add command-line interface for execution. --- example/server_check.py | 44 ++++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/example/server_check.py b/example/server_check.py index 04a6e3a..bc8e9ba 100644 --- a/example/server_check.py +++ b/example/server_check.py @@ -1,20 +1,32 @@ -#%% +# %% from cellmap_flow.server import CellMapFlowServer -from cellmap_flow.utils.data import ModelConfig, BioModelConfig, DaCapoModelConfig, ScriptModelConfig -#%% -# dataset = "/nrs/cellmap/data/jrc_mus-cerebellum-1/jrc_mus-cerebellum-1.zarr/recon-1/em/fibsem-uint8/s0" -# script_path = "/groups/cellmap/cellmap/zouinkhim/cellmap-flow/example/model_setup04.py" -# script_path = "/groups/cellmap/cellmap/zouinkhim/cellmap-flow/example/dacapo_run_retrieve.py" +from cellmap_flow.utils.data import ( + ModelConfig, + BioModelConfig, + DaCapoModelConfig, + ScriptModelConfig, +) +import argparse -script_path = "/groups/cellmap/cellmap/zouinkhim/cellmap-flow/example/model_spec.py" -dataset = "/groups/cellmap/cellmap/ackermand/for_hideo/jrc_pri_neuron_0710Dish4/jrc_pri_neuron_0710Dish4.n5/em/fibsem-uint8/s0" -model_config = ScriptModelConfig(script_path=script_path) -server = CellMapFlowServer(dataset, model_config) -# %% -chunk_x = 2 -chunk_y = 2 -chunk_z = 2 +def server_check(script_path, dataset): + model_config = ScriptModelConfig(script_path=script_path) + server = CellMapFlowServer(dataset, model_config) + chunk_x = 2 + chunk_y = 2 + chunk_z = 2 -server.chunk(None, None, chunk_x, chunk_y, chunk_z, None) -# %% + server._chunk_impl(None, None, chunk_x, chunk_y, chunk_z, None) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Test run a CellMapFlow server") + parser.add_argument( + "--script_path", + "-s", + type=str, + help="Path to the Python script containing model specification", + ) + parser.add_argument("--dataset", "-d", type=str, help="Path to the dataset") + args = parser.parse_args() + server_check(args.script_path, args.dataset) From fb561544f18c8c53334d925bc0c714e932aaef04 Mon Sep 17 00:00:00 2001 From: rhoadesScholar Date: Wed, 29 Jan 2025 15:50:35 -0500 Subject: [PATCH 04/15] Add confirmation message for successful server check --- example/server_check.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/example/server_check.py b/example/server_check.py index bc8e9ba..ca03753 100644 --- a/example/server_check.py +++ b/example/server_check.py @@ -18,6 +18,8 @@ def server_check(script_path, dataset): server._chunk_impl(None, None, chunk_x, chunk_y, chunk_z, None) + print("Server check passed") + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Test run a CellMapFlow server") From 8104ef1cec044ea388d38c29e129294f44eee969 Mon Sep 17 00:00:00 2001 From: rhoadesScholar Date: Wed, 29 Jan 2025 16:02:19 -0500 Subject: [PATCH 05/15] Add script server check command to CLI for model validation --- cellmap_flow/cli/cli.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/cellmap_flow/cli/cli.py b/cellmap_flow/cli/cli.py index 4665b94..784c5b3 100644 --- a/cellmap_flow/cli/cli.py +++ b/cellmap_flow/cli/cli.py @@ -2,7 +2,9 @@ import logging import click +from cellmap_flow.server import CellMapFlowServer from cellmap_flow.utils.bsub_utils import start_hosts +from cellmap_flow.utils.data import ScriptModelConfig from cellmap_flow.utils.neuroglancer_utils import generate_neuroglancer_link @@ -152,6 +154,26 @@ def bioimage(model_path, data_path, queue, charge_group): run(command, data_path, queue, charge_group) +@cli.command() +@click.option( + "--script_path", + "-s", + type=str, + help="Path to the Python script containing model specification", +) +@click.option("--dataset", "-d", type=str, help="Path to the dataset") +def script_server_check(script_path, dataset): + model_config = ScriptModelConfig(script_path=script_path) + server = CellMapFlowServer(dataset, model_config) + chunk_x = 2 + chunk_y = 2 + chunk_z = 2 + + server._chunk_impl(None, None, chunk_x, chunk_y, chunk_z, None) + + print("Server check passed") + + def run( command, dataset_path, From ebcaa5fa9662407917c888cc1c8da14202249788 Mon Sep 17 00:00:00 2001 From: rhoadesScholar Date: Wed, 29 Jan 2025 16:08:00 -0500 Subject: [PATCH 06/15] Revert server_check.py to use hardcoded script path and dataset; add print statement for successful completion --- example/server_check.py | 34 +++++++++++++--------------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/example/server_check.py b/example/server_check.py index ca03753..aa83fb0 100644 --- a/example/server_check.py +++ b/example/server_check.py @@ -6,29 +6,21 @@ DaCapoModelConfig, ScriptModelConfig, ) -import argparse +# %% +# dataset = "/nrs/cellmap/data/jrc_mus-cerebellum-1/jrc_mus-cerebellum-1.zarr/recon-1/em/fibsem-uint8/s0" +# script_path = "/groups/cellmap/cellmap/zouinkhim/cellmap-flow/example/model_setup04.py" +# script_path = "/groups/cellmap/cellmap/zouinkhim/cellmap-flow/example/dacapo_run_retrieve.py" -def server_check(script_path, dataset): - model_config = ScriptModelConfig(script_path=script_path) - server = CellMapFlowServer(dataset, model_config) - chunk_x = 2 - chunk_y = 2 - chunk_z = 2 - - server._chunk_impl(None, None, chunk_x, chunk_y, chunk_z, None) +script_path = "/groups/cellmap/cellmap/zouinkhim/cellmap-flow/example/model_spec.py" +dataset = "/groups/cellmap/cellmap/ackermand/for_hideo/jrc_pri_neuron_0710Dish4/jrc_pri_neuron_0710Dish4.n5/em/fibsem-uint8/s0" - print("Server check passed") +model_config = ScriptModelConfig(script_path=script_path) +server = CellMapFlowServer(dataset, model_config) +chunk_x = 2 +chunk_y = 2 +chunk_z = 2 +server._chunk_impl(None, None, chunk_x, chunk_y, chunk_z, None) -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Test run a CellMapFlow server") - parser.add_argument( - "--script_path", - "-s", - type=str, - help="Path to the Python script containing model specification", - ) - parser.add_argument("--dataset", "-d", type=str, help="Path to the dataset") - args = parser.parse_args() - server_check(args.script_path, args.dataset) +print("Server check passed") From 36842e56a11b62938d9a680b7cec9d5419465d9a Mon Sep 17 00:00:00 2001 From: rhoadesScholar Date: Wed, 29 Jan 2025 16:13:07 -0500 Subject: [PATCH 07/15] Fix model_setup04.py and server_check.py for improved readability and path handling --- example/model_setup04.py | 13 +++++++------ example/server_check.py | 11 +++++++---- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/example/model_setup04.py b/example/model_setup04.py index c41bee0..ad94405 100644 --- a/example/model_setup04.py +++ b/example/model_setup04.py @@ -1,21 +1,22 @@ -#%% +# %% # pip install fly-organelles from daisy.coordinate import Coordinate import numpy as np output_voxel_size = Coordinate((4, 4, 4)) -voxel_size = Coordinate((8, 8, 8)) +input_voxel_size = voxel_size = Coordinate((8, 8, 8)) read_shape = Coordinate((216, 216, 216)) * Coordinate(voxel_size) write_shape = Coordinate((68, 68, 68)) * Coordinate(output_voxel_size) -#%% +# %% import torch import cellmap_models.pytorch.cosem as cosem_models -model = cosem_models.load_model('setup04/1820500') -#%% +model = cosem_models.load_model("setup04/1820500") + +# %% if torch.cuda.is_available(): device = torch.device("cuda") else: @@ -26,5 +27,5 @@ output_channels = 14 # 0:all_mem,1:organelle,2:mito,3:er,4:nucleus,5:pm,6:vs,7:ld -block_shape = np.array((68, 68, 68,14)) +block_shape = np.array((68, 68, 68, 14)) # # %% diff --git a/example/server_check.py b/example/server_check.py index aa83fb0..d5919de 100644 --- a/example/server_check.py +++ b/example/server_check.py @@ -6,14 +6,15 @@ DaCapoModelConfig, ScriptModelConfig, ) +import os # %% -# dataset = "/nrs/cellmap/data/jrc_mus-cerebellum-1/jrc_mus-cerebellum-1.zarr/recon-1/em/fibsem-uint8/s0" -# script_path = "/groups/cellmap/cellmap/zouinkhim/cellmap-flow/example/model_setup04.py" +# dataset = "/groups/cellmap/cellmap/ackermand/for_hideo/jrc_pri_neuron_0710Dish4/jrc_pri_neuron_0710Dish4.n5/em/fibsem-uint8/s0" +# script_path = "/groups/cellmap/cellmap/zouinkhim/cellmap-flow/example/model_spec.py" # script_path = "/groups/cellmap/cellmap/zouinkhim/cellmap-flow/example/dacapo_run_retrieve.py" -script_path = "/groups/cellmap/cellmap/zouinkhim/cellmap-flow/example/model_spec.py" -dataset = "/groups/cellmap/cellmap/ackermand/for_hideo/jrc_pri_neuron_0710Dish4/jrc_pri_neuron_0710Dish4.n5/em/fibsem-uint8/s0" +script_path = os.path.join(os.path.dirname(__file__), "model_setup04.py") +dataset = "/nrs/cellmap/data/jrc_mus-cerebellum-1/jrc_mus-cerebellum-1.zarr/recon-1/em/fibsem-uint8/s0" model_config = ScriptModelConfig(script_path=script_path) server = CellMapFlowServer(dataset, model_config) @@ -24,3 +25,5 @@ server._chunk_impl(None, None, chunk_x, chunk_y, chunk_z, None) print("Server check passed") + +# %% From 9bc8706dc802ac0836c9d9b56adc49d41c84ec83 Mon Sep 17 00:00:00 2001 From: Marwan Zouinkhi Date: Thu, 30 Jan 2025 00:29:37 -0500 Subject: [PATCH 08/15] fix typo --- cellmap_flow/image_data_interface.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cellmap_flow/image_data_interface.py b/cellmap_flow/image_data_interface.py index e70bb71..5d1267c 100644 --- a/cellmap_flow/image_data_interface.py +++ b/cellmap_flow/image_data_interface.py @@ -51,5 +51,4 @@ def to_ndarray_ts(self, roi=None): self.swap_axes, self.custom_fill_value, ) - self.ts = None return res From 786f16a9d7e8d920697284ad564fe7864a6c5405 Mon Sep 17 00:00:00 2001 From: Marwan Zouinkhi Date: Fri, 31 Jan 2025 10:50:39 -0500 Subject: [PATCH 09/15] minor fix --- cellmap_flow/utils/neuroglancer_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cellmap_flow/utils/neuroglancer_utils.py b/cellmap_flow/utils/neuroglancer_utils.py index 25891d9..354e2c7 100644 --- a/cellmap_flow/utils/neuroglancer_utils.py +++ b/cellmap_flow/utils/neuroglancer_utils.py @@ -9,7 +9,7 @@ from cellmap_flow.image_data_interface import ImageDataInterface # TODO support multiresolution datasets def get_raw_layer(dataset_path, filetype): - if filetype == "zarr": + if filetype == "n5": axis = ["x", "y", "z"] else: axis = ["z", "y", "x"] From b4586d14dc1710e4c2e1a3ad8cc36c2145b532b3 Mon Sep 17 00:00:00 2001 From: Marwan Zouinkhi Date: Fri, 31 Jan 2025 11:50:37 -0500 Subject: [PATCH 10/15] support multiscale --- cellmap_flow/image_data_interface.py | 1 - cellmap_flow/inferencer.py | 13 ++- cellmap_flow/utils/neuroglancer_utils.py | 65 +++++++++---- cellmap_flow/utils/scale_pyramid.py | 113 +++++++++++++++++++++++ 4 files changed, 170 insertions(+), 22 deletions(-) create mode 100644 cellmap_flow/utils/scale_pyramid.py diff --git a/cellmap_flow/image_data_interface.py b/cellmap_flow/image_data_interface.py index 5d1267c..67ee753 100644 --- a/cellmap_flow/image_data_interface.py +++ b/cellmap_flow/image_data_interface.py @@ -31,7 +31,6 @@ def __init__( self.output_voxel_size = output_voxel_size else: self.output_voxel_size = self.voxel_size - @property def ts(self): diff --git a/cellmap_flow/inferencer.py b/cellmap_flow/inferencer.py index 8b463e7..067fe90 100644 --- a/cellmap_flow/inferencer.py +++ b/cellmap_flow/inferencer.py @@ -28,7 +28,7 @@ def predict(read_roi, write_roi, config, **kwargs): device = kwargs.get("device") if device is None: raise ValueError("device must be provided in kwargs") - + use_half_prediction = kwargs.get("use_half_prediction", False) raw_input = idi.to_ndarray_ts(read_roi) @@ -54,7 +54,7 @@ def __init__(self, model_config: ModelConfig, use_half_prediction=False): self.model_config.config, "write_shape" ): self.context = ( - Coordinate(self.model_config.config.read_shape) + Coordinate(self.model_config.config.read_shape) - Coordinate(self.model_config.config.write_shape) ) / 2 @@ -115,7 +115,12 @@ def process_chunk_basic(self, idi, roi): input_roi = output_roi.grow(self.context, self.context) result = self.model_config.config.predict( - input_roi, output_roi, self.model_config.config, idi=idi, device=self.device, use_half_prediction=self.use_half_prediction + input_roi, + output_roi, + self.model_config.config, + idi=idi, + device=self.device, + use_half_prediction=self.use_half_prediction, ) write_data = self.model_config.config.normalize_output(result) @@ -175,4 +180,4 @@ def process_chunk_bioimagezoo(self, idi, roi): output = 255 * output output = output.astype(np.uint8) - return output \ No newline at end of file + return output diff --git a/cellmap_flow/utils/neuroglancer_utils.py b/cellmap_flow/utils/neuroglancer_utils.py index 354e2c7..80cf6a2 100644 --- a/cellmap_flow/utils/neuroglancer_utils.py +++ b/cellmap_flow/utils/neuroglancer_utils.py @@ -1,28 +1,55 @@ import neuroglancer import itertools import logging +import os neuroglancer.set_server_bind_address("0.0.0.0") logger = logging.getLogger(__name__) from cellmap_flow.image_data_interface import ImageDataInterface +from cellmap_flow.utils.scale_pyramid import ScalePyramid + + # TODO support multiresolution datasets -def get_raw_layer(dataset_path, filetype): +def get_raw_layer(dataset_path, filetype, is_multiscale=False): if filetype == "n5": axis = ["x", "y", "z"] else: axis = ["z", "y", "x"] - image = ImageDataInterface(dataset_path) - return neuroglancer.ImageLayer( - source=neuroglancer.LocalVolume( - data=image.ts, - dimensions=neuroglancer.CoordinateSpace( - names=axis, - units="nm", - scales=image.voxel_size, - ), - voxel_offset=image.offset, + + layers = [] + + if is_multiscale: + scales = [ + f for f in os.listdir(dataset_path) if f[0] == "s" and f[1:].isdigit() + ] + scales.sort(key=lambda x: int(x[1:])) + for scale in scales: + image = ImageDataInterface(f"{os.path.join(dataset_path, scale)}") + layers.append( + neuroglancer.LocalVolume( + data=image.ts, + dimensions=neuroglancer.CoordinateSpace( + names=axis, + units="nm", + scales=image.voxel_size, + ), + voxel_offset=image.offset, + ) + ) + return ScalePyramid(layers) + else: + image = ImageDataInterface(dataset_path) + return neuroglancer.ImageLayer( + source=neuroglancer.LocalVolume( + data=image.ts, + dimensions=neuroglancer.CoordinateSpace( + names=axis, + units="nm", + scales=image.voxel_size, + ), + voxel_offset=image.offset, ) ) @@ -33,12 +60,15 @@ def generate_neuroglancer_link(dataset_path, inference_dict): # Add a layer to the viewer with viewer.txn() as s: + is_multi_scale = False # if multiscale dataset - # if ( - # dataset_path.split("/")[-1].startswith("s") - # and dataset_path.split("/")[-1][1:].isdigit() - # ): - # dataset_path = dataset_path.rsplit("/", 1)[0] + if ( + dataset_path.split("/")[-1].startswith("s") + and dataset_path.split("/")[-1][1:].isdigit() + ): + dataset_path = dataset_path.rsplit("/", 1)[0] + is_multi_scale = True + if ".zarr" in dataset_path: filetype = "zarr" elif ".n5" in dataset_path: @@ -46,7 +76,8 @@ def generate_neuroglancer_link(dataset_path, inference_dict): else: filetype = "precomputed" if dataset_path.startswith("/"): - s.layers["raw"] = get_raw_layer(dataset_path, filetype) + layer = get_raw_layer(dataset_path, filetype, is_multi_scale) + s.layers.append("raw", layer) # if "nrs/cellmap" in dataset_path: # security = "https" # dataset_path = dataset_path.replace("/nrs/cellmap/", "nrs/") diff --git a/cellmap_flow/utils/scale_pyramid.py b/cellmap_flow/utils/scale_pyramid.py new file mode 100644 index 0000000..dbe7928 --- /dev/null +++ b/cellmap_flow/utils/scale_pyramid.py @@ -0,0 +1,113 @@ +# copied from https://github.com/funkelab/funlib.show.neuroglancer/blob/master/funlib/show/neuroglancer/scale_pyramid.py + +import neuroglancer +import operator +import logging + +import numpy as np + + +logger = logging.getLogger(__name__) + + +class ScalePyramid(neuroglancer.LocalVolume): + """A neuroglancer layer that provides volume data on different scales. + Mimics a LocalVolume. + + Args: + + volume_layers (``list`` of ``LocalVolume``): + + One ``LocalVolume`` per provided resolution. + """ + + def __init__(self, volume_layers): + volume_layers = volume_layers + + super(neuroglancer.LocalVolume, self).__init__() + + logger.info("Creating scale pyramid...") + + self.min_voxel_size = min( + [tuple(layer.dimensions.scales) for layer in volume_layers] + ) + self.max_voxel_size = max( + [tuple(layer.dimensions.scales) for layer in volume_layers] + ) + + self.dims = len(volume_layers[0].dimensions.scales) + self.volume_layers = { + tuple( + int(x) + for x in map( + operator.truediv, layer.dimensions.scales, self.min_voxel_size + ) + ): layer + for layer in volume_layers + } + + logger.info("min_voxel_size: %s", self.min_voxel_size) + logger.info("scale keys: %s", self.volume_layers.keys()) + logger.info(self.info()) + + @property + def volume_type(self): + return self.volume_layers[(1,) * self.dims].volume_type + + @property + def token(self): + return self.volume_layers[(1,) * self.dims].token + + def info(self): + reference_layer = self.volume_layers[(1,) * self.dims] + # return reference_layer.info() + + reference_info = reference_layer.info() + + info = { + "dataType": reference_info["dataType"], + "encoding": reference_info["encoding"], + "generation": reference_info["generation"], + "coordinateSpace": reference_info["coordinateSpace"], + "shape": reference_info["shape"], + "volumeType": reference_info["volumeType"], + "voxelOffset": reference_info["voxelOffset"], + "chunkLayout": reference_info["chunkLayout"], + "downsamplingLayout": reference_info["downsamplingLayout"], + "maxDownsampling": int( + np.prod(np.array(self.max_voxel_size) // np.array(self.min_voxel_size)) + ), + "maxDownsampledSize": reference_info["maxDownsampledSize"], + "maxDownsamplingScales": reference_info["maxDownsamplingScales"], + } + + return info + + def get_encoded_subvolume(self, data_format, start, end, scale_key=None): + if scale_key is None: + scale_key = ",".join(("1",) * self.dims) + + scale = tuple(int(s) for s in scale_key.split(",")) + closest_scale = None + min_diff = np.inf + for volume_scales in self.volume_layers.keys(): + scale_diff = np.array(scale) // np.array(volume_scales) + if any(scale_diff < 1): + continue + scale_diff = scale_diff.max() + if scale_diff < min_diff: + min_diff = scale_diff + closest_scale = volume_scales + + assert closest_scale is not None + relative_scale = np.array(scale) // np.array(closest_scale) + + return self.volume_layers[closest_scale].get_encoded_subvolume( + data_format, start, end, scale_key=",".join(map(str, relative_scale)) + ) + + def get_object_mesh(self, object_id): + return self.volume_layers[(1,) * self.dims].get_object_mesh(object_id) + + def invalidate(self): + return self.volume_layers[(1,) * self.dims].invalidate() From cca3e50021e8d3047de6f084689780564ab04964 Mon Sep 17 00:00:00 2001 From: Marwan Zouinkhi Date: Fri, 31 Jan 2025 12:15:15 -0500 Subject: [PATCH 11/15] update norm --- cellmap_flow/norm/input_normalize.py | 4 ++-- example/check_norm.py | 26 ++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/cellmap_flow/norm/input_normalize.py b/cellmap_flow/norm/input_normalize.py index 1c7d585..fd0a2e7 100644 --- a/cellmap_flow/norm/input_normalize.py +++ b/cellmap_flow/norm/input_normalize.py @@ -33,13 +33,13 @@ def __init__(self, min_value=0.0, max_value=255.0): def normalize(self, data: np.ndarray) -> np.ndarray: data = data.astype(np.float32) - data.clip(self.min_value, self.max_value) + data = data.clip(self.min_value, self.max_value) return ((data - self.min_value) / (self.max_value - self.min_value)).astype( np.float32 ) -NormalizationMethods = [f.name() for f in InputNormalizer.__subclasses__()] +NormalizationMethods = [f for f in InputNormalizer.__subclasses__()] def get_normalization(elms: dict) -> InputNormalizer: diff --git a/example/check_norm.py b/example/check_norm.py index e69de29..7aab4bd 100644 --- a/example/check_norm.py +++ b/example/check_norm.py @@ -0,0 +1,26 @@ +#%% +import cellmap_flow.norm.input_normalize +from importlib import reload +reload(cellmap_flow.norm.input_normalize) +from cellmap_flow.norm.input_normalize import MinMaxNormalizer, NormalizationMethods, get_normalization, InputNormalizer +# %% +NormalizationMethods +# %% +dic = { "name": "min_max", "min_value": 0, "max_value": 1} +# %% +get_normalization(dic) +# %% +class TINput(InputNormalizer): + @classmethod + def name(cls): + return "tin" + +# %% +NormalizationMethods = [f.name() for f in InputNormalizer.__subclasses__()] +# %% +NormalizationMethods +# %% +get_normalization({"name": "tin"}) +# %% +NormalizationMethods +# %% From 9c75ef7b008a8246f35cc739705d09d50cf4905b Mon Sep 17 00:00:00 2001 From: Marwan Zouinkhi Date: Fri, 31 Jan 2025 13:21:18 -0500 Subject: [PATCH 12/15] add needed deps --- pyproject.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 634e3f5..79d648e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,12 @@ dependencies = [ "marshmallow", "scikit-image", "flasgger", + "numcodecs", + "zarr==2.18.4", + "h5py", + "torch", + "universal_pathlib", + "neuroglancer", ] # extras From 5c9f6e8d9ab487714f8c7fe89bcf3291cabed3d3 Mon Sep 17 00:00:00 2001 From: David Ackerman Date: Wed, 5 Feb 2025 14:12:59 -0500 Subject: [PATCH 13/15] fix: :bug: fix issues where non-uniform block_shapes are used --- cellmap_flow/server.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/cellmap_flow/server.py b/cellmap_flow/server.py index bfdf599..98d0b2a 100644 --- a/cellmap_flow/server.py +++ b/cellmap_flow/server.py @@ -30,7 +30,8 @@ def __init__(self, dataset_name: str, model_config: ModelConfig): """ Initialize the server and set up routes via decorators. """ - self.block_shape = [int(x) for x in model_config.config.block_shape] + self.n5_block_shape = [int(x) for x in model_config.config.block_shape] + self.read_block_shape = self.n5_block_shape.copy() self.input_voxel_size = Coordinate(model_config.config.input_voxel_size) self.output_voxel_size = Coordinate(model_config.config.output_voxel_size) self.output_channels = model_config.config.output_channels @@ -41,6 +42,12 @@ def __init__(self, dataset_name: str, model_config: ModelConfig): self.idi_raw = ImageDataInterface( dataset_name, target_resolution=self.input_voxel_size ) + + # we want our chunk shapes to be swapped to correspond to the zyx of tensorstore reads? + self.n5_block_shape[0], self.n5_block_shape[2] = ( + self.n5_block_shape[2], + self.n5_block_shape[0], + ) if ".zarr" in dataset_name: # Convert from (z, y, x) -> (x, y, z) plus channels self.vol_shape = np.array( @@ -56,7 +63,7 @@ def __init__(self, dataset_name: str, model_config: ModelConfig): # Chunk encoding for N5 self.chunk_encoder = N5ChunkWrapper( - np.uint8, self.block_shape, compressor=numcodecs.GZip() + np.uint8, self.n5_block_shape, compressor=numcodecs.GZip() ) # Create and configure Flask @@ -269,7 +276,7 @@ def _attributes_impl(self, dataset, scale): "translate": [0.0, 0.0, 0.0, 0.0], }, "compression": {"type": "gzip", "useZlib": False, "level": -1}, - "blockSize": list(self.block_shape), + "blockSize": list(self.n5_block_shape), "dataType": "uint8", "dimensions": self.vol_shape.tolist(), } @@ -292,8 +299,8 @@ def _input_normalize_impl(self, norm_type, min_value, max_value): ) def _chunk_impl(self, dataset, scale, chunk_x, chunk_y, chunk_z, chunk_c): - corner = self.block_shape[:3] * np.array([chunk_z, chunk_y, chunk_x]) - box = np.array([corner, self.block_shape[:3]]) * self.output_voxel_size + corner = self.read_block_shape[:3] * np.array([chunk_z, chunk_y, chunk_x]) + box = np.array([corner, self.read_block_shape[:3]]) * self.output_voxel_size roi = Roi(box[0], box[1]) chunk_data = self.inferencer.process_chunk(self.idi_raw, roi) return ( From 3d145a227f666fde9272f41da213938bee24a4f1 Mon Sep 17 00:00:00 2001 From: David Ackerman Date: Wed, 5 Feb 2025 14:26:58 -0500 Subject: [PATCH 14/15] refactor: :recycle: move stuff around --- cellmap_flow/server.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/cellmap_flow/server.py b/cellmap_flow/server.py index 98d0b2a..02bae95 100644 --- a/cellmap_flow/server.py +++ b/cellmap_flow/server.py @@ -30,8 +30,17 @@ def __init__(self, dataset_name: str, model_config: ModelConfig): """ Initialize the server and set up routes via decorators. """ - self.n5_block_shape = [int(x) for x in model_config.config.block_shape] - self.read_block_shape = self.n5_block_shape.copy() + + # this is zyx + self.read_block_shape = [int(x) for x in model_config.config.block_shape] + + # this needs to have z and x swapped + self.n5_block_shape = self.read_block_shape.copy() + self.n5_block_shape[0], self.n5_block_shape[2] = ( + self.n5_block_shape[2], + self.n5_block_shape[0], + ) + self.input_voxel_size = Coordinate(model_config.config.input_voxel_size) self.output_voxel_size = Coordinate(model_config.config.output_voxel_size) self.output_channels = model_config.config.output_channels @@ -43,11 +52,6 @@ def __init__(self, dataset_name: str, model_config: ModelConfig): dataset_name, target_resolution=self.input_voxel_size ) - # we want our chunk shapes to be swapped to correspond to the zyx of tensorstore reads? - self.n5_block_shape[0], self.n5_block_shape[2] = ( - self.n5_block_shape[2], - self.n5_block_shape[0], - ) if ".zarr" in dataset_name: # Convert from (z, y, x) -> (x, y, z) plus channels self.vol_shape = np.array( From 70f1fd8d929c4fc91ba5b6a10052a30a1d53392f Mon Sep 17 00:00:00 2001 From: David Ackerman Date: Thu, 6 Feb 2025 14:04:57 -0500 Subject: [PATCH 15/15] fix: :bug: adjust shape appropriately when input and output resolutions differ --- cellmap_flow/server.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cellmap_flow/server.py b/cellmap_flow/server.py index 02bae95..c0dd967 100644 --- a/cellmap_flow/server.py +++ b/cellmap_flow/server.py @@ -51,18 +51,24 @@ def __init__(self, dataset_name: str, model_config: ModelConfig): self.idi_raw = ImageDataInterface( dataset_name, target_resolution=self.input_voxel_size ) + output_shape = ( + np.array(self.idi_raw.shape) + * np.array(self.input_voxel_size) + / np.array(self.output_voxel_size) + ) if ".zarr" in dataset_name: # Convert from (z, y, x) -> (x, y, z) plus channels self.vol_shape = np.array( - [*np.array(self.idi_raw.shape)[::-1], self.output_channels] + [ + *output_shape[::-1], + self.output_channels, + ] ) self.axis = ["x", "y", "z", "c^"] else: # For non-Zarr data - self.vol_shape = np.array( - [*np.array(self.idi_raw.shape), self.output_channels] - ) + self.vol_shape = np.array([*output_shape, self.output_channels]) self.axis = ["z", "y", "x", "c^"] # Chunk encoding for N5