update http_tensorstore

Models.md

@@ -0,0 +1,13 @@
+# To run existing model need to do :
+```bash
+cellmap_flow cellmap-model -n NAME -f /nrs/cellmap/models/[GROUP]/[MODEL_NAME] -P Billing_group -d DATA_PATH
+```
+
+# Existing models:
+- Cellmap
+    - jrc_mus-livers_16nm_to_8nm_mito
+    - jrc_mus-livers_16nm_to_8nm_peroxisome
+- saalfeldlab
+    - fly_organelles_run07_432000
+    - fly_organelles_run07_700000
+    - fly_organelles_run08_438000

README.md

@@ -13,6 +13,20 @@ Please feel free to explore and contribute, but note that there may be frequent
 
 </div>
 
+## Prerequisites
+
+This package requires Rust and Cargo to compile native extensions. Please ensure you have Rust installed before proceeding.
+
+1. **Install Rust:**
+Visit [rustup.rs](https://rustup.rs/) and follow the instructions to install Rust. This installation will also include Cargo, Rust’s package manager.
+
+2. **Verify Cargo Installation:**
+Open your terminal and run:
+     ```bash
+     cargo --version
+     ```
+You should see the Cargo version output, which confirms that Cargo is installed and accessible.
+
 
 ## Installation
 

cellmap_flow/cli/cli.py

@@ -2,18 +2,17 @@
 import logging
 import click
 
-from cellmap_flow.utils.bsub_utils import start_hosts
-from cellmap_flow.utils.neuroglancer_utils import generate_neuroglancer_link
+from cellmap_flow.server import CellMapFlowServer
+from cellmap_flow.utils.bsub_utils import start_hosts, SERVER_COMMAND
+from cellmap_flow.utils.data import ScriptModelConfig
+from cellmap_flow.utils.neuroglancer_utils import generate_neuroglancer_url
 
 
 logging.basicConfig()
 
 logger = logging.getLogger(__name__)
 
 
-SERVER_COMMAND = "cellmap_flow_server"
-
-
 @click.group()
 @click.option(
     "--log-level",
@@ -83,12 +82,7 @@ def cli(log_level):
 )
 def dacapo(run_name, iteration, data_path, queue, charge_group):
     command = f"{SERVER_COMMAND} dacapo -r {run_name} -i {iteration} -d {data_path}"
-    run(
-        command,
-        data_path,
-        queue,
-        charge_group,
-    )
+    run(command, data_path, queue, charge_group, run_name)
     raise NotImplementedError("This command is not yet implemented.")
 
 
@@ -121,7 +115,8 @@ def dacapo(run_name, iteration, data_path, queue, charge_group):
 )
 def script(script_path, data_path, queue, charge_group):
     command = f"{SERVER_COMMAND} script -s {script_path} -d {data_path}"
-    run(command, data_path, queue, charge_group)
+    base_name = script_path.split("/")[-1].split(".")[0]
+    run(command, data_path, queue, charge_group, base_name)
 
 
 @cli.command()
@@ -131,6 +126,13 @@ def script(script_path, data_path, queue, charge_group):
 @click.option(
     "-d", "--data_path", required=True, type=str, help="The path to the data."
 )
+@click.option(
+    "-e",
+    "--edge_length_to_process",
+    required=False,
+    type=int,
+    help="For 2D models, the desired edge length of the chunk to process; batch size (z) will be adjusted to match as close as possible.",
+)
 @click.option(
     "-q",
     "--queue",
@@ -147,22 +149,70 @@ def script(script_path, data_path, queue, charge_group):
     help="The chargeback group to use when submitting",
     default=None,
 )
-def bioimage(model_path, data_path, queue, charge_group):
-    command = f"{SERVER_COMMAND} bioimage -m {model_path} -d {data_path}"
-    run(command, data_path, queue, charge_group)
 
+def bioimage(model_path, data_path, edge_length_to_process, queue, charge_group):
+    command = f"{SERVER_COMMAND} bioimage -m {model_path} -d {data_path} -e {edge_length_to_process}"
+    base_name = model_path.split("/")[-1].split(".")[0]
+    run(command, data_path, queue, charge_group, base_name)
+
+
+@cli.command()
+@click.option(
+    "-f", "--config_folder", required=True, type=str, help="Path to the model folder"
+)
+@click.option("-n", "--name", required=True, type=str, help="Name of the model")
+@click.option(
+    "-d", "--data_path", required=True, type=str, help="The path to the data."
+)
+@click.option(
+    "-q",
+    "--queue",
+    required=False,
+    type=str,
+    help="The queue to use when submitting",
+    default="gpu_h100",
+)
+@click.option(
+    "-P",
+    "--charge_group",
+    required=False,
+    type=str,
+    help="The chargeback group to use when submitting",
+    default=None,
+)
+def cellmap_model(config_folder, name, data_path, queue, charge_group):
+    """Run the CellMapFlow with a CellMap model."""
+    command = (
+        f"{SERVER_COMMAND} cellmap-model -f {config_folder} -n {name} -d {data_path}"
+    )
+    run(command, data_path, queue, charge_group, name)
+
+
+
+@cli.command()
+@click.option(
+    "--script_path",
+    "-s",
+    type=str,
+    help="Path to the Python script containing model specification",
+)
+@click.option("--dataset", "-d", type=str, help="Path to the dataset")
+def script_server_check(script_path, dataset):
+    model_config = ScriptModelConfig(script_path=script_path)
+    server = CellMapFlowServer(dataset, model_config)
+    chunk_x = 2
+    chunk_y = 2
+    chunk_z = 2
+
+    server._chunk_impl(None, None, chunk_x, chunk_y, chunk_z, None)
+
+    print("Server check passed")
 
-def run(
-    command,
-    dataset_path,
-    queue,
-    charge_group,
-):
 
-    host = start_hosts(command, queue, charge_group)
-    if host is None:
-        raise Exception("Could not start host")
+def run(command, dataset_path, queue, charge_group, name):
 
-    inference_dict = {host: "prediction"}
+    start_hosts(command, queue, charge_group, name)
 
-    generate_neuroglancer_link(dataset_path, inference_dict)
+    neuroglancer_url = generate_neuroglancer_url(dataset_path)
+    while True:
+        pass

cellmap_flow/cli/multiple_cli.py

@@ -1,17 +1,22 @@
 import sys
-from cellmap_flow.utils.data import DaCapoModelConfig, BioModelConfig, ScriptModelConfig
+from cellmap_flow.utils.data import (
+    DaCapoModelConfig,
+    BioModelConfig,
+    ScriptModelConfig,
+    CellMapModelConfig,
+)
 import logging
-from cellmap_flow.utils.bsub_utils import start_hosts
-from cellmap_flow.utils.neuroglancer_utils import generate_neuroglancer_link
+from cellmap_flow.utils.bsub_utils import start_hosts, SERVER_COMMAND
+from cellmap_flow.utils.neuroglancer_utils import generate_neuroglancer_url
+import cellmap_flow.globals as g
 
 
 data_args = ["-d", "--data-path"]
-charge_back_arg = ["-P", "--project"]
+charge_group_arg = ["-P", "--project"]
 server_queue_arg = ["-q", "--queue"]
 
 DEFAULT_SERVER_QUEUE = "gpu_h100"
 
-SERVER_COMMAND = "cellmap_flow_server"
 
 logger = logging.getLogger(__name__)
 
@@ -29,6 +34,10 @@ def main():
 
     args = sys.argv[1:]
 
+    if "--help" in args:
+        print(main.__doc__)
+        sys.exit(0)
+
     if not args:
         logger.error("No arguments provided.")
         sys.exit(1)
@@ -37,7 +46,7 @@ def main():
         logger.error("Missing required argument: --data-path")
         sys.exit(1)
 
-    if charge_back_arg[0] not in args and charge_back_arg[1] not in args:
+    if charge_group_arg[0] not in args and charge_group_arg[1] not in args:
         logger.error("Missing required argument: --project")
         sys.exit(1)
 
@@ -47,7 +56,12 @@ def main():
         )
         args.extend([server_queue_arg[0], DEFAULT_SERVER_QUEUE])
 
-    if "--dacapo" not in args and "--script" not in args and "--bioimage" not in args:
+    if (
+        "--dacapo" not in args
+        and "--script" not in args
+        and "--bioimage" not in args
+        and "--cellmap-model" not in args
+    ):
         logger.error(
             "Missing required argument at least one should exist: --dacapo, --script, or --bioimage"
         )
@@ -58,16 +72,16 @@ def main():
 
     # Extract data path
     data_path = None
-    charge_back = None
+    charge_group = None
     queue = None
     models = []
 
     for i, arg in enumerate(args):
-        if arg in charge_back_arg:
-            if charge_back is not None:
+        if arg in charge_group_arg:
+            if charge_group is not None:
                 logger.error("Multiple charge back projects provided.")
                 sys.exit(1)
-            charge_back = args[i + 1]
+            charge_group = args[i + 1]
         if arg in server_queue_arg:
             if queue is not None:
                 logger.error("Multiple server queues provided.")
@@ -84,7 +98,7 @@ def main():
         logger.error("Data path not provided.")
         sys.exit(1)
 
-    if not charge_back:
+    if not charge_group:
         logger.error("Charge back project not provided.")
         sys.exit(1)
 
@@ -126,6 +140,26 @@ def main():
             i = j
             continue
 
+        if token == "--cellmap-model":
+            config_folder = None
+            name = None
+            j = i + 1
+            while j < len(args) and not args[j].startswith("--"):
+                if args[j] in ("-f", "--config_folder"):
+                    config_folder = args[j + 1]
+                    j += 2
+                elif args[j] in ("-n", "--name"):
+                    name = args[j + 1]
+                    j += 2
+                else:
+                    j += 1
+            if not config_folder:
+                logger.error(
+                    "Missing -c/--config_folder for --celmmap-model sub-command."
+                )
+                sys.exit(1)
+            models.append(CellMapModelConfig(config_folder, name=name))
+
         elif token == "--script":
             # We expect: --script -s script_path -n "some name"
             script_path = None
@@ -182,21 +216,21 @@ def main():
     for model in models:
         print(model)
 
-    run_multiple(models, data_path, charge_back, queue)
+    run_multiple(models, data_path, charge_group, queue)
 
 
 if __name__ == "__main__":
     main()
 
 
-def run_multiple(models, dataset_path, charge_back, queue):
-    inference_dict = {}
+def run_multiple(models, dataset_path, charge_group, queue):
+    g.queue = queue
+    g.charge_group = charge_group
     for model in models:
         command = f"{SERVER_COMMAND} {model.command} -d {dataset_path}"
-        host = start_hosts(
-            command, job_name=model.name, queue=queue, charge_group=charge_back
+        start_hosts(
+            command, job_name=model.name, queue=queue, charge_group=charge_group
         )
-        if host is None:
-            raise Exception("Could not start host")
-        inference_dict[host] = model.name
-    generate_neuroglancer_link(dataset_path, inference_dict)
+    generate_neuroglancer_url(dataset_path)
+    while True:
+        pass