Skip to content

Commit 9736bf3

Browse files
mzouinkmzouink
authored
Merge pull request #14 from janelia-cellmap/norm
- multiple cli add charge group and queue - remove funlib persistence - dacapo / bioimage dependency are options now
2 parents bcd1b52 + 232ae25 commit 9736bf3

File tree

15 files changed

+1154
-781
lines changed

15 files changed

+1154
-781
lines changed

‎README.md‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,43 @@ Please feel free to explore and contribute, but note that there may be frequent
55

66
<div align="center">
77

8-
![CellMapFlow Logo](img/CMFLO_dark.png)
8+
![CellMapFlow Logo](img/CMFLOW_dark.png)
99

1010
### Real-time inference is performed using Torch/Tensorflow, Dacapo, and bioimage models on local data or any cloud-hosted data.
1111
[![Under Construction](https://img.shields.io/badge/Status-Under_Construction-orange.svg)](#)
1212
---
1313

14-
1514
</div>
1615

16+
17+
## Installation
18+
19+
To install CellMapFlow, you can use pip:
20+
21+
```bash
22+
pip install cellmap_flow
23+
```
24+
25+
Note that the basic installation does not include DaCapo and BioImage.io core dependencies. To install CellMapFlow with DaCapo support, use the following command:
26+
27+
```bash
28+
pip install cellmap_flow[dacapo]
29+
```
30+
31+
To install CellMapFlow with BioImage.io support, use the following command:
32+
33+
```bash
34+
pip install cellmap_flow[bioimage]
35+
```
36+
37+
To install CellMapFlow with both DaCapo and BioImage.io support, use the following command:
38+
39+
```bash
40+
pip install cellmap_flow[dacapo,bioimage]
41+
```
42+
43+
## Usage
44+
1745
```bash
1846
$ cellmap_flow
1947

‎cellmap_flow/cli/multiple_cli.py‎

Lines changed: 187 additions & 143 deletions
Original file line numberDiff line numberDiff line change
@@ -6,153 +6,197 @@
66

77

88
data_args = ["-d", "--data-path"]
9+
charge_back_arg = ["-P", "--project"]
10+
server_queue_arg = ["-q", "--queue"]
11+
12+
DEFAULT_SERVER_QUEUE = "gpu_h100"
13+
914
SERVER_COMMAND = "cellmap_flow_server"
1015

1116
logger = logging.getLogger(__name__)
12-
def main():
13-
"""
14-
Allows chaining multiple model calls in one command, for example:
15-
16-
\b
17-
cellmap_flow_multiple --data-path /some/shared/path --dacapo -r run_1 -it 60 --dacapo -r run_2 -it 50 --script -s /path/to/script
18-
19-
This will parse the arguments and dispatch the appropriate logic
20-
for each sub-command (dacapo, script, etc.).
21-
"""
22-
23-
args = sys.argv[1:]
24-
25-
if not args:
26-
logger.error("No arguments provided.")
27-
sys.exit(1)
28-
29-
if data_args[0] not in args and data_args[1] not in args:
30-
logger.error("Missing required argument: --data-path")
31-
sys.exit(1)
32-
33-
if "--dacapo" not in args and "--script" not in args and "--bioimage" not in args:
34-
logger.error("Missing required argument at least one should exist: --dacapo, --script, or --bioimage")
35-
logger.error("Example: cellmap_flow_multiple --data-path /some/shared/path --dacapo -r run_1 -it 60 --dacapo -r run_2 -it 50 --script -s /path/to/script")
36-
logger.error("Now we will just open the raw data ..")
37-
38-
# Extract data path
39-
data_path = None
40-
models = []
41-
42-
for i, arg in enumerate(args):
43-
if arg in data_args:
44-
if data_path is not None:
45-
logger.error("Multiple data paths provided.")
46-
sys.exit(1)
47-
data_path = args[i + 1]
48-
49-
if not data_path:
50-
logger.error("Data path not provided.")
51-
sys.exit(1)
52-
53-
print("Data path:", data_path)
54-
55-
i = 0
56-
while i < len(args):
57-
token = args[i]
58-
59-
if token == "--dacapo":
60-
# We expect: --dacapo -r run_name -it iteration -n "some name"
61-
run_name = None
62-
iteration = 0
63-
name = None
64-
65-
j = i + 1
66-
while j < len(args) and not args[j].startswith("--"):
67-
if args[j] in ("-r", "--run-name"):
68-
run_name = args[j+1]
69-
j += 2
70-
elif args[j] in ("-i", "--iteration"):
71-
iteration = int(args[j+1])
72-
j += 2
73-
elif args[j] in ("-n", "--name"):
74-
name = args[j+1]
75-
j += 2
76-
else:
77-
j += 1
78-
79-
if not run_name:
80-
logger.error("Missing -r/--run-name for --dacapo sub-command.")
81-
sys.exit(1)
82-
83-
models.append(DaCapoModelConfig(run_name, iteration, name=name))
84-
i = j
85-
continue
86-
87-
elif token == "--script":
88-
# We expect: --script -s script_path -n "some name"
89-
script_path = None
90-
name = None
91-
92-
j = i + 1
93-
while j < len(args) and not args[j].startswith("--"):
94-
if args[j] in ("-s", "--script_path"):
95-
script_path = args[j+1]
96-
j += 2
97-
elif args[j] in ("-n", "--name"):
98-
name = args[j+1]
99-
j += 2
100-
else:
101-
j += 1
102-
103-
if not script_path:
104-
logger.error("Missing -s/--script_path for --script sub-command.")
105-
sys.exit(1)
106-
107-
models.append(ScriptModelConfig(script_path, name=name))
108-
i = j
109-
continue
110-
111-
elif token == "--bioimage":
112-
# We expect: --bioimage -m model_path -n "some name"
113-
model_path = None
114-
name = None
115-
116-
j = i + 1
117-
while j < len(args) and not args[j].startswith("--"):
118-
if args[j] in ("-m", "--model_path"):
119-
model_path = args[j+1]
120-
j += 2
121-
elif args[j] in ("-n", "--name"):
122-
name = args[j+1]
123-
j += 2
124-
else:
125-
j += 1
126-
127-
if not model_path:
128-
logger.error("Missing -m/--model_path for --bioimage sub-command.")
129-
sys.exit(1)
130-
131-
models.append(BioModelConfig(model_path, name=name))
132-
i = j
133-
continue
134-
135-
else:
136-
# If we don't recognize the token, just move on
137-
i += 1
138-
139-
# Print out the model configs for debugging
140-
for model in models:
141-
print(model)
142-
143-
run_multiple(models, data_path)
14417

145-
if __name__ == "__main__":
146-
main()
14718

19+
def main():
20+
"""
21+
Allows chaining multiple model calls in one command, for example:
22+
23+
\b
24+
cellmap_flow_multiple --data-path /some/shared/path --dacapo -r run_1 -it 60 --dacapo -r run_2 -it 50 --script -s /path/to/script
25+
26+
This will parse the arguments and dispatch the appropriate logic
27+
for each sub-command (dacapo, script, etc.).
28+
"""
29+
30+
args = sys.argv[1:]
31+
32+
if not args:
33+
logger.error("No arguments provided.")
34+
sys.exit(1)
35+
36+
if data_args[0] not in args and data_args[1] not in args:
37+
logger.error("Missing required argument: --data-path")
38+
sys.exit(1)
39+
40+
if charge_back_arg[0] not in args and charge_back_arg[1] not in args:
41+
logger.error("Missing required argument: --project")
42+
sys.exit(1)
43+
44+
if server_queue_arg[0] not in args and server_queue_arg[1] not in args:
45+
logger.warning(
46+
f"Missing required argument: --queue, using default queue {DEFAULT_SERVER_QUEUE}"
47+
)
48+
args.extend([server_queue_arg[0], DEFAULT_SERVER_QUEUE])
49+
50+
if "--dacapo" not in args and "--script" not in args and "--bioimage" not in args:
51+
logger.error(
52+
"Missing required argument at least one should exist: --dacapo, --script, or --bioimage"
53+
)
54+
logger.error(
55+
"Example: cellmap_flow_multiple --data-path /some/shared/path --dacapo -r run_1 -it 60 --dacapo -r run_2 -it 50 --script -s /path/to/script"
56+
)
57+
logger.error("Now we will just open the raw data ..")
58+
59+
# Extract data path
60+
data_path = None
61+
charge_back = None
62+
queue = None
63+
models = []
64+
65+
for i, arg in enumerate(args):
66+
if arg in charge_back_arg:
67+
if charge_back is not None:
68+
logger.error("Multiple charge back projects provided.")
69+
sys.exit(1)
70+
charge_back = args[i + 1]
71+
if arg in server_queue_arg:
72+
if queue is not None:
73+
logger.error("Multiple server queues provided.")
74+
sys.exit(1)
75+
queue = args[i + 1]
76+
77+
if arg in data_args:
78+
if data_path is not None:
79+
logger.error("Multiple data paths provided.")
80+
sys.exit(1)
81+
data_path = args[i + 1]
82+
83+
if not data_path:
84+
logger.error("Data path not provided.")
85+
sys.exit(1)
86+
87+
if not charge_back:
88+
logger.error("Charge back project not provided.")
89+
sys.exit(1)
90+
91+
if not queue:
92+
logger.error("Server queue not provided.")
93+
sys.exit(1)
94+
95+
print("Data path:", data_path)
96+
97+
i = 0
98+
while i < len(args):
99+
token = args[i]
100+
101+
if token == "--dacapo":
102+
# We expect: --dacapo -r run_name -it iteration -n "some name"
103+
run_name = None
104+
iteration = 0
105+
name = None
106+
107+
j = i + 1
108+
while j < len(args) and not args[j].startswith("--"):
109+
if args[j] in ("-r", "--run-name"):
110+
run_name = args[j + 1]
111+
j += 2
112+
elif args[j] in ("-i", "--iteration"):
113+
iteration = int(args[j + 1])
114+
j += 2
115+
elif args[j] in ("-n", "--name"):
116+
name = args[j + 1]
117+
j += 2
118+
else:
119+
j += 1
120+
121+
if not run_name:
122+
logger.error("Missing -r/--run-name for --dacapo sub-command.")
123+
sys.exit(1)
124+
125+
models.append(DaCapoModelConfig(run_name, iteration, name=name))
126+
i = j
127+
continue
128+
129+
elif token == "--script":
130+
# We expect: --script -s script_path -n "some name"
131+
script_path = None
132+
name = None
133+
134+
j = i + 1
135+
while j < len(args) and not args[j].startswith("--"):
136+
if args[j] in ("-s", "--script_path"):
137+
script_path = args[j + 1]
138+
j += 2
139+
elif args[j] in ("-n", "--name"):
140+
name = args[j + 1]
141+
j += 2
142+
else:
143+
j += 1
144+
145+
if not script_path:
146+
logger.error("Missing -s/--script_path for --script sub-command.")
147+
sys.exit(1)
148+
149+
models.append(ScriptModelConfig(script_path, name=name))
150+
i = j
151+
continue
152+
153+
elif token == "--bioimage":
154+
# We expect: --bioimage -m model_path -n "some name"
155+
model_path = None
156+
name = None
157+
158+
j = i + 1
159+
while j < len(args) and not args[j].startswith("--"):
160+
if args[j] in ("-m", "--model_path"):
161+
model_path = args[j + 1]
162+
j += 2
163+
elif args[j] in ("-n", "--name"):
164+
name = args[j + 1]
165+
j += 2
166+
else:
167+
j += 1
168+
169+
if not model_path:
170+
logger.error("Missing -m/--model_path for --bioimage sub-command.")
171+
sys.exit(1)
172+
173+
models.append(BioModelConfig(model_path, name=name))
174+
i = j
175+
continue
176+
177+
else:
178+
# If we don't recognize the token, just move on
179+
i += 1
180+
181+
# Print out the model configs for debugging
182+
for model in models:
183+
print(model)
184+
185+
run_multiple(models, data_path, charge_back, queue)
148186

149-
def run_multiple(models,dataset_path):
150-
inference_dict = {}
151-
for model in models:
152-
command = f"{SERVER_COMMAND} {model.command} -d {dataset_path}"
153-
host = start_hosts(command,job_name=model.name)
154-
if host is None:
155-
raise Exception("Could not start host")
156-
inference_dict[host] = model.name
157-
generate_neuroglancer_link(dataset_path, inference_dict)
158187

188+
if __name__ == "__main__":
189+
main()
190+
191+
192+
def run_multiple(models, dataset_path, charge_back, queue):
193+
inference_dict = {}
194+
for model in models:
195+
command = f"{SERVER_COMMAND} {model.command} -d {dataset_path}"
196+
host = start_hosts(
197+
command, job_name=model.name, queue=queue, charge_group=charge_back
198+
)
199+
if host is None:
200+
raise Exception("Could not start host")
201+
inference_dict[host] = model.name
202+
generate_neuroglancer_link(dataset_path, inference_dict)

0 commit comments

Comments
 (0)