Multiscale support

.github/workflows/precommit.yml

@@ -0,0 +1,11 @@
+name: Precommit
+on: [push, pull_request]
+jobs:
+  # https://github.com/pre-commit/action
+  pre-commit:
+    name: Lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - uses: pre-commit/[email protected]

.gitignore

@@ -1,6 +1,7 @@
 venv/
 .idea/
 *.iml
+*egg-info
 build/
 dist/
-src/__pycache__/
+__pycache__

.isort.cfg

@@ -0,0 +1,2 @@
+[settings]
+known_third_party = numpy,ome_zarr,omero,omero_zarr,setuptools,zarr

.pre-commit-config.yaml

@@ -1,18 +1,84 @@
+---
 repos:
-- repo: https://github.com/ambv/black
-  rev: 19.10b0
-  hooks:
-  - id: black
-    # default black line length is 88
-    args:
-      - --target-version=py35
-      - --line-length=79
-- repo: https://gitlab.com/pycqa/flake8
-  rev: 3.7.9
-  hooks:
-  - id: flake8
-    args:
-      # flake8 is not PEP8 compliant
-      # https://black.readthedocs.io/en/stable/the_black_code_style.html
-      - --ignore=E203,W503
-    # default flake8 line length is 79
+
+  - repo: https://github.com/asottile/seed-isort-config
+    rev: v1.9.3
+    hooks:
+      - id: seed-isort-config
+
+  - repo: https://github.com/timothycrosley/isort
+    rev: 5.3.2
+    hooks:
+      - id: isort
+
+  - repo: https://github.com/ambv/black
+    rev: 19.10b0
+    hooks:
+      - id: black
+        args: [--target-version=py36]
+
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v2.7.2
+    hooks:
+      - id: pyupgrade
+        args:
+          - --py36-plus
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v1.2.3
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-json
+        files: \.(json)$
+      - id: check-yaml
+      - id: fix-encoding-pragma
+        args:
+          - --remove
+      - id: trailing-whitespace
+      - id: check-case-conflict
+      - id: check-merge-conflict
+      - id: check-symlinks
+      - id: pretty-format-json
+        args:
+          - --autofix
+
+  - repo: https://gitlab.com/pycqa/flake8
+    rev: 3.8.3
+    hooks:
+      - id: flake8
+        additional_dependencies: [
+          flake8-blind-except,
+          flake8-builtins,
+          flake8-rst-docstrings,
+          flake8-logging-format,
+        ]
+        args: [
+          # default black line length is 88
+          "--max-line-length=88",
+          # Conflicts with black: E203 whitespace before ':'
+          "--ignore=E203",
+          "--rst-roles=class,func,ref,module,const",
+        ]
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.782
+    hooks:
+      - id: mypy
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.782
+    hooks:
+      - id: mypy
+        args: [
+          --disallow-untyped-defs,
+          --ignore-missing-imports,
+        ]
+        exclude: tests/*|setup.py
+
+  - repo: https://github.com/adrienverge/yamllint.git
+    rev: v1.24.2
+    hooks:
+      - id: yamllint
+        # args: [--config-data=relaxed]
+        #

README.md

@@ -2,7 +2,7 @@ OMERO CLI Zarr plugin
 =====================
 
 This OMERO command-line plugin allows you to export images from
-OMERO as zarr files, according to the spec at 
+OMERO as zarr files, according to the spec at
 https://github.com/ome/omero-ms-zarr/blob/master/spec.md.
 
 These are 5D arrays of shape `(t, c, z, y, x)`.
@@ -27,7 +27,7 @@ To export images via the OMERO API:
 $ omero zarr export Image:1
 
 # Specify an output directory
-$ omero zarr export Image:1 --output /home/user/zarr_files
+$ omero zarr --output /home/user/zarr_files export Image:1
 
 # Cache each plane as a numpy file.npy. If connection is lost, and you need
 # to export again, we can use these instead of downloading again
@@ -41,15 +41,15 @@ To export images via bioformats2raw we use the ```--bf``` flag:
 export MANAGED_REPO=/var/omero/data/ManagedRepository
 export BF2RAW=/opt/tools/bioformats2raw-0.2.0-SNAPSHOT
 
-$ omero zarr export 1 --bf --output /home/user/zarr_files
+$ omero zarr --output /home/user/zarr_files export 1 --bf
 Image exported to /home/user/zarr_files/2chZT.lsm
 ```
 
 To export masks for an Image:
 
 ```
 # Saved under zarr_files/1.zarr/labels/0
-$ omero zarr masks Image:1 --output /home/user/zarr_files
+$ omero zarr --output /home/user/zarr_files masks Image:1
 
 # Specify the label-path (default 'labels').
 # e.g. Export to 1.zarr/my_labels:

src/omero/plugins/zarr.py

@@ -1,3 +1,3 @@
-from omero_zarr.cli import ZarrControl, HELP
+from omero_zarr.cli import HELP, ZarrControl
 
-register("zarr", ZarrControl, HELP)  # noqa
+register("zarr", ZarrControl, HELP)  # type: ignore # noqa

src/omero_zarr/cli.py

@@ -1,16 +1,16 @@
-import sys
+import argparse
 import subprocess
-from pathlib import Path
+import sys
 from functools import wraps
+from pathlib import Path
+from typing import Any, Callable
 
-from omero.cli import BaseControl
-from omero.cli import CLI
-from omero.cli import ProxyStringType
-from omero.gateway import BlitzGateway
+from omero.cli import CLI, BaseControl, Parser, ProxyStringType
+from omero.gateway import BlitzGateway, BlitzObjectWrapper
 from omero.model import ImageI
 
+from .masks import MASK_DTYPE_SIZE, image_masks_to_zarr
 from .raw_pixels import image_to_zarr
-from .masks import image_masks_to_zarr, MASK_DTYPE_SIZE
 
 HELP = """Export data in zarr format.
 
@@ -35,21 +35,20 @@
   --style
 
      'labeled': 5D integer values (default but overlaps are not supported!)
-     '6d': masks are stored in a 6D array
      'split': one group per ROI
 
 """
 
 
-def gateway_required(func):
+def gateway_required(func: Callable) -> Callable:
     """
   Decorator which initializes a client (self.client),
   a BlitzGateway (self.gateway), and makes sure that
   all services of the Blitzgateway are closed again.
   """
 
     @wraps(func)
-    def _wrapper(self, *args, **kwargs):
+    def _wrapper(self: Any, *args: Any, **kwargs: Any) -> Callable:
         self.client = self.ctx.conn(*args)
         self.gateway = BlitzGateway(client_obj=self.client)
 
@@ -59,7 +58,7 @@ def _wrapper(self, *args, **kwargs):
             if self.gateway is not None:
                 self.gateway.close(hard=False)
                 self.gateway = None
-                self.client = None
+                self.client = None  # type: ignore
 
     return _wrapper
 
@@ -69,7 +68,7 @@ class ZarrControl(BaseControl):
     gateway = None
     client = None
 
-    def _configure(self, parser):
+    def _configure(self, parser: Parser) -> None:
         parser.add_login_arguments()
 
         parser.add_argument(
@@ -90,6 +89,14 @@ def _configure(self, parser):
             type=ProxyStringType("Image"),
             help="The Image from which to export Masks.",
         )
+        masks.add_argument(
+            "--source-image",
+            help=(
+                "Path to the multiscales group containing the source image. "
+                "By default, use the output directory"
+            ),
+            default=None,
+        )
         masks.add_argument(
             "--label-path",
             help=(
@@ -101,14 +108,13 @@ def _configure(self, parser):
         masks.add_argument(
             "--label-name",
             help=(
-                "Name of the array that will be stored. "
-                "Ignored for --style=split"
+                "Name of the array that will be stored. " "Ignored for --style=split"
             ),
             default="0",
         )
         masks.add_argument(
             "--style",
-            choices=("6d", "split", "labeled"),
+            choices=("split", "labeled"),
             default="labeled",
             help=("Choice of storage for ROIs [breaks ome-zarr]"),
         )
@@ -131,9 +137,7 @@ def _configure(self, parser):
 
         export = parser.add(sub, self.export, EXPORT_HELP)
         export.add_argument(
-            "--bf",
-            action="store_true",
-            help="Use bioformats2raw to export the image.",
+            "--bf", action="store_true", help="Use bioformats2raw to export the image.",
         )
         export.add_argument(
             "--tile_width", default=None, help="For use with bioformats2raw"
@@ -148,13 +152,11 @@ def _configure(self, parser):
             "--max_workers", default=None, help="For use with bioformats2raw"
         )
         export.add_argument(
-            "object",
-            type=ProxyStringType("Image"),
-            help="The Image to export.",
+            "object", type=ProxyStringType("Image"), help="The Image to export.",
         )
 
     @gateway_required
-    def masks(self, args):
+    def masks(self, args: argparse.Namespace) -> None:
         """Export masks on the Image as zarr files."""
         if isinstance(args.object, ImageI):
             image_id = args.object.id
@@ -163,12 +165,14 @@ def masks(self, args):
             image_masks_to_zarr(image, args)
 
     @gateway_required
-    def export(self, args):
+    def export(self, args: argparse.Namespace) -> None:
         if isinstance(args.object, ImageI):
             image = self._lookup(self.gateway, "Image", args.object.id)
             inplace = image.getInplaceImport()
 
             if args.bf:
+                if self.client is None:
+                    raise Exception("This cannot happen")  # mypy is confused
                 prx, desc = self.client.getManagedRepository(description=True)
                 repo_path = Path(desc._path._val) / Path(desc._name._val)
                 if inplace:
@@ -180,15 +184,17 @@ def export(self, args):
             else:
                 image_to_zarr(image, args)
 
-    def _lookup(self, gateway, type, oid):
+    def _lookup(
+        self, gateway: BlitzGateway, otype: str, oid: int
+    ) -> BlitzObjectWrapper:
         """Find object of type by ID."""
         gateway.SERVICE_OPTS.setOmeroGroup("-1")
-        obj = gateway.getObject(type, oid)
+        obj = gateway.getObject(otype, oid)
         if not obj:
-            self.ctx.die(110, "No such %s: %s" % (type, oid))
+            self.ctx.die(110, f"No such {otype}: {oid}")
         return obj
 
-    def _bf_export(self, abs_path, args):
+    def _bf_export(self, abs_path: Path, args: argparse.Namespace) -> None:
         target = (Path(args.output) or Path.cwd()) / Path(abs_path).name
         target.mkdir(exist_ok=True)
 
@@ -203,8 +209,7 @@ def _bf_export(self, abs_path, args):
             options += " --max_workers=" + args.max_workers
 
         self.ctx.dbg(
-            "bioformats2raw %s %s %s"
-            % (options, abs_path.resolve(), target.resolve())
+            f"bioformats2raw {options} {abs_path.resolve()} {target.resolve()}"
         )
         process = subprocess.Popen(
             ["bioformats2raw", options, abs_path.resolve(), target.resolve()],
@@ -215,11 +220,11 @@ def _bf_export(self, abs_path, args):
         if stderr:
             self.ctx.err(stderr)
         else:
-            self.ctx.out("Image exported to {}".format(target.resolve()))
+            self.ctx.out(f"Image exported to {target.resolve()}")
 
 
 try:
-    register("zarr", ZarrControl, HELP)
+    register("zarr", ZarrControl, HELP)  # type: ignore
 except NameError:
     if __name__ == "__main__":
         cli = CLI()