Cache mode (#12)

* Add `hasdata` to storagemanager for cachecheck

* Add intern

* Improve mypy type notations

* Add awareness for terminal mode

* Add relay (use intern to communicate upstream)

* Add simple cache manager

* Add documentation

Author: j6k4m8

Pipfile

@@ -1,24 +1,21 @@
 [[source]]
-
 url = "https://pypi.python.org/simple"
 verify_ssl = true
 name = "pypi"
 
-
 [packages]
-
 flask = "*"
 numpy = "*"
 blosc = "*"
 intern = "*"
 
-
 [dev-packages]
-
 pylint = "*"
 mypy = "*"
 wheel = "*"
 twine = "*"
 jupyter = "*"
 matplotlib = "*"
 pandas = "*"
+"autopep8" = "*"
+pydocstyle = "*"

Pipfile.lock

@@ -15,6 +15,7 @@
 """
 
 from abc import ABC, abstractmethod
+from typing import Tuple
 
 import numpy as np
 
@@ -29,11 +30,22 @@ class StorageManager(ABC):
     """
 
     @abstractmethod
-    def getdata(self, col: str, exp: str, chan: str,
-                res: int, xs: [int, int], ys: [int, int], zs: [int, int]):
-        pass
+    def getdata(
+            self, col: str, exp: str, chan: str, res: int,
+            xs: Tuple[int, int], ys: Tuple[int, int], zs: Tuple[int, int]
+    ):
+        ...
 
     @abstractmethod
-    def setdata(self, data: np.array, col: str, exp: str, chan: str,
-                res: int, xs: [int, int], ys: [int, int], zs: [int, int]):
-        pass
+    def hasdata(
+            self, col: str, exp: str, chan: str, res: int,
+            xs: Tuple[int, int], ys: Tuple[int, int], zs: Tuple[int, int]
+    ):
+        ...
+
+    @abstractmethod
+    def setdata(
+            self, data: np.array, col: str, exp: str, chan: str, res: int,
+            xs: Tuple[int, int], ys: Tuple[int, int], zs: Tuple[int, int]
+    ):
+        ...

bossphorus/storagemanager/StorageManager.py

@@ -13,6 +13,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+from typing import Tuple
 
 import os
 import numpy as np
@@ -27,19 +28,31 @@ class FilesystemStorageManager(StorageManager):
     Contains logic for reading and writing to local filesystem.
     """
 
-    def __init__(self, storage_path: str, block_size: [int, int, int]):
+    def __init__(
+            self, storage_path: str, block_size: Tuple[int, int, int],
+            is_terminal=True
+    ):
         """
         Create a new FileSystemStorageManager.
 
         Arguments:
             storage_path: Where to store the data tree
             block_size: How much data should go in each file
         """
+        self.is_terminal = is_terminal
         self.storage_path = storage_path
         self.block_size = block_size
 
+    def hasdata(
+            self, col: str, exp: str, chan: str, res: int,
+            xs: Tuple[int, int], ys: Tuple[int, int], zs: Tuple[int, int]
+    ):
+        # TODO: Should know when it has data and return false even if it's
+        # in terminal mode
+        return self.is_terminal
+
     def setdata(self, data: np.array, col: str, exp: str, chan: str, res: int,
-                xs: [int, int], ys: [int, int], zs: [int, int]):
+                xs: Tuple[int, int], ys: Tuple[int, int], zs: Tuple[int, int]):
         """
         Upload the file.
 
@@ -74,7 +87,7 @@ def setdata(self, data: np.array, col: str, exp: str, chan: str, res: int,
             data_partial = self.store(data_partial, col, exp, chan, res, f)
 
     def getdata(self, col: str, exp: str, chan: str, res: int,
-                xs: [int, int], ys: [int, int], zs: [int, int]):
+                xs: Tuple[int, int], ys: Tuple[int, int], zs: Tuple[int, int]):
         """
         Get the data from disk.
 
@@ -118,7 +131,7 @@ def getdata(self, col: str, exp: str, chan: str, res: int,
         return payload
 
     def store(self, data: np.array, col: str, exp: str, chan: str, res: int,
-              b: [int, int, int]):
+              b: Tuple[int, int, int]):
         """
         Store a single block file.
 
@@ -142,7 +155,7 @@ def store(self, data: np.array, col: str, exp: str, chan: str, res: int,
         return np.save(fname, data)
 
     def retrieve(self, col: str, exp: str, chan: str, res: int,
-                 b: [int, int, int]):
+                 b: Tuple[int, int, int]):
         """
         Pull a single block from disk.
 
@@ -165,4 +178,3 @@ def retrieve(self, col: str, exp: str, chan: str, res: int,
             (b[2], b[2] + self.block_size[2]),
         )
         return np.load(fname)
-    

bossphorus/storagemanager/_FilesystemStorageManager.py

@@ -0,0 +1,71 @@
+"""
+Copyright 2018 The Johns Hopkins University Applied Physics Laboratory.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from typing import Tuple
+
+import numpy as np
+
+from intern.remote.boss import BossRemote
+
+from .StorageManager import StorageManager
+
+
+class RelayStorageManager(StorageManager):
+    """
+
+    """
+
+    def __init__(self, **kwargs):
+        """
+        Create a new RelayStorageManager.
+
+        Arguments:
+
+            block_size: How much data should go in each file
+        """
+        self.block_size = kwargs.get("block_size", (256, 256, 16))
+
+        if "boss_remote" in kwargs:
+            self.boss_remote = kwargs["boss_remote"]
+        elif "upstream_uri" in kwargs:
+            self.boss_remote = BossRemote({
+                "host": kwargs["upstream_uri"],
+                "protocol": kwargs.get("protocol", "http"),
+                "token": kwargs.get("token", "no-token")
+            })
+
+    def hasdata(
+            self, col: str, exp: str, chan: str, res: int,
+            xs: Tuple[int, int], ys: Tuple[int, int], zs: Tuple[int, int]
+    ):
+        return True
+
+    def setdata(
+            self, data: np.array, col: str, exp: str, chan: str, res: int,
+            xs: Tuple[int, int], ys: Tuple[int, int], zs: Tuple[int, int]
+    ):
+        return self.boss_remote.create_cutout(
+            self.boss_remote.get_channel(chan, col, exp), res, xs, ys, zs, data
+        )
+
+    def getdata(
+            self, col: str, exp: str, chan: str, res: int,
+            xs: Tuple[int, int], ys: Tuple[int, int], zs: Tuple[int, int]
+    ) -> np.array:
+        return self.boss_remote.get_cutout(
+            self.boss_remote.get_channel(chan, col, exp), res, xs, ys, zs
+        )
+
+

bossphorus/storagemanager/_RelayStorageManager.py

@@ -0,0 +1,66 @@
+"""
+Copyright 2018 The Johns Hopkins University Applied Physics Laboratory.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from typing import List, Tuple
+
+import numpy as np
+
+from intern.remote.boss import BossRemote
+
+from .StorageManager import StorageManager
+
+
+class SimpleCacheStorageManager(StorageManager):
+    """
+    A SimpleCache is a naïve "cascade" of storage managers.
+
+    Reads:
+        Retrieves data from the first manager in `layers` that can satisfy the
+            request for data
+
+    Writes:
+        Writes to ALL layers.
+    """
+
+    def __init__(self, layers: List) -> None:
+        """
+        Create a new SimpleCacheStorageManager.
+        """
+        self.layers = layers
+
+    def hasdata(
+            self, col: str, exp: str, chan: str, res: int,
+            xs: Tuple[int, int], ys: Tuple[int, int], zs: Tuple[int, int]
+    ):
+        return any([
+            layer.hasdata(col, exp, chan, res, xs, ys, zs)
+            for layer in self.layers
+        ])
+
+    def setdata(
+            self, data: np.array, col: str, exp: str, chan: str, res: int,
+            xs: Tuple[int, int], ys: Tuple[int, int], zs: Tuple[int, int]
+    ):
+        for layer in self.layers:
+            layer.setdata(data, col, exp, chan, res, xs, ys, zs)
+
+    def getdata(
+            self, col: str, exp: str, chan: str, res: int,
+            xs: Tuple[int, int], ys: Tuple[int, int], zs: Tuple[int, int]
+    ) -> np.array:
+        for layer in self.layers:
+            if layer.hasdata(col, exp, chan, res, xs, ys, zs):
+                return layer.getdata(col, exp, chan, res, xs, ys, zs)
+        raise ValueError("Data could not be retrieved.")

bossphorus/storagemanager/_SimpleCacheStorageManager.py

@@ -0,0 +1,35 @@
+# Storage Managers
+
+There are a variety of available storage managers, each with their own strengths and weaknesses. They are listed below for your reference.
+
+*All* storage managers have `getdata`, `setdata`, and `hasdata` functions. If `hasdata` returns `True`, that is an informal promise that `getdata` with the same arguments will successfully return data without error.
+
+## FilesystemStorageManager
+
+Uses the filesystem as a storage mechanism for data.
+
+Data are stored in numpy-compressed format, and are block-chunked to enable parallel data access.
+
+## RelayStorageManager
+
+Uses `intern` (`pip install intern`) to point to an upstream bossDB or bossphorus node.
+
+## SimpleCacheStorageManager
+
+Provides no smarts on its own; instead, acts as a naïve 'cascade' cache for a list of other storage managers.
+
+For example, you could use this as a multilevel cache for a bossDB instance:
+
+```python
+
+SCSM = SimpleCacheStorageManager(
+    layers=[
+        RelayStorageManager(upstream_uri="localhost:3000"),
+        RelayStorageManager(upstream_uri="my-lab-bossdb/"),
+        RelayStorageManager(upstream_uri="bossdb.my-institution.edu/"),
+        RelayStorageManager(upstream_uri="my-boss-instance.com/"),
+    ]
+)
+```
+
+...which will attempt a local cache, then a labwide cache, then an institutional cache, and finally a cloud-based bossDB, in that order.