Add tests and demo

Author: j6k4m8

Demo 1.ipynb

@@ -4,24 +4,59 @@
 import json
 import os
 
-from utils import block_compute
+from typing import List
 
 from flask import Flask, request, Response, send_file, jsonify
 import numpy as np
 
 APP = Flask(__name__)
 
 UPLOADS_PATH = "./uploads"
-BLOCK_SIZE = (256, 256, 256)
+BLOCK_SIZE = (64, 64, 64)
 
 
-def file_compute(x_start, x_stop,
+def file_compute(
+        x_start, x_stop,
+        y_start, y_stop,
+        z_start, z_stop,
+        origin=(0, 0, 0),
+        block_size=(256, 256, 256)
+    ):
+    """
+    Which files do we need to pull for this volume?
+    """
+    # x
+
+    x_block_origins = [
+        b
+        for b in range(0, x_stop + block_size[0], block_size[0])
+        if b > (x_start - block_size[0]) and b < x_stop
+    ]
+    y_block_origins = [
+        b
+        for b in range(0, y_stop + block_size[1], block_size[1])
+        if b > (y_start - block_size[1]) and b < y_stop
+    ]
+    z_block_origins = [
+        b
+        for b in range(0, z_stop + block_size[2], block_size[2])
+        if b > (z_start - block_size[2]) and b < z_stop
+    ]
+
+    files = []
+    for x in x_block_origins:
+        for y in y_block_origins:
+            for z in z_block_origins:
+                files.append((x, y, z))
+    return files
+
+def block_compute(x_start, x_stop,
                  y_start, y_stop,
                  z_start, z_stop,
                  origin=(0, 0, 0),
                  block_size=(256, 256, 256)):
     """
-    Which files do we need to pull for this box?
+    What are the block-aligned indices for this volume?
     """
     # x
 
@@ -30,22 +65,34 @@ def file_compute(x_start, x_stop,
         for b in range(0, x_stop + block_size[0], block_size[0])
         if b > (x_start - block_size[0]) and b < x_stop
     ]
+    x_block_origins[0] = max(x_block_origins[0], x_start)
+    x_block_origins[-1] = min(x_block_origins[-1], x_stop)
+
     y_block_origins = [
         b
         for b in range(0, y_stop + block_size[1], block_size[1])
         if b > (y_start - block_size[1]) and b < y_stop
     ]
+    y_block_origins[0] = max(y_block_origins[0], y_start)
+    y_block_origins[-1] = min(y_block_origins[-1], y_stop)
+
     z_block_origins = [
         b
         for b in range(0, z_stop + block_size[2], block_size[2])
         if b > (z_start - block_size[2]) and b < z_stop
     ]
+    z_block_origins[0] = max(z_block_origins[0], z_start)
+    z_block_origins[-1] = min(z_block_origins[-1], z_stop)
 
     files = []
     for x in x_block_origins:
         for y in y_block_origins:
             for z in z_block_origins:
-                files.append((x, y, z))
+                files.append((
+                    (x, min(x_stop, x+block_size[0])),
+                    (y, min(y_stop, y+block_size[1])),
+                    (z, min(z_stop, z+block_size[2]))
+                ))
     return files
 
 
@@ -84,7 +131,6 @@ def blockfile_indices(xs, ys, zs, origin=(0, 0, 0), block_size=(256, 256, 256)):
     return inds
 
 
-
 class StorageManager:
     """
     Abstract class.
@@ -108,27 +154,35 @@ def setdata(self, data, col, exp, chan, res, xs, ys, zs):
             origin=(0, 0, 0),
             block_size=self.block_size
         )
+        files = file_compute(
+            xs[0], xs[1], ys[0], ys[1], zs[0], zs[1],
+            origin=(0, 0, 0),
+            block_size=self.block_size,
+        )
+        indices = blockfile_indices(
+            xs, ys, zs,
+            origin=(0, 0, 0),
+            block_size=self.block_size
+        )
+
+        print(blocks)
 
-        for b in blocks:
-            # Check to see if the file already exists
-
-            # TODO: All of this assumes 0-padded block alignment
-            self.store(
-                data[b[0][0] - xs[0]: b[0][1] - xs[0],
-                     b[1][0] - ys[0]: b[1][1] - ys[0],
-                     b[2][0] - zs[0]: b[2][1] - zs[0]],
-                col, exp, chan, res, b
-            )
-
-            # if self._exists(col, exp, chan, res, b):
-            #     # Open file, add new data, write file
-            #     # data = self.read(col, exp, chan, res, b)
-            #     pass
-            # else:
-            #     # Create the file with the appropriate contents
-            #     # and 0pad if necessary.
-            #     # pad0_data = np.zeros((self.block_size))
-            #     pass
+        for b, f, i in zip(blocks, files, indices):
+            try:
+                data_partial = self.retrieve(col, exp, chan, res, f)
+            except:
+                data_partial = np.zeros(self.block_size, dtype="uint8")
+
+            data_partial[
+                i[0][0]:i[0][1],
+                i[1][0]:i[1][1],
+                i[2][0]:i[2][1],
+            ] = data[
+                (f[0] + i[0][0]) - xs[0]: (f[0] + i[0][1]) - xs[0],
+                (f[1] + i[1][0]) - ys[0]: (f[1] + i[1][1]) - ys[0],
+                (f[2] + i[2][0]) - zs[0]: (f[2] + i[2][1]) - zs[0],
+            ]
+            data_partial = self.store(data_partial, col, exp, chan, res, f)
 
     def getdata(self, col, exp, chan, res, xs, ys, zs):
         """
@@ -162,22 +216,18 @@ def getdata(self, col, exp, chan, res, xs, ys, zs):
                     i[1][0]:i[1][1],
                     i[2][0]:i[2][1],
                 ]
-                payload[
-                    f[0] + i[0][0]: f[0] + i[0][1],
-                    f[1] + i[1][0]: f[1] + i[1][1],
-                    f[2] + i[2][0]: f[2] + i[2][1],
-                ] = data_partial
-
-            except ValueError:
-                payload[
-                    f[0] + i[0][0]: f[0] + i[0][1],
-                    f[1] + i[1][0]: f[1] + i[1][1],
-                    f[2] + i[2][0]: f[2] + i[2][1],
-                ] = np.zeros(self.block_size, dtype="uint8")[
+            except:
+                data_partial = np.zeros(self.block_size, dtype="uint8")[
                     i[0][0]:i[0][1],
                     i[1][0]:i[1][1],
                     i[2][0]:i[2][1],
                 ]
+            payload[
+                (f[0] + i[0][0]) - xs[0]: (f[0] + i[0][1]) - xs[0],
+                (f[1] + i[1][0]) - ys[0]: (f[1] + i[1][1]) - ys[0],
+                (f[2] + i[2][0]) - zs[0]: (f[2] + i[2][1]) - zs[0],
+            ] = data_partial
+
         return payload
 
     def store(self, data, col, exp, chan, res, b):
@@ -188,7 +238,10 @@ def store(self, data, col, exp, chan, res, b):
         fname = "{}/{}/{}/{}/{}-{}-{}-{}.npy".format(
             UPLOADS_PATH,
             col, exp, chan,
-            res, b[0], b[1], b[2],
+            res,
+            (b[0], b[0] + self.block_size[0]),
+            (b[1], b[1] + self.block_size[1]),
+            (b[2], b[2] + self.block_size[2]),
         )
         # print(fname)
         return np.save(fname, data)