I guess I forgot to commit the entire time

j6k4m8 · j6k4m8 · commit 98381237f38d · 2018-03-10T23:54:46.000-05:00
diff --git a/bosslet/__init__.py b/bosslet/__init__.py
@@ -4,58 +4,249 @@
 import json
 import os
 
-from flask import Flask, request, Response
+from utils import block_compute
+
+from flask import Flask, request, Response, send_file, jsonify
 import numpy as np
 # from intern.remote.boss import BossRemote
 # from intern.resource.boss.resource import ChannelResource
 # from intern.utils.parallel import block_compute
 # from requests import codes, post
 
 APP = Flask(__name__)
+
 UPLOADS_PATH = "./uploads"
+BLOCK_SIZE = (256, 256, 256)
 
 
 """
 https://api.theboss.io/v1/cutout/:collection/:experiment/:channel/:resolution/:x_range/:y_range/:z_range/:time_range/?iso=:iso
 """
 
 
-@APP.route("/v1/cutout/<collection>/<experiment>/<channel>/<resolution>/<x_range>/<y_range>/<z_range>/", methods=["POST"])
-def upload_cutout_xyz(collection, experiment, channel, resolution, x_range, y_range, z_range):
+def file_compute(x_start, x_stop,
+                 y_start, y_stop,
+                 z_start, z_stop,
+                 origin=(0, 0, 0),
+                 block_size=(256, 256, 256)):
     """
-    Upload a volume
+    Which files do we need to pull for this box?
+    """
+    # x
+
+    x_block_origins = [
+        b
+        for b in range(0, x_stop + block_size[0], block_size[0])
+        if b > (x_start - block_size[0]) and b < x_stop
+    ]
+    y_block_origins = [
+        b
+        for b in range(0, y_stop + block_size[1], block_size[1])
+        if b > (y_start - block_size[1]) and b < y_stop
+    ]
+    z_block_origins = [
+        b
+        for b in range(0, z_stop + block_size[2], block_size[2])
+        if b > (z_start - block_size[2]) and b < z_stop
+    ]
+
+    files = []
+    for x in x_block_origins:
+        for y in y_block_origins:
+            for z in z_block_origins:
+                files.append((x, y, z))
+    return files
+
+
+def blockfile_indices(xs, ys, zs, origin=(0, 0, 0), block_size=(256, 256, 256)):
+    blocks = block_compute(
+        xs[0], xs[1], ys[0], ys[1], zs[0], zs[1],
+        origin, block_size
+    )
+    files = file_compute(
+        xs[0], xs[1], ys[0], ys[1], zs[0], zs[1],
+        origin, block_size
+    )
+
+    inds = []
+    for b, f in zip(blocks, files):
+        inds.append([
+            (b[0][0] - f[0], b[0][1] - f[0]),
+            (b[1][0] - f[1], b[1][1] - f[1]),
+            (b[2][0] - f[2], b[2][1] - f[2])
+        ])
 
+    return inds
+
+
+
+class StorageManager:
     """
-    for _, f in request.files.items():
-        memfile = io.BytesIO()
-        f.save(memfile)
-        data = np.fromstring(memfile.getvalue(), dtype=np.uint8)
+    Abstract class.
+    """
+    pass
+
+
+class FilesystemStorageManager(StorageManager):
+
+    def __init__(self, upload_path=UPLOADS_PATH, block_size=BLOCK_SIZE):
+        self.upload_path = upload_path
+        self.block_size = block_size
+
+    def setdata(self, data, col, exp, chan, res, xs, ys, zs):
+        """
+        Uploads the file.
+        """
+        # Chunk the file into its parts
+        blocks = block_compute(
+            xs[0], xs[1], ys[0], ys[1], zs[0], zs[1],
+            origin=(0, 0, 0),
+            block_size=self.block_size
+        )
+
+        for b in blocks:
+            # Check to see if the file already exists
+
+            # TODO: All of this assumes 0-padded block alignment
+            self.store(
+                data[b[0][0] - xs[0]: b[0][1] - xs[0],
+                     b[1][0] - ys[0]: b[1][1] - ys[0],
+                     b[2][0] - zs[0]: b[2][1] - zs[0]],
+                col, exp, chan, res, b
+            )
+
+            # if self._exists(col, exp, chan, res, b):
+            #     # Open file, add new data, write file
+            #     # data = self.read(col, exp, chan, res, b)
+            #     pass
+            # else:
+            #     # Create the file with the appropriate contents
+            #     # and 0pad if necessary.
+            #     # pad0_data = np.zeros((self.block_size))
+            #     pass
+
+    def getdata(self, col, exp, chan, res, xs, ys, zs):
+        """
+        Gets the data from disk.
+        """
+        blocks = block_compute(
+            xs[0], xs[1], ys[0], ys[1], zs[0], zs[1],
+            origin=(0, 0, 0),
+            block_size=self.block_size
+        )
+        files = file_compute(
+            xs[0], xs[1], ys[0], ys[1], zs[0], zs[1],
+            origin=(0, 0, 0),
+            block_size=self.block_size,
+        )
+        indices = blockfile_indices(
+            xs, ys, zs,
+            origin=(0, 0, 0),
+            block_size=self.block_size
+        )
+
+        payload = np.zeros((
+            (xs[1] - xs[0]),
+            (ys[1] - ys[0]),
+            (zs[1] - zs[0])
+        ), dtype="uint8")
+        for b, f, i in zip(blocks, files, indices):
+            try:
+                data_partial = self.retrieve(col, exp, chan, res, f)[
+                    i[0][0]:i[0][1],
+                    i[1][0]:i[1][1],
+                    i[2][0]:i[2][1],
+                ]
+                payload[
+                    f[0] + i[0][0]: f[0] + i[0][1],
+                    f[1] + i[1][0]: f[1] + i[1][1],
+                    f[2] + i[2][0]: f[2] + i[2][1],
+                ] = data_partial
+
+            except:
+                payload[
+                    f[0] + i[0][0]: f[0] + i[0][1],
+                    f[1] + i[1][0]: f[1] + i[1][1],
+                    f[2] + i[2][0]: f[2] + i[2][1],
+                ] = np.zeros(self.block_size, dtype="uint8")[
+                    i[0][0]:i[0][1],
+                    i[1][0]:i[1][1],
+                    i[2][0]:i[2][1],
+                ]
+        return payload
+
+    def store(self, data, col, exp, chan, res, b):
         os.makedirs("{}/{}/{}/{}/".format(
             UPLOADS_PATH,
-            collection, experiment, channel
+            col, exp, chan
         ), exist_ok=True)
-        np.save("{}/{}/{}/{}/{}-{}-{}-{}.npy".format(
+        fname = "{}/{}/{}/{}/{}-{}-{}-{}.npy".format(
             UPLOADS_PATH,
-            collection, experiment, channel,
-            resolution, x_range, y_range, z_range,
-        ))
-    return ""
+            col, exp, chan,
+            res, b[0], b[1], b[2],
+        )
+        # print(fname)
+        return np.save(fname, data)
 
+    def retrieve(self, col, exp, chan, res, b):
+        fname = "{}/{}/{}/{}/{}-{}-{}-{}.npy".format(
+            UPLOADS_PATH,
+            col, exp, chan,
+            res,
+            (b[0], b[0] + self.block_size[0]),
+            (b[1], b[1] + self.block_size[1]),
+            (b[2], b[2] + self.block_size[2]),
+        )
+        return np.load(fname)
 
-@APP.route("/v1/cutout/<collection>/<experiment>/<channel>/<resolution>/<x_range>/<y_range>/<z_range>/", methods=["GET"])
-def get_cutout_xyz(collection, experiment, channel, resolution, x_range, y_range, z_range):
+
+
+MANAGER = FilesystemStorageManager()
+
+@APP.route("/v1/cutout/<collection>/<experiment>/<channel>/<resolution>/<x_range>/<y_range>/<z_range>/", methods=["POST"])
+def upload_cutout_xyz(collection, experiment, channel, resolution, x_range, y_range, z_range):
     """
     Upload a volume
 
     """
     for _, f in request.files.items():
         memfile = io.BytesIO()
         f.save(memfile)
-        data = np.fromstring(memfile.getvalue(), dtype=np.uint8)
-        break
+        data = np.fromstring(memfile.getvalue(), dtype="uint8")
+        xs = [int(i) for i in x_range.split(":")]
+        ys = [int(i) for i in y_range.split(":")]
+        zs = [int(i) for i in z_range.split(":")]
+        data = data.reshape(xs[1] - xs[0], ys[1] - ys[0], zs[1] - zs[0])
+        MANAGER.setdata(data, collection, experiment, channel, resolution, xs, ys, zs)
     return ""
 
 
+@APP.route("/v1/cutout/<collection>/<experiment>/<channel>/<resolution>/<x_range>/<y_range>/<z_range>/", methods=["GET"])
+def get_cutout_xyz(collection, experiment, channel, resolution, x_range, y_range, z_range):
+    """
+    Download a volume
+    """
+    xs = [int(i) for i in x_range.split(":")]
+    ys = [int(i) for i in y_range.split(":")]
+    zs = [int(i) for i in z_range.split(":")]
+    if (
+        os.path.isdir("{}/{}".format(UPLOADS_PATH, collection)) and
+        os.path.isdir("{}/{}/{}".format(UPLOADS_PATH, collection, experiment)) and
+        os.path.isdir("{}/{}/{}/{}".format(UPLOADS_PATH, collection, experiment, channel))
+    ):
+        data = MANAGER.getdata(collection, experiment, channel, resolution, xs, ys, zs)
+        res = {}
+        res["dtype"] = str(data.dtype)
+        res["data"] = data.tolist()
+        return jsonify(res)
+    else:
+        return Response(
+            json.dumps({"message": "DNE"}),
+            status=402,
+            mimetype="application/json"
+        )
+
+
 @APP.route("/")
 def hello():
     """Root."""
diff --git a/bosslet/utils.py b/bosslet/utils.py
@@ -0,0 +1,121 @@
+# Copyright 2016 NeuroData (http://neurodata.io)
+# Copyright 2016 The Johns Hopkins University Applied Physics Laboratory
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+import numpy
+from six.moves import range
+
+
+def snap_to_cube(q_start, q_stop, chunk_depth=16, q_index=1):
+    """
+    For any q in {x, y, z, t}
+    Takes in a q-range and returns a 1D bound that starts at a cube
+    boundary and ends at another cube boundary and includes the volume
+    inside the bounds. For instance, snap_to_cube(2, 3) = (1, 17)
+    Arguments:
+        q_start (int): The lower bound of the q bounding box of volume
+        q_stop (int): The upper bound of the q bounding box of volume
+        chunk_depth (int : CHUNK_DEPTH) The size of the chunk in this
+            volume (use ``get_info()``)
+        q_index (int : 1): The starting index of the volume (in q)
+    Returns:
+        2-tuple: (lo, hi) bounding box for the volume
+    """
+    lo = 0
+    hi = 0
+    # Start by indexing everything at zero for our own sanity
+    q_start -= q_index
+    q_stop -= q_index
+
+    if q_start % chunk_depth == 0:
+        lo = q_start
+    else:
+        lo = q_start - (q_start % chunk_depth)
+
+    if q_stop % chunk_depth == 0:
+        hi = q_stop
+    else:
+        hi = q_stop + (chunk_depth - q_stop % chunk_depth)
+
+    return [lo + q_index, hi + q_index + 1]
+
+
+def block_compute(x_start, x_stop,
+                  y_start, y_stop,
+                  z_start, z_stop,
+                  origin=(0, 0, 0),
+                  block_size=(512, 512, 16)):
+    """
+    Get bounding box coordinates (in 3D) of small cutouts to request in
+    order to reconstitute a larger cutout.
+    Arguments:
+        x_start (int): The lower bound of dimension x
+        x_stop (int): The upper bound of dimension x
+        y_start (int): The lower bound of dimension y
+        y_stop (int): The upper bound of dimension y
+        z_start (int): The lower bound of dimension z
+        z_stop (int): The upper bound of dimension z
+    Returns:
+        [((x_start, x_stop), (y_start, y_stop), (z_start, z_stop)), ... ]
+    """
+    # x
+    x_bounds = range(origin[0], x_stop + block_size[0], block_size[0])
+    x_bounds = [x for x in x_bounds if (x > x_start and x < x_stop)]
+    if len(x_bounds) is 0:
+        x_slices = [(x_start, x_stop)]
+    else:
+        x_slices = []
+        for start_x in x_bounds[:-1]:
+            x_slices.append((start_x, start_x + block_size[0]))
+        x_slices.append((x_start, x_bounds[0]))
+        x_slices.append((x_bounds[-1], x_stop))
+
+    # y
+    y_bounds = range(origin[1], y_stop + block_size[1], block_size[1])
+    y_bounds = [y for y in y_bounds if (y > y_start and y < y_stop)]
+    if len(y_bounds) is 0:
+        y_slices = [(y_start, y_stop)]
+    else:
+        y_slices = []
+        for start_y in y_bounds[:-1]:
+            y_slices.append((start_y, start_y + block_size[1]))
+        y_slices.append((y_start, y_bounds[0]))
+        y_slices.append((y_bounds[-1], y_stop))
+
+    # z
+    z_bounds = range(origin[2], z_stop + block_size[2], block_size[2])
+    z_bounds = [z for z in z_bounds if (z > z_start and z < z_stop)]
+    if len(z_bounds) is 0:
+        z_slices = [(z_start, z_stop)]
+    else:
+        z_slices = []
+        for start_z in z_bounds[:-1]:
+            z_slices.append((start_z, start_z + block_size[2]))
+        z_slices.append((z_start, z_bounds[0]))
+        z_slices.append((z_bounds[-1], z_stop))
+
+    # alright, yuck. but now we have {x, y, z}_slices, each of which hold the
+    # start- and end-points of each cube-aligned boundary. For instance, if you
+    # requested z-slices 4 through 20, it holds [(4, 16), (16, 20)].
+
+    # For my next trick, I'll convert these to a list of:
+    # ((x_start, x_stop), (y_start, y_stop), (z_start, z_stop))
+
+    chunks = []
+    for x in x_slices:
+        for y in y_slices:
+            for z in z_slices:
+                chunks.append((x, y, z))
+    return chunks
