Support local file uploads from client (#15)

* Support local file uploads from client

Author: jisantuc

rasterfoundry/models/tests/__init__.py

@@ -0,0 +1,78 @@
+import os
+import shutil
+import random
+from string import ascii_letters
+
+
+import pytest
+
+
+from ..upload import Upload
+
+
+@pytest.fixture
+def datasource():
+    return 'fooDat'
+
+
+@pytest.fixture
+def organization():
+    return 'fooOrg'
+
+
+@pytest.fixture
+def random_tifs():
+    return [
+        ''.join([random.choice(ascii_letters) for _ in range(10)]) + '.tif'
+        for _ in range(10)
+    ]
+
+
+@pytest.fixture
+def bogus_bucket():
+    return 'fooBucket'
+
+
+@pytest.fixture
+def bogus_prefix():
+    return 'fooPrefix'
+
+
+def test_file_globbing(datasource, organization, random_tifs, bogus_bucket,
+                       bogus_prefix):
+    tifs = os.path.join('/tmp', 'tifs')
+    other_tifs = os.path.join('/tmp', 'other_tifs')
+    os.mkdir(tifs)
+    os.mkdir(other_tifs)
+    paths = []
+    for i, t in enumerate(random_tifs):
+        if i % 2 == 0:
+            out_path = os.path.join(tifs, t)
+        else:
+            out_path = os.path.join(other_tifs, t)
+        with open(out_path, 'w') as outf:
+            outf.write('a tif')
+            paths.append(out_path)
+
+    upload_create = Upload.upload_create_from_files(
+        datasource, organization, '/tmp/**/*.tif', bogus_bucket, bogus_prefix,
+        dry_run=True
+    )
+    upload_fnames = [os.path.split(f)[-1] for f in upload_create['files']]
+    src_fnames = [os.path.split(f)[-1] for f in paths]
+    assert set(upload_fnames) == set(src_fnames)
+
+    shutil.rmtree(tifs)
+    shutil.rmtree(other_tifs)
+
+
+def test_no_file_globbing(datasource, organization, bogus_bucket,
+                          bogus_prefix):
+    files = ['bar.tif', 'foo.tif']
+    upload_create = Upload.upload_create_from_files(
+        datasource, organization, files, bogus_bucket, bogus_prefix,
+        dry_run=True
+    )
+
+    upload_fnames = [os.path.split(f)[-1] for f in upload_create['files']]
+    assert upload_fnames == files

rasterfoundry/models/tests/test_upload.py

@@ -0,0 +1,115 @@
+"""An Upload is raw data to be transformed into a Scene"""
+import glob
+import os
+
+import boto3
+
+
+class Upload(object):
+    """A Raster Foundry upload"""
+
+    s3_client = boto3.client('s3')
+
+    def __repr__(self):
+        return '<Upload - {}>'.format(self.name)
+
+    def __init__(self, upload, api):
+        """Instantiate a new Upload
+
+        Args:
+            upload (Upload): generated Upload object from specification
+            api (API): api used to make requests on behalf of an upload
+        """
+
+        self._upload = upload
+        self.api = api
+
+        self.id = upload.id
+        self.upload_type = upload.uploadType
+        self.metadata = upload.metadata
+        self.files = upload.files
+
+    @classmethod
+    def upload_create_from_files(
+            cls, datasource, organization, paths_to_tifs,
+            dest_bucket, dest_prefix, metadata={}, visibility='PRIVATE',
+            project_id=None, dry_run=False
+    ):
+        """Create an Upload from a set of tifs
+
+        Args:
+            datasource (str): UUID of the datasource this upload belongs to
+            organization (str): UUID of the organization this upload belongs to
+            paths_to_tifs (str | str[]): which tifs to upload. If passed a
+                string, files will be the unix path expansion of the passed
+                string, e.g., '*.tif' will become an array of all of the tifs
+                in the current working directory. If passed a list, files will
+                be exactly those files in the list.
+            dest_bucket (str): s3 bucket to upload local files to. If the
+                Raster Foundry application does not have permission to read
+                from this location, the upload will fail to process.
+            dest_prefix (str): s3 prefix to upload local files to. If the
+                Raster Foundry application does not have permission to read
+                from this location, the upload will fail to process.
+            metadata (dict): Additional information to store with this upload.
+                acquisitionDate and cloudCover will be parsed into any created
+                scenes.
+            visibility (str): PUBLIC, PRIVATE, or ORGANIZATION visibility level
+                for the created scenes
+            project_id (str): UUID of the project scenes from this upload
+                should be added to
+            dry_run (bool): whether to perform side-effecting actions like
+                uploads to s3
+
+        Returns:
+            dict: splattable object to post to /uploads/
+        """
+        if isinstance(paths_to_tifs, str):
+            paths = glob.glob(paths_to_tifs)
+        else:
+            paths = paths_to_tifs
+        upload_status = 'UPLOADED'
+        file_type = 'GEOTIFF'
+
+        files = []
+        for f in paths:
+            fname = os.path.split(f)[-1]
+            key = '/'.join([x for x in [dest_prefix, fname] if x])
+            dest_path = 's3://' + '/'.join(
+                [x for x in [dest_bucket, dest_prefix, fname] if x]
+            )
+            if not dry_run:
+                with open(f, 'r') as inf:
+                    cls.s3_client.put_object(
+                        Body=inf.read(),
+                        Bucket=dest_bucket,
+                        Key=key
+                    )
+            files.append(dest_path)
+
+        return dict(
+            uploadStatus=upload_status,
+            files=files,
+            uploadType='S3',
+            fileType=file_type,
+            datasource=datasource,
+            organizationId=organization,
+            metadata=metadata,
+            visibility=visibility,
+            projectId=project_id
+        )
+
+    @classmethod
+    def create(cls, api, upload_create):
+        """Post an upload to Raster Foundry for processing
+
+        Args:
+            api (API): API to use for requests
+            upload_create (dict): post parameters for /uploads. See
+                upload_create_from_files
+
+        Returns:
+            Upload: created object in Raster Foundry
+        """
+
+        return api.client.Imagery.post_uploads(Upload=upload_create).result()

rasterfoundry/models/upload.py

@@ -24,7 +24,8 @@
         'cryptography == 1.8.1',
         'pyasn1 >= 0.2.3',
         'requests >= 2.9.1',
-        'bravado >= 8.4.0'
+        'bravado >= 8.4.0',
+        'boto3 >= 1.4.4'
     ],
     extras_require={
         'notebook': [