From abdffa36f19a53df85c67c749f7728930778761a Mon Sep 17 00:00:00 2001 From: Aaron Su Date: Tue, 18 Sep 2018 18:15:06 -0400 Subject: [PATCH 1/5] Start on datasources --- examples/Datasources.ipynb | 86 ++++++++++++++++++++++++++++++ rasterfoundry/api.py | 13 +++-- rasterfoundry/models/__init__.py | 1 + rasterfoundry/models/datasource.py | 72 +++++++++++++++++++++++++ 4 files changed, 167 insertions(+), 5 deletions(-) create mode 100644 examples/Datasources.ipynb create mode 100644 rasterfoundry/models/datasource.py diff --git a/examples/Datasources.ipynb b/examples/Datasources.ipynb new file mode 100644 index 0000000..0bd7b01 --- /dev/null +++ b/examples/Datasources.ipynb @@ -0,0 +1,86 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Datasources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "sys.path.append('/Users/xunzesu/Documents/work/azavea/raster-foundry-python-client/rasterfoundry')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from rasterfoundry.api import API\n", + "from rasterfoundry.models.datasource import Datasource\n", + "refresh_token = \"\"\n", + "api = API(refresh_token=refresh_token)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "api.get_datasources()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "band_one = api.create_datasource_band('red', 0, '')\n", + "band_two = api.create_datasource_band('green', 1, '')\n", + "band_three = api.create_datasource_band('blue', 2, '')\n", + "bands = [band_one, band_two, band_three]\n", + "bands" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "result = Datasource.create(api, 'Test Datasource', bands)\n", + "result" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.15" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/rasterfoundry/api.py b/rasterfoundry/api.py index b75d0e2..0b02963 100644 --- a/rasterfoundry/api.py +++ b/rasterfoundry/api.py @@ -10,7 +10,7 @@ from .aws.s3 import str_to_file from .exceptions import RefreshTokenException -from .models import Analysis, MapToken, Project, Export +from .models import Analysis, MapToken, Project, Export, Datasource from .settings import RV_TEMP_URI try: @@ -18,10 +18,10 @@ except ImportError: from urlparse import urlparse - SPEC_PATH = os.getenv( 'RF_API_SPEC_PATH', - 'https://raw.githubusercontent.com/raster-foundry/raster-foundry-api-spec/1.4.0/spec/spec.yml' + 'https://raw.githubusercontent.com/raster-foundry/raster-foundry-api-spec/feature' + + '/asu/update-first-class-objects-crud-specs/spec/spec.yml' ) @@ -167,8 +167,11 @@ def exports(self): exports.append(Export(export, self)) return exports - def get_datasources(self, **kwargs): - return self.client.Datasources.get_datasources(**kwargs).result() + def get_datasources(self): + datasources = [] + for datasource in self.client.Datasources.get_datasources().result().results: + datasources.append(Datasource(datasource, self)) + return datasources def get_scenes(self, **kwargs): bbox = kwargs.get('bbox') diff --git a/rasterfoundry/models/__init__.py b/rasterfoundry/models/__init__.py index 6d1a9a1..76ba4a1 100644 --- a/rasterfoundry/models/__init__.py +++ b/rasterfoundry/models/__init__.py @@ -3,3 +3,4 @@ from .upload import Upload # NOQA from .analysis import Analysis # NOQA from .export import Export # NOQA +from .datasource import Datasource # NOQA diff --git a/rasterfoundry/models/datasource.py b/rasterfoundry/models/datasource.py new file mode 100644 index 0000000..d8049c3 --- /dev/null +++ b/rasterfoundry/models/datasource.py @@ -0,0 +1,72 @@ +"""A Datasource is a source of data a scene uses""" + + +class Datasource(object): + """A Raster Foundry datasource""" + + def __repr__(self): + return ''.format(self.name) + + def __init__(self, datasource, api): + """Instantiate a new Datasource + + Args: + datasource (Datasource): generated Datasource object from specification + api (API): api used to make requests on behalf of a datasource + """ + self._datasource = datasource + self.api = api + + # A few things we care about + self.name = datasource.name + self.id = datasource.id + + @classmethod + def create_datasource_band(cls, name, number, wavelength): + """Create a datasource band + + Args: + name (str): name of band, e.g. 'read', 'blue', 'infrared', etc. + number (str): band number + wavelength (str): wavelength + + Returns: + dict: a band definition of a datasource band + """ + return dict( + name=name, + number=number, + wavelength=wavelength + ) + + @classmethod + def create(cls, api, name, bands, visibility='PRIVATE', extras={}): + """Post an upload to Raster Foundry for processing + + Args: + api (API): API to use for requests + name (str): name of datasource + bands (array): an array of create_datasource_band + visibility (str): datasource visibility, 'PRIVATE' by default + extras (dict): additional related information for a datasource, {} by default + + Returns: + Datasource: created datasource object in Raster Foundry + """ + datasource_created = dict( + name=name, + bands=bands, + visibility=visibility, + extras=extras, + composites={ + 'natural': { + 'label': 'Default', + 'value': { + 'redBand': 0, + 'greenBand': 1, + 'blueBand': 2 + } + } + } + ) + return api.client.Datasources.post_datasources(Datasource=datasource_created).result() From eb22872afae79587800cc83fe713a8c887e86eba Mon Sep 17 00:00:00 2001 From: Aaron Su Date: Mon, 24 Sep 2018 18:02:08 -0400 Subject: [PATCH 2/5] Make progress on datasource --- examples/Datasources.ipynb | 37 +++++++++++++++++++----------- rasterfoundry/api.py | 7 ++++-- rasterfoundry/models/datasource.py | 7 +++++- 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/examples/Datasources.ipynb b/examples/Datasources.ipynb index 0bd7b01..4a6bebf 100644 --- a/examples/Datasources.ipynb +++ b/examples/Datasources.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "import sys\n", - "sys.path.append('/Users/xunzesu/Documents/work/azavea/raster-foundry-python-client/rasterfoundry')" + "sys.path.append('/Users/asu/Documents/work/azavea/raster-foundry-python-client')" ] }, { @@ -25,8 +25,8 @@ "source": [ "from rasterfoundry.api import API\n", "from rasterfoundry.models.datasource import Datasource\n", - "refresh_token = \"\"\n", - "api = API(refresh_token=refresh_token)" + "refresh_token = '2oECtdfzvjjbmao4iNtCyLhrYeMb6Y_r3gQIAkMiSDkbg'\n", + "api = API(refresh_token=refresh_token, host='localhost:9091', scheme='http')" ] }, { @@ -44,11 +44,10 @@ "metadata": {}, "outputs": [], "source": [ - "band_one = api.create_datasource_band('red', 0, '')\n", - "band_two = api.create_datasource_band('green', 1, '')\n", - "band_three = api.create_datasource_band('blue', 2, '')\n", - "bands = [band_one, band_two, band_three]\n", - "bands" + "band_one = Datasource.create_datasource_band('red', '0', 100.0)\n", + "band_two = Datasource.create_datasource_band('green', '1', 200.0)\n", + "band_three = Datasource.create_datasource_band('blue', '2', 300.0)\n", + "result = Datasource.create(api, 'Test Datasource 5', [band_one, band_two, band_three])" ] }, { @@ -57,9 +56,21 @@ "metadata": {}, "outputs": [], "source": [ - "result = Datasource.create(api, 'Test Datasource', bands)\n", - "result" + "datasource = api.get_datasource_by_id(result.id)\n", + "datasource.name = datasource.name + ' updated'\n", + "datasource_dict = vars(datasource)['_Model__dict']\n", + "datasource_dict['createdAt'] = datasource.createdAt.isoformat()\n", + "datasource_dict['modifiedAt'] = datasource.modifiedAt.isoformat()\n", + "datasource_dict\n", + "Datasource.update(api, datasource.id, datasource_dict)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -71,14 +82,14 @@ "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.15" + "pygments_lexer": "ipython3", + "version": "3.7.0" } }, "nbformat": 4, diff --git a/rasterfoundry/api.py b/rasterfoundry/api.py index 0b02963..fa366e8 100644 --- a/rasterfoundry/api.py +++ b/rasterfoundry/api.py @@ -20,8 +20,7 @@ SPEC_PATH = os.getenv( 'RF_API_SPEC_PATH', - 'https://raw.githubusercontent.com/raster-foundry/raster-foundry-api-spec/feature' + - '/asu/update-first-class-objects-crud-specs/spec/spec.yml' + 'https://raw.githubusercontent.com/raster-foundry/raster-foundry-api-spec/feature/asu/update-first-class-object-crud-spec/spec/spec.yml' # NOQA ) @@ -173,6 +172,10 @@ def get_datasources(self): datasources.append(Datasource(datasource, self)) return datasources + def get_datasource_by_id(self, datasource_id): + return self.client.Datasources.get_datasources_datasourceID( + datasourceID=datasource_id).result() + def get_scenes(self, **kwargs): bbox = kwargs.get('bbox') if bbox and hasattr(bbox, 'bounds'): diff --git a/rasterfoundry/models/datasource.py b/rasterfoundry/models/datasource.py index d8049c3..7f18dbc 100644 --- a/rasterfoundry/models/datasource.py +++ b/rasterfoundry/models/datasource.py @@ -69,4 +69,9 @@ def create(cls, api, name, bands, visibility='PRIVATE', extras={}): } } ) - return api.client.Datasources.post_datasources(Datasource=datasource_created).result() + return api.client.Datasources.post_datasources(datasource=datasource_created).result() + + @classmethod + def update(cls, api, datasource_id, datasource): + return api.client.Datasources.put_datasources_datasourceID( + datasourceID=datasource_id, datasource=datasource).result() From ffad9d9e47731d48d743eec1a5824a064b8b69e9 Mon Sep 17 00:00:00 2001 From: Aaron Su Date: Tue, 25 Sep 2018 13:04:50 -0400 Subject: [PATCH 3/5] Finish datasource crud funcs and notebook --- examples/Datasources.ipynb | 82 +++++++++++++++++++++++++----- rasterfoundry/api.py | 2 +- rasterfoundry/models/datasource.py | 5 ++ 3 files changed, 76 insertions(+), 13 deletions(-) diff --git a/examples/Datasources.ipynb b/examples/Datasources.ipynb index 4a6bebf..c13b9f5 100644 --- a/examples/Datasources.ipynb +++ b/examples/Datasources.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "import sys\n", - "sys.path.append('/Users/asu/Documents/work/azavea/raster-foundry-python-client')" + "sys.path.append('/path/to/raster-foundry-python-client')" ] }, { @@ -25,8 +25,15 @@ "source": [ "from rasterfoundry.api import API\n", "from rasterfoundry.models.datasource import Datasource\n", - "refresh_token = '2oECtdfzvjjbmao4iNtCyLhrYeMb6Y_r3gQIAkMiSDkbg'\n", - "api = API(refresh_token=refresh_token, host='localhost:9091', scheme='http')" + "refresh_token = ''\n", + "api = API(refresh_token=refresh_token)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## List datasources" ] }, { @@ -38,6 +45,13 @@ "api.get_datasources()" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create datasource bands" + ] + }, { "cell_type": "code", "execution_count": null, @@ -47,7 +61,30 @@ "band_one = Datasource.create_datasource_band('red', '0', 100.0)\n", "band_two = Datasource.create_datasource_band('green', '1', 200.0)\n", "band_three = Datasource.create_datasource_band('blue', '2', 300.0)\n", - "result = Datasource.create(api, 'Test Datasource 5', [band_one, band_two, band_three])" + "bands = [band_one, band_two, band_three]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Add a new datasource" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "new_datasource = Datasource.create(api, 'Test Datasource 1', bands)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Get datasource by ID" ] }, { @@ -56,13 +93,14 @@ "metadata": {}, "outputs": [], "source": [ - "datasource = api.get_datasource_by_id(result.id)\n", - "datasource.name = datasource.name + ' updated'\n", - "datasource_dict = vars(datasource)['_Model__dict']\n", - "datasource_dict['createdAt'] = datasource.createdAt.isoformat()\n", - "datasource_dict['modifiedAt'] = datasource.modifiedAt.isoformat()\n", - "datasource_dict\n", - "Datasource.update(api, datasource.id, datasource_dict)" + "new_datasource_back = api.get_datasource_by_id(new_datasource.id)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Update a datasource" ] }, { @@ -70,7 +108,27 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "new_datasource_back.name = new_datasource_back.name + ' updated'\n", + "datasource_dict = vars(new_datasource_back)['_Model__dict']\n", + "Datasource.update(api, new_datasource_back.id, datasource_dict)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Delete a datasource" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "Datasource.delete(api, new_datasource_back.id)" + ] } ], "metadata": { diff --git a/rasterfoundry/api.py b/rasterfoundry/api.py index fa366e8..f5c9c93 100644 --- a/rasterfoundry/api.py +++ b/rasterfoundry/api.py @@ -20,7 +20,7 @@ SPEC_PATH = os.getenv( 'RF_API_SPEC_PATH', - 'https://raw.githubusercontent.com/raster-foundry/raster-foundry-api-spec/feature/asu/update-first-class-object-crud-spec/spec/spec.yml' # NOQA + 'https://raw.githubusercontent.com/raster-foundry/raster-foundry-api-spec/1.4.0/spec/spec.yml' # NOQA ) diff --git a/rasterfoundry/models/datasource.py b/rasterfoundry/models/datasource.py index 7f18dbc..bf36aa2 100644 --- a/rasterfoundry/models/datasource.py +++ b/rasterfoundry/models/datasource.py @@ -75,3 +75,8 @@ def create(cls, api, name, bands, visibility='PRIVATE', extras={}): def update(cls, api, datasource_id, datasource): return api.client.Datasources.put_datasources_datasourceID( datasourceID=datasource_id, datasource=datasource).result() + + @classmethod + def delete(cls, api, datasource_id): + return api.client.Datasources.delete_datasources_datasourceID( + datasourceID=datasource_id).result() From 6383c641b3d9d7602ad934566d85c0279395c290 Mon Sep 17 00:00:00 2001 From: Aaron Su Date: Wed, 26 Sep 2018 11:43:37 -0400 Subject: [PATCH 4/5] Update datasource notebook --- examples/Datasources.ipynb | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/examples/Datasources.ipynb b/examples/Datasources.ipynb index c13b9f5..4bb0315 100644 --- a/examples/Datasources.ipynb +++ b/examples/Datasources.ipynb @@ -7,16 +7,6 @@ "# Datasources" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import sys\n", - "sys.path.append('/path/to/raster-foundry-python-client')" - ] - }, { "cell_type": "code", "execution_count": null, From 19d6dcab9d022bfa25cbccbb83282f327ab6e70e Mon Sep 17 00:00:00 2001 From: Aaron Su Date: Thu, 27 Sep 2018 10:56:36 -0400 Subject: [PATCH 5/5] Update spec version and function calls --- rasterfoundry/api.py | 4 ++-- rasterfoundry/models/analysis.py | 4 ++-- rasterfoundry/models/export.py | 9 +++++---- rasterfoundry/models/project.py | 16 ++++++++-------- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/rasterfoundry/api.py b/rasterfoundry/api.py index f5c9c93..e778622 100644 --- a/rasterfoundry/api.py +++ b/rasterfoundry/api.py @@ -20,7 +20,7 @@ SPEC_PATH = os.getenv( 'RF_API_SPEC_PATH', - 'https://raw.githubusercontent.com/raster-foundry/raster-foundry-api-spec/1.4.0/spec/spec.yml' # NOQA + 'https://raw.githubusercontent.com/raster-foundry/raster-foundry-api-spec/1.11.0/spec/spec.yml' # NOQA ) @@ -203,7 +203,7 @@ def get_project_config(self, project_ids, annotations_uris=None): project_configs = [] for project_ind, project_id in enumerate(project_ids): proj = Project( - self.client.Imagery.get_projects_uuid(uuid=project_id).result(), + self.client.Imagery.get_projects_projectID(projectID=project_id).result(), self) if annotations_uris is None: diff --git a/rasterfoundry/models/analysis.py b/rasterfoundry/models/analysis.py index 3f4abcf..6cf8284 100644 --- a/rasterfoundry/models/analysis.py +++ b/rasterfoundry/models/analysis.py @@ -162,8 +162,8 @@ def get_center(self): # fetch first one from api and use that project's coordinates input = inputs.pop() project = Project( - self.api.client.Imagery.get_projects_uuid( - uuid=input.get('projId') + self.api.client.Imagery.get_projects_projectID( + projectID=input.get('projId') ).result(), self.api ) diff --git a/rasterfoundry/models/export.py b/rasterfoundry/models/export.py index adbd188..9268cfe 100644 --- a/rasterfoundry/models/export.py +++ b/rasterfoundry/models/export.py @@ -35,7 +35,8 @@ def __init__(self, export, api): @property def files(self): try: - fnames_res = self.api.client.Imagery.get_exports_uuid_files(uuid=self.id).result() + fnames_res = self.api.client.Imagery.get_exports_exportID_files( + exportID=self.id).result() fnames = filter( lambda name: name.upper() != 'RFUploadAccessTestFile'.upper(), fnames_res) @@ -74,11 +75,11 @@ def poll_export_status(cls, 'function will never return. You may have left off FAILED by accident. ' 'If that is the case, you should include FAILED in until and try again.' ) - export = api.client.Imagery.get_exports_uuid(uuid=export_id).result() + export = api.client.Imagery.get_exports_exportID(exportID=export_id).result() while export.exportStatus not in until: time.sleep(delay) - export = api.client.Imagery.get_exports_uuid( - uuid=export_id).result() + export = api.client.Imagery.get_exports_exportID( + exportID=export_id).result() return Export(export, api) @classmethod diff --git a/rasterfoundry/models/project.py b/rasterfoundry/models/project.py index fd78ff8..9253f57 100644 --- a/rasterfoundry/models/project.py +++ b/rasterfoundry/models/project.py @@ -189,13 +189,13 @@ def post_annotations(self, annotations_uri): 'confidence': properties['score'] } - self.api.client.Imagery.post_projects_uuid_annotations( - uuid=self.id, annotations=rf_annotations).future.result() + self.api.client.Imagery.post_projects_projectID_annotations( + projectID=self.id, annotations=rf_annotations).future.result() def get_annotations(self): def get_page(page): - return self.api.client.Imagery.get_projects_uuid_annotations( - uuid=self.id, page=page).result() + return self.api.client.Imagery.get_projects_projectID_annotations( + projectID=self.id, page=page).result() return get_all_paginated(get_page, list_field='features') @@ -214,15 +214,15 @@ def json_serial(obj): def get_scenes(self): def get_page(page): - return self.api.client.Imagery.get_projects_uuid_scenes( - uuid=self.id, page=page).result() + return self.api.client.Imagery.get_projects_projectID_scenes( + projectID=self.id, page=page).result() return get_all_paginated(get_page) def get_ordered_scene_ids(self): def get_page(page): - return self.api.client.Imagery.get_projects_uuid_order( - uuid=self.id, page=page).result() + return self.api.client.Imagery.get_projects_projectID_order( + projectID=self.id, page=page).result() # Need to reverse so that order is from bottom-most to top-most layer. return list(reversed(get_all_paginated(get_page)))