Add method to evaluate model

Author: lewfish

examples/Ship Detection.ipynb

@@ -9,12 +9,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
-   "metadata": {},
+   "execution_count": 1,
+   "metadata": {
+    "collapsed": true
+   },
    "outputs": [],
    "source": [
     "from rasterfoundry.api import API\n",
-    "refresh_token = '<INSERT-TOKEN>'\n",
+    "refresh_token = '<'\n",
     "api = API(refresh_token=refresh_token)"
    ]
   },
@@ -520,6 +522,18 @@
     "# Raster Vision"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "from rasterfoundry.utils import RasterVisionBatchClient\n",
+    "rv_batch_client = RasterVisionBatchClient()"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -544,9 +558,6 @@
     }
    ],
    "source": [
-    "from rasterfoundry.utils import RasterVisionBatchClient\n",
-    "rv_batch_client = RasterVisionBatchClient(branch_name='lf/filter')\n",
-    "\n",
     "# A set of projects and corresponding annotation GeoJSON files.\n",
     "project_ids = [\n",
     "    '357ddf1f-2e5e-4420-8e75-0eed12d2d20f',\n",
@@ -610,6 +621,53 @@
     "                          label_map_uri, predictions_uri)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "collapsed": true
+   },
+   "source": [
+    "## Evaluating a model on a set of projects"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "u'47caf222-729c-4351-85e3-9fe8e2edf1ed'"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# A set of projects and corresponding annotation GeoJSON files.\n",
+    "project_ids = [\n",
+    "    '357ddf1f-2e5e-4420-8e75-0eed12d2d20f',\n",
+    "    '1ab2734c-d2b1-4e0b-8c54-24f4f121c77b',\n",
+    "]\n",
+    "\n",
+    "annotation_uris = [\n",
+    "    's3://raster-vision-od/raw-data/annotations/jm-ships/2017-09-18-singapore-final.geojson',\n",
+    "    's3://raster-vision-od/raw-data/annotations/jm-ships/2017-07-04-panama-final.geojson',\n",
+    "]\n",
+    "\n",
+    "output_uri = 's3://raster-vision-od/evals/singapore-panama.json'\n",
+    "label_map_uri = 's3://raster-vision-od/configs/label-maps/jm-ships-single-label.pbtxt'\n",
+    "inference_graph_uri = 's3://raster-vision-od/trained-models/lhf-ships-neg0/inference_graph.pb'\n",
+    "\n",
+    "api.start_eval_model_job(\n",
+    "    rv_batch_client, inference_graph_uri,\n",
+    "    project_ids, annotation_uris, label_map_uri,\n",
+    "    output_uri)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,

rasterfoundry/api.py

@@ -208,3 +208,48 @@ def start_prep_train_data_job(self, rv_batch_client, project_ids,
         job_name = 'prep_train_data_{}'.format(uuid.uuid1())
         job_id = rv_batch_client.start_raster_vision_job(job_name, command)
         return job_id
+
+    def start_eval_model_job(self, rv_batch_client, inference_graph_uri,
+                             project_ids, annotation_uris, label_map_uri,
+                             output_uri,
+                             proj_config_dir_uri=RV_PROJ_CONFIG_DIR_URI,
+                             channel_order=None):
+        """Start a Batch job to evaluate a model using a set of projects.
+
+        Args:
+            rv_batch_client: a RasterVisionBatchClient object used to start
+                Batch jobs
+            project_ids (list of str): ids of projects to make train data for
+            annotation_uris (list of str): annotation URIs for projects
+            label_map_uri (str): URI of output label map
+            output_uri (str): URI of output zip file
+            proj_config_dir_uri (str): The root of generated URIs for config
+                files
+            channel_order: list of length 3 with GeoTIFF channel indices to
+                map to RGB.
+
+        Returns:
+            job_id (str): job_id of job started on Batch
+        """
+        project_configs = self.get_project_configs(
+            project_ids, annotation_uris)
+        projects_uri = upload_raster_vision_config(
+            project_configs, proj_config_dir_uri)
+
+        base_command = \
+            'python -m rv.run eval_model --chip-size 300 '
+
+        channel_order_opt = ''
+        if channel_order is not None:
+            channel_order_str = ' '.join([
+                str(channel_ind) for channel_ind in channel_order])
+            channel_order_opt = ('--channel-order {} '
+                                 .format(channel_order_str))
+
+        command = (base_command + channel_order_opt + '{} {} {} {}')
+        command = command.format(inference_graph_uri, projects_uri,
+                                 label_map_uri, output_uri)
+
+        job_name = 'eval_model_{}'.format(uuid.uuid1())
+        job_id = rv_batch_client.start_raster_vision_job(job_name, command)
+        return job_id