Merge pull request #54 from azavea/feature/dpd/gdal-dem-processing

Add GDALDemProcessing, other cleanup

.eslintrc

@@ -51,7 +51,7 @@
     "keyword-spacing": [2, {"before": true, "after": true}],
     "linebreak-style": 0,
     "max-depth": 0,
-    "max-len": [2, 120, 4],
+    "max-len": [2, 100, 4],
     "max-nested-callbacks": 0,
     "max-params": 0,
     "max-statements": 0,
@@ -155,7 +155,7 @@
     "operator-linebreak": [2, "after"],
     "padded-blocks": 0,
     "quote-props": 0,
-    "quotes": [2, "single", "avoid-escape"],
+    "quotes": [2, "single", {"allowTemplateLiterals": true, "avoidEscape": true}],
     "radix": 2,
     "semi": [2, "always"],
     "semi-spacing": 0,

CHANGELOG

@@ -1,6 +1,8 @@
 ## Upcoming release
 - Add information on contributing to the README
 - Apply code auto-formatting
+- Improve error messages with originating function names
+- Add GDALDataset.render() to provide gdaldem functionality
 
 ## 1.0.0-rc.1 (2020-07-24)
 - Add loam.rasterize() wrapper for GDALRasterize()

README.md

@@ -138,6 +138,19 @@ Image reprojection and warping utility. This is the equivalent of the [gdalwarp]
 #### Return value
 A promise that resolves to a new `GDALDataset`.
 
+<br />
+
+### `GDALDataset.render(mode, args, colors)`
+Utility for rendering and computing DEM metrics. This is the equivalent of the [gdaldem](https://gdal.org/programs/gdaldem.html) command.
+
+**Note**: This returns a new `GDALDataset` object but does not perform any immediate calculation. Instead, calls to `.render()` are evaluated lazily (as with `convert()` and `warp()`, above). The render operation is only evaluated when necessary in order to access some property of the dataset, such as its size, bytes, or band count. Successive calls to `.warp()` and `.convert()` can be lazily chained onto datasets produced by `.render()`, and vice-versa.
+#### Parameters
+- `mode`: One of ['hillshade', 'slope','aspect', 'color-relief', 'TRI', 'TPI', 'roughness']. See the [`gdaldem documentation`](https://gdal.org/programs/gdaldem.html#cmdoption-arg-mode) for an explanation of the function of each mode.
+- `args`: An array of strings, each representing a single [command-line argument](https://gdal.org/programs/gdaldem.html#synopsis) accepted by the `gdaldem` command. The `inputdem` and `output_xxx_map` parameters should be omitted; these are handled by `GDALDataset`. Example: `ds.render('hillshade', ['-of', 'PNG'])`
+- `colors`: If (and only if) `mode` is equal to 'color-relief', an array of strings representing lines in [the color text file](https://gdal.org/programs/gdaldem.html#color-relief). Example: `ds.render('color-relief', ['-of', 'PNG'], ['993.0 255 0 0'])`. See the [`gdaldem documentation`](https://gdal.org/programs/gdaldem.html#cmdoption-arg-color_text_file) for an explanation of the text file syntax.
+#### Return value
+A promise that resolves to a new `GDALDataset`.
+
 # Developing
 
 After cloning,

src/gdalDataset.js

@@ -70,6 +70,24 @@ export class GDALDataset {
         });
     }
 
+    render(mode, args, colors) {
+        return new Promise((resolve, reject) => {
+            // DEMProcessing requires an auxiliary color definition file in some cases, so the API
+            // can't be easily represented as an array of strings. This packs the user-friendly
+            // interface of render() into an array that the worker communication machinery can
+            // easily make use of. It'll get unpacked inside the worker. Yet another reason to use
+            // something like comlink (#49)
+            const cliOrderArgs = [mode, colors].concat(args);
+
+            resolve(
+                new GDALDataset(
+                    this.source,
+                    this.operations.concat(new DatasetOperation('GDALDEMProcessing', cliOrderArgs))
+                )
+            );
+        });
+    }
+
     close() {
         return new Promise((resolve, reject) => {
             const warningMsg =

src/stringParamAllocator.js

@@ -23,16 +23,17 @@ export default class ParamParser {
         let argPtrsArray = Uint32Array.from(
             self.args
                 .map((argStr) => {
-                    return Module._malloc(Module.lengthBytesUTF8(argStr) + 1); // +1 for the null terminator byte
+                    // +1 for the null terminator byte
+                    return Module._malloc(Module.lengthBytesUTF8(argStr) + 1);
                 })
                 .concat([0])
         );
-        // ^ In addition to each individual argument being null-terminated, the GDAL docs specify that
-        // GDALTranslateOptionsNew takes its options passed in as a null-terminated array of
+        // ^ In addition to each individual argument being null-terminated, the GDAL docs specify
+        // that GDALTranslateOptionsNew takes its options passed in as a null-terminated array of
         // pointers, so we have to add on a null (0) byte at the end.
 
-        // Next, we need to write each string from the JS string array into the Emscripten heap space
-        // we've allocated for it.
+        // Next, we need to write each string from the JS string array into the Emscripten heap
+        // space we've allocated for it.
         self.args.forEach(function (argStr, i) {
             Module.stringToUTF8(argStr, argPtrsArray[i], Module.lengthBytesUTF8(argStr) + 1);
         });

src/worker.js

@@ -6,6 +6,7 @@
 import wGDALOpen from './wrappers/gdalOpen.js';
 import wGDALRasterize from './wrappers/gdalRasterize.js';
 import wGDALClose from './wrappers/gdalClose.js';
+import wGDALDEMProcessing from './wrappers/gdalDemProcessing.js';
 import wGDALGetRasterCount from './wrappers/gdalGetRasterCount.js';
 import wGDALGetRasterXSize from './wrappers/gdalGetRasterXSize.js';
 import wGDALGetRasterYSize from './wrappers/gdalGetRasterYSize.js';
@@ -143,6 +144,19 @@ self.Module = {
                 errorHandling,
                 DATASETPATH
             );
+            registry.GDALDEMProcessing = wGDALDEMProcessing(
+                self.Module.cwrap('GDALDEMProcessing', 'number', [
+                    'string', // Destination dataset path or NULL
+                    'number', // GDALDatasetH destination dataset
+                    // eslint-disable-next-line max-len
+                    'string', // The processing to apply (one of "hillshade", "slope", "aspect", "color-relief", "TRI", "TPI", "roughness")
+                    'string', // Color file path (when previous is "hillshade") or NULL (otherwise)
+                    'number', // GDALDEMProcessingOptions *
+                    'number', // int * to use for error reporting
+                ]),
+                errorHandling,
+                DATASETPATH
+            );
             registry.LoamFlushFS = function () {
                 let datasetFolders = FS.lookupPath(DATASETPATH).node.contents;
 
@@ -230,6 +244,7 @@ onmessage = function (msg) {
             postMessage({
                 success: false,
                 message:
+                // eslint-disable-next-line max-len
                     'Worker could not parse message: either func + args or accessor + dataset is required',
                 id: msg.data.id,
             });

src/wrappers/gdalClose.js

@@ -14,13 +14,14 @@ export default function (GDALClose, errorHandling) {
         let errorType = errorHandling.CPLGetLastErrorType();
 
         // Check for errors; throw if error is detected
+        // Note that due to https://github.com/ddohler/gdal-js/issues/38 this can only check for
+        // CEFatal errors in order to avoid raising an exception on GDALClose
         if (
-            errorType === errorHandling.CPLErr.CEFailure ||
             errorType === errorHandling.CPLErr.CEFatal
         ) {
             let message = errorHandling.CPLGetLastErrorMsg();
 
-            throw new Error(message);
+            throw new Error('Error in GDALClose: ' + message);
         } else {
             return result;
         }

src/wrappers/gdalDemProcessing.js

@@ -0,0 +1,144 @@
+/* global Module, FS, MEMFS */
+import randomKey from '../randomKey.js';
+import guessFileExtension from '../guessFileExtension.js';
+import ParamParser from '../stringParamAllocator.js';
+
+// TODO: This is another good reason to switch to Typescript #55
+const DEMProcessingModes = Object.freeze({
+    hillshade: 'hillshade',
+    slope: 'slope',
+    aspect: 'aspect',
+    'color-relief': 'color-relief',
+    TRI: 'TRI',
+    TPI: 'TPI',
+    roughness: 'roughness',
+});
+
+export default function (GDALDEMProcessing, errorHandling, rootPath) {
+    /* mode: one of the options in DEMProcessingModes
+     * colors: Array of strings matching the format of the color file defined at
+     *   https://gdal.org/programs/gdaldem.html#color-relief
+     * args: Array of strings matching the remaining arguments of gdaldem, excluding output filename
+     */
+    return function (dataset, packedArgs) {
+        // TODO: Make this unnecessary by switching to comlink or similar (#49)
+        const mode = packedArgs[0];
+        const colors = packedArgs[1];
+        const args = packedArgs.slice(2);
+
+        if (!mode || !DEMProcessingModes.hasOwnProperty(mode)) {
+            throw new Error(`mode must be one of {Object.keys(DEMProcessingModes)}`);
+        } else if (mode === DEMProcessingModes['color-relief'] && !colors) {
+            throw new Error(
+                'A color definition array must be provided if `mode` is "color-relief"'
+            );
+        } else if (mode !== DEMProcessingModes['color-relief'] && colors && colors.length > 0) {
+            throw new Error(
+                'A color definition array should not be provided if `mode` is not "color-relief"'
+            );
+        }
+
+        // If mode is hillshade, we need to create a color file path
+        let colorFilePath = null;
+
+        if (mode === DEMProcessingModes['color-relief']) {
+            colorFilePath = rootPath + randomKey() + '.txt';
+
+            FS.writeFile(colorFilePath, colors.join('\n'));
+        }
+        let params = new ParamParser(args);
+
+        params.allocate();
+
+        // argPtrsArrayPtr is now the address of the start of the list of
+        // pointers in Emscripten heap space. Each pointer identifies the address of the start of a
+        // parameter string, also stored in heap space. This is the direct equivalent of a char **,
+        // which is what GDALDEMProcessingOptionsNew requires.
+        const demOptionsPtr = Module.ccall(
+            'GDALDEMProcessingOptionsNew',
+            'number',
+            ['number', 'number'],
+            [params.argPtrsArrayPtr, null]
+        );
+
+        // Validate that the options were correct
+        const optionsErrType = errorHandling.CPLGetLastErrorType();
+
+        if (
+            optionsErrType === errorHandling.CPLErr.CEFailure ||
+            optionsErrType === errorHandling.CPLErr.CEFatal
+        ) {
+            if (colorFilePath) {
+                FS.unlink(colorFilePath);
+            }
+            params.deallocate();
+            const message = errorHandling.CPLGetLastErrorMsg();
+
+            throw new Error('Error in GDALDEMProcessing: ' + message);
+        }
+
+        // Now that we have our options, we need to make a file location to hold the output.
+        let directory = rootPath + randomKey();
+
+        FS.mkdir(directory);
+        // This makes it easier to remove later because we can just unmount rather than recursing
+        // through the whole directory structure.
+        FS.mount(MEMFS, {}, directory);
+        let filename = randomKey(8) + '.' + guessFileExtension(args);
+
+        let filePath = directory + '/' + filename;
+
+        // And then we can kick off the actual processing.
+        // The last parameter is an int* that can be used to detect certain kinds of errors,
+        // but I'm not sure how it works yet and whether it gives the same or different information
+        // than CPLGetLastErrorType.
+        // Malloc ourselves an int and set it to 0 (False)
+        let usageErrPtr = Module._malloc(Int32Array.BYTES_PER_ELEMENT);
+
+        Module.setValue(usageErrPtr, 0, 'i32');
+
+        let newDatasetPtr = GDALDEMProcessing(
+            filePath, // Output
+            dataset,
+            mode,
+            colorFilePath,
+            demOptionsPtr,
+            usageErrPtr
+        );
+
+        let errorType = errorHandling.CPLGetLastErrorType();
+        // If we ever want to use the usage error pointer:
+        // let usageErr = Module.getValue(usageErrPtr, 'i32');
+
+        // The final set of cleanup we need to do, in a function to avoid writing it twice.
+        function cleanUp() {
+            if (colorFilePath) {
+                FS.unlink(colorFilePath);
+            }
+            Module.ccall('GDALDEMProcessingOptionsFree', null, ['number'], [demOptionsPtr]);
+            Module._free(usageErrPtr);
+            params.deallocate();
+        }
+
+        // Check for errors; clean up and throw if error is detected
+        if (
+            errorType === errorHandling.CPLErr.CEFailure ||
+            errorType === errorHandling.CPLErr.CEFatal
+        ) {
+            cleanUp();
+            const message = errorHandling.CPLGetLastErrorMsg();
+
+            throw new Error('Error in GDALDEMProcessing: ' + message);
+        } else {
+            const result = {
+                datasetPtr: newDatasetPtr,
+                filePath: filePath,
+                directory: directory,
+                filename: filename,
+            };
+
+            cleanUp();
+            return result;
+        }
+    };
+}

src/wrappers/gdalGetGeoTransform.js

@@ -33,10 +33,10 @@ export default function (GDALGetGeoTransform, errorHandling) {
             Module._free(byteOffset);
             let message = errorHandling.CPLGetLastErrorMsg();
 
-            throw new Error(message);
+            throw new Error('Error in GDALGetGeoTransform: ' + message);
         } else {
-            // To avoid memory leaks in the Emscripten heap, we need to free up the memory we allocated
-            // after we've converted it into a Javascript object.
+            // To avoid memory leaks in the Emscripten heap, we need to free up the memory we
+            // allocated after we've converted it into a Javascript object.
             let result = Array.from(geoTransform);
 
             Module._free(byteOffset);

src/wrappers/gdalGetProjectionRef.js

@@ -11,7 +11,7 @@ export default function (GDALGetProjectionRef, errorHandling) {
         ) {
             let message = errorHandling.CPLGetLastErrorMsg();
 
-            throw new Error(message);
+            throw new Error('Error in GDALGetProjectionRef: ' + message);
         } else {
             return result;
         }

src/wrappers/gdalGetRasterCount.js

@@ -11,7 +11,7 @@ export default function (GDALGetRasterCount, errorHandling) {
         ) {
             let message = errorHandling.CPLGetLastErrorMsg();
 
-            throw new Error(message);
+            throw new Error('Error in GDALGetRasterCount: ' + message);
         } else {
             return result;
         }

src/wrappers/gdalGetRasterXSize.js

@@ -11,7 +11,7 @@ export default function (GDALGetRasterXSize, errorHandling) {
         ) {
             let message = errorHandling.CPLGetLastErrorMsg();
 
-            throw new Error(message);
+            throw new Error('Error in GDALGetRasterXSize: ' + message);
         } else {
             return result;
         }

src/wrappers/gdalGetRasterYSize.js

@@ -11,7 +11,7 @@ export default function (GDALGetRasterYSize, errorHandling) {
         ) {
             let message = errorHandling.CPLGetLastErrorMsg();
 
-            throw Error(message);
+            throw Error('Error in GDALGetRasterYSize: ' + message);
         } else {
             return result;
         }

src/wrappers/gdalOpen.js

@@ -31,7 +31,7 @@ export default function (GDALOpen, errorHandling, rootPath) {
             FS.rmdir(directory);
             let message = errorHandling.CPLGetLastErrorMsg();
 
-            throw new Error(message);
+            throw new Error('Error in GDALOpen: ' + message);
         } else {
             return {
                 datasetPtr: datasetPtr,

src/wrappers/gdalRasterize.js

@@ -40,11 +40,12 @@ export default function (GDALRasterize, errorHandling, rootPath) {
             params.deallocate();
             const message = errorHandling.CPLGetLastErrorMsg();
 
-            throw new Error(message);
+            throw new Error('Error in GDALRasterize: ' + message);
         }
 
-        // Now that we have our translate options, we need to make a file location to hold the output.
-        let directory = rootPath + '/' + randomKey();
+        // Now that we have our translate options, we need to make a file location to hold the
+        // output.
+        let directory = rootPath + randomKey();
 
         FS.mkdir(directory);
         // This makes it easier to remove later because we can just unmount rather than recursing
@@ -93,7 +94,7 @@ export default function (GDALRasterize, errorHandling, rootPath) {
             cleanUp();
             const message = errorHandling.CPLGetLastErrorMsg();
 
-            throw new Error(message);
+            throw new Error('Error in GDALRasterize: ' + message);
         } else {
             const result = {
                 datasetPtr: newDatasetPtr,