Merge pull request #1033 from AtlasOfLivingAustralia/feature/issue2789

shape file upload and shape conversion to GeoJson AtlasOfLivingAustralia/fieldcapture#2789

grails-app/controllers/au/org/ala/ecodata/SpatialController.groovy

@@ -0,0 +1,120 @@
+package au.org.ala.ecodata
+
+import au.org.ala.ecodata.spatial.SpatialConversionUtils
+import au.org.ala.ecodata.spatial.SpatialUtils
+import org.apache.commons.fileupload.servlet.ServletFileUpload
+import org.apache.commons.io.IOUtils
+import org.apache.commons.lang3.tuple.Pair
+import org.locationtech.jts.geom.Geometry
+import org.springframework.web.multipart.MultipartFile
+
+import javax.servlet.http.HttpServletResponse
+@au.ala.org.ws.security.RequireApiKey(scopesFromProperty=["app.readScope"])
+class SpatialController {
+
+    static responseFormats = ['json', 'xml']
+    static allowedMethods = [uploadShapeFile: "POST", getShapeFileFeatureGeoJson: "GET"]
+
+    @au.ala.org.ws.security.RequireApiKey(scopesFromProperty=["app.writeScope"])
+    def uploadShapeFile() {
+        // Use linked hash map to maintain key ordering
+        Map<Object, Object> retMap = new LinkedHashMap<Object, Object>()
+
+        File tmpZipFile = File.createTempFile("shpUpload", ".zip")
+
+        if (ServletFileUpload.isMultipartContent(request)) {
+            // Parse the request
+            Map<String, MultipartFile> items = request.getFileMap()
+
+            if (items.size() == 1) {
+                MultipartFile fileItem = items.values()[0]
+                IOUtils.copy(fileItem.getInputStream(), new FileOutputStream(tmpZipFile))
+                retMap.putAll(handleZippedShapeFile(tmpZipFile))
+                response.setStatus(HttpServletResponse.SC_OK)
+            } else {
+                response.setStatus(HttpServletResponse.SC_BAD_REQUEST)
+                retMap.put("error", "Multiple files sent in request. A single zipped shape file should be supplied.")
+            }
+        }
+
+        respond retMap
+    }
+
+    @au.ala.org.ws.security.RequireApiKey(scopesFromProperty=["app.writeScope"])
+    def getShapeFileFeatureGeoJson() {
+        Map retMap
+        String shapeId = params.shapeFileId
+        String featureIndex = params.featureId
+        if (featureIndex != null && shapeId != null) {
+
+            retMap = processShapeFileFeatureRequest(shapeId, featureIndex)
+            if(retMap.geoJson == null) {
+                response.setStatus(HttpServletResponse.SC_BAD_REQUEST)
+            }
+            else {
+                response.setStatus(HttpServletResponse.SC_OK)
+            }
+        }
+        else {
+            response.setStatus(HttpServletResponse.SC_BAD_REQUEST)
+            retMap = ["error": "featureId and shapeFileId must be supplied"]
+        }
+
+        respond retMap
+    }
+
+    private Map<String, String> processShapeFileFeatureRequest(String shapeFileId, String featureIndex) {
+        Map<String, Object> retMap = new HashMap<String, Object>()
+
+        try {
+            File shpFileDir = new File(System.getProperty("java.io.tmpdir"), shapeFileId)
+            Geometry geoJson = SpatialUtils.getShapeFileFeaturesAsGeometry(shpFileDir, featureIndex)
+
+            if (geoJson == null) {
+                retMap.put("error", "Invalid geometry")
+                return retMap
+            }
+            else {
+                if (geoJson.getCoordinates().flatten().size() > grailsApplication.config.getProperty("shapefile.simplify.threshhold", Integer,  50_000)) {
+                    geoJson = GeometryUtils.simplify(geoJson, grailsApplication.config.getProperty("shapefile.simplify.tolerance", Double, 0.0001))
+                }
+
+                retMap.put("geoJson", GeometryUtils.geometryToGeoJsonMap(geoJson, grailsApplication.config.getProperty("shapefile.geojson.decimal", Integer, 20)))
+            }
+        } catch (Exception ex) {
+            log.error("Error processsing shapefile feature request", ex)
+            retMap.put("error", ex.getMessage())
+        }
+
+        return retMap
+    }
+
+    private static Map<Object, Object> handleZippedShapeFile(File zippedShp) throws IOException {
+        // Use linked hash map to maintain key ordering
+        Map<Object, Object> retMap = new LinkedHashMap<Object, Object>()
+
+        Pair<String, File> idFilePair = SpatialConversionUtils.extractZippedShapeFile(zippedShp)
+        String uploadedShpId = idFilePair.getLeft()
+        File shpFile = idFilePair.getRight()
+
+        retMap.put("shp_id", uploadedShpId)
+
+        List<List<Pair<String, Object>>> manifestData = SpatialConversionUtils.getShapeFileManifest(shpFile)
+
+        int featureIndex = 0
+        for (List<Pair<String, Object>> featureData : manifestData) {
+            // Use linked hash map to maintain key ordering
+            Map<String, Object> featureDataMap = new LinkedHashMap<String, Object>()
+
+            for (Pair<String, Object> fieldData : featureData) {
+                featureDataMap.put(fieldData.getLeft(), fieldData.getRight())
+            }
+
+            retMap.put(featureIndex, featureDataMap)
+
+            featureIndex++
+        }
+
+        return retMap
+    }
+}

grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy

@@ -42,6 +42,9 @@ class UrlMappings {
 
         "/ws/output/getOutputSpeciesUUID/"(controller: "output"){ action = [GET:"getOutputSpeciesUUID"] }
 
+        "/ws/shapefile" (controller: "spatial"){ action = [POST:"uploadShapeFile"] }
+        "/ws/shapefile/geojson/$shapeFileId/$featureId"(controller: "spatial"){ action = [GET:"getShapeFileFeatureGeoJson"] }
+
         "/ws/activitiesForProject/$id" {
             controller = 'activity'
             action = 'activitiesForProject'

src/integration-test/groovy/au/org/ala/ecodata/SpatialControllerIntegrationSpec.groovy

@@ -0,0 +1,74 @@
+package au.org.ala.ecodata
+
+import grails.testing.mixin.integration.Integration
+import grails.util.GrailsWebMockUtil
+import groovy.json.JsonSlurper
+import org.apache.http.HttpStatus
+import org.grails.plugins.testing.GrailsMockHttpServletRequest
+import org.grails.plugins.testing.GrailsMockHttpServletResponse
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.mock.web.MockMultipartFile
+import org.springframework.web.context.WebApplicationContext
+import spock.lang.Specification
+
+@Integration
+class SpatialControllerIntegrationSpec extends Specification {
+
+    @Autowired
+    SpatialController spatialController
+
+    @Autowired
+    WebApplicationContext ctx
+
+    def setup() {
+        setRequestResponse()
+    }
+
+    def cleanup() {
+    }
+
+    def setRequestResponse() {
+        GrailsMockHttpServletRequest grailsMockHttpServletRequest = new GrailsMockHttpServletRequest()
+        GrailsMockHttpServletResponse grailsMockHttpServletResponse = new GrailsMockHttpServletResponse()
+        GrailsWebMockUtil.bindMockWebRequest(ctx, grailsMockHttpServletRequest, grailsMockHttpServletResponse)
+    }
+
+    void "test uploadShapeFile with resource zip file"() {
+        given:
+        // Read the zip file from resources
+        def zipFileResourceStream = spatialController.class.getResourceAsStream("/projectExtent.zip")
+        byte[] zipFileBytes = zipFileResourceStream.bytes
+
+        // Mock the request
+        MockMultipartFile mockMultipartFile = new MockMultipartFile("file", "projectExtent.zip", "application/zip", zipFileBytes)
+        spatialController.request.addFile(mockMultipartFile)
+        spatialController.request.method = 'POST'
+
+        when:
+        // Call the method
+        spatialController.uploadShapeFile()
+
+        then:
+        // Verify the response
+        spatialController.response.status == HttpStatus.SC_OK
+        println spatialController.response.contentAsString
+        def responseContent = new JsonSlurper().parseText(spatialController.response.contentAsString)
+        responseContent.shp_id != null
+        responseContent["0"].siteId == "340cfe6a-f230-4bb9-a034-23e9bff125c7"
+        responseContent["0"].name == "Project area for Southern Tablelands Koala Habitat Restoration Project"
+
+        when:
+        setRequestResponse()
+        spatialController.request.method = 'GET'
+        spatialController.params.shapeFileId = responseContent.shp_id
+        spatialController.params.featureId = "0"
+        spatialController.getShapeFileFeatureGeoJson()
+
+        then:
+        spatialController.response.status == HttpStatus.SC_OK
+        println spatialController.response.contentAsString
+        def responseJSON = new JsonSlurper().parseText(spatialController.response.contentAsString)
+        responseJSON.geoJson != null
+        responseJSON.geoJson.type == "MultiPolygon"
+    }
+}

src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy

@@ -247,11 +247,15 @@ class GeometryUtils {
         new GeometryJSON().read(json)
     }
 
-    static Map geometryToGeoJsonMap(Geometry input) {
-        ByteArrayOutputStream byteOut = new ByteArrayOutputStream()
-        new GeometryJSON().write(input, new OutputStreamWriter(byteOut, 'UTF-8'))
+    static Map geometryToGeoJsonMap(Geometry input, int decimals = 4) {
+        String geoJson = geometryToGeoJsonString(input, decimals)
+        JSON.parse(geoJson)
+    }
 
-        JSON.parse(byteOut.toString('UTF-8'))
+    static String geometryToGeoJsonString(Geometry input, int decimals = 4) {
+        ByteArrayOutputStream byteOut = new ByteArrayOutputStream()
+        new GeometryJSON(decimals).write(input, new OutputStreamWriter(byteOut, 'UTF-8'))
+        byteOut.toString('UTF-8')
     }
 
     /**
@@ -261,13 +265,15 @@ class GeometryUtils {
      * @return
      */
     static Map simplify(Map geoJson, double tolerance) {
-
         Geometry input = geoJsonMapToGeometry(geoJson)
-
-        Geometry result = TopologyPreservingSimplifier.simplify(input, tolerance)
+        Geometry result = simplifyGeometry(input, tolerance)
         geometryToGeoJsonMap(result)
     }
 
+    static Geometry simplifyGeometry(Geometry input, double tolerance) {
+        TopologyPreservingSimplifier.simplify(input, tolerance)
+    }
+
 
     /**
      * Iterates through the supplied features and determines which features are neighbours using an

src/main/groovy/au/org/ala/ecodata/spatial/SpatialConversionUtils.groovy

@@ -0,0 +1,112 @@
+package au.org.ala.ecodata.spatial
+
+import com.google.common.io.Files
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.commons.io.IOUtils
+import org.apache.commons.lang3.tuple.Pair
+import org.geotools.data.FileDataStore
+import org.geotools.data.FileDataStoreFinder
+import org.geotools.data.simple.SimpleFeatureCollection
+import org.geotools.data.simple.SimpleFeatureIterator
+import org.geotools.data.simple.SimpleFeatureSource
+import org.opengis.feature.Property
+import org.opengis.feature.simple.SimpleFeature
+import org.opengis.feature.type.GeometryType
+
+import java.util.zip.ZipEntry
+import java.util.zip.ZipFile
+/**
+ * Utilities for converting spatial data between formats
+ *
+ * @author ChrisF
+ */
+@Slf4j
+@CompileStatic
+class SpatialConversionUtils {
+    static Pair<String, File> extractZippedShapeFile(File zippedShpFile) throws IOException {
+
+        File tempDir = Files.createTempDir()
+
+        // Unpack the zipped shape file into the temp directory
+        ZipFile zf = null
+        File shpFile = null
+        try {
+            zf = new ZipFile(zippedShpFile)
+
+            boolean shpPresent = false
+            boolean shxPresent = false
+            boolean dbfPresent = false
+
+            Enumeration<? extends ZipEntry> entries = zf.entries()
+
+            while (entries.hasMoreElements()) {
+                ZipEntry entry = entries.nextElement()
+                InputStream inStream = zf.getInputStream(entry)
+                File f = new File(tempDir, entry.getName())
+                if (!f.getName().startsWith(".")) {
+                    if (entry.isDirectory()) {
+                        f.mkdirs()
+                    } else {
+                        FileOutputStream outStream = new FileOutputStream(f)
+                        IOUtils.copy(inStream, outStream)
+
+                        if (entry.getName().endsWith(".shp")) {
+                            shpPresent = true
+                            shpFile = f
+                        } else if (entry.getName().endsWith(".shx") && !f.getName().startsWith("/")) {
+                            shxPresent = true
+                        } else if (entry.getName().endsWith(".dbf") && !f.getName().startsWith("/")) {
+                            dbfPresent = true
+                        }
+                    }
+                }
+            }
+
+            if (!shpPresent || !shxPresent || !dbfPresent) {
+                throw new IllegalArgumentException("Invalid archive. Must contain .shp, .shx and .dbf at a minimum.")
+            }
+        } catch (Exception e) {
+            log.error(e.getMessage(), e)
+        } finally {
+            if (zf != null) {
+                try {
+                    zf.close()
+                } catch (Exception e) {
+                    log.error(e.getMessage(), e)
+                }
+            }
+        }
+
+        if (shpFile == null) {
+            return null
+        } else {
+            return Pair.of(shpFile.getParentFile().getName(), shpFile)
+        }
+    }
+
+    static List<List<Pair<String, Object>>> getShapeFileManifest(File shpFile) throws IOException {
+        List<List<Pair<String, Object>>> manifestData = new ArrayList<List<Pair<String, Object>>>()
+
+        FileDataStore store = FileDataStoreFinder.getDataStore(shpFile)
+
+        SimpleFeatureSource featureSource = store.getFeatureSource(store.getTypeNames()[0])
+        SimpleFeatureCollection featureCollection = featureSource.getFeatures()
+        SimpleFeatureIterator it = featureCollection.features()
+
+        while (it.hasNext()) {
+            SimpleFeature feature = it.next()
+            List<Pair<String, Object>> pairList = new ArrayList<Pair<String, Object>>()
+            for (Property prop : feature.getProperties()) {
+                if (!(prop.getType() instanceof GeometryType)) {
+                    Pair<String, Object> pair = Pair.of(prop.getName().toString(), feature.getAttribute(prop.getName()))
+                    pairList.add(pair)
+                }
+            }
+            manifestData.add(pairList)
+        }
+
+        return manifestData
+    }
+}
+