FITS extract to be performed using commandline utility instead of webservice under certain conditions, currently based on characters in file path and size of the file (#1284)

Author: bbpennel

deposit/src/main/java/edu/unc/lib/deposit/validate/ExtractTechnicalMetadataJob.java

@@ -42,17 +42,12 @@
 
 import javax.annotation.PostConstruct;
 
-import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.io.IOUtils;
-import org.apache.http.HttpEntity;
 import org.apache.http.HttpStatus;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpUriRequest;
 import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.entity.mime.MultipartEntityBuilder;
-import org.apache.http.entity.mime.content.InputStreamBody;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.jena.rdf.model.Model;
 import org.apache.jena.rdf.model.Resource;
@@ -96,8 +91,9 @@ public class ExtractTechnicalMetadataJob extends AbstractConcurrentDepositJob {
     private String baseFitsUri;
     // URI to the examine servlet in the FITS application
     private URI fitsExamineUri;
-
-    private boolean processFilesLocally;
+    private String fitsHomePath;
+    private Path fitsCommandPath;
+    private int maxFileSizeForWebService;
 
     private Model model;
 
@@ -109,6 +105,7 @@ public ExtractTechnicalMetadataJob(String uuid, String depositUUID) {
     public void initJob() {
         init();
         fitsExamineUri = URI.create(URIUtil.join(baseFitsUri, FITS_EXAMINE_PATH));
+        fitsCommandPath = Paths.get(fitsHomePath, "fits.sh");
     }
 
     @Override
@@ -318,7 +315,6 @@ private void overrideDepositMimetype(String fitsExtractMimetype) {
      * @return
      */
     private Document getFitsDocument(PID objPid, String stagedUriString) {
-        HttpUriRequest request;
         URI stagedUri = URI.create(stagedUriString);
         Path stagedPath;
         if (!stagedUri.isAbsolute()) {
@@ -328,45 +324,41 @@ private Document getFitsDocument(PID objPid, String stagedUriString) {
             stagedPath = Paths.get(stagedUri);
         }
 
-        Path sanitizedPath = null;
-        // FITS cannot currently handle file paths that contain unicode characters, so need to upload
-        if (processFilesLocally) {
-            // Files are available locally to FITS, so just pass along path
-            URI fitsUri = null;
-            try {
-                sanitizedPath = sanitizePath(stagedPath);
-                URIBuilder builder = new URIBuilder(fitsExamineUri);
-                builder.addParameter("file", (sanitizedPath == null ? stagedPath : sanitizedPath).toString());
-                fitsUri = builder.build();
-
-                log.debug("Requesting FITS document for {} using local file via URI {}", objPid, fitsUri);
-            } catch (URISyntaxException e) {
-                failJob(e, "Failed to construct FITs report uri for {0}", objPid);
-            } catch (IOException e) {
-                failJob(e, "Failed to create symbolic link to file for extract {0} for {1}", stagedPath, objPid);
-            }
-
-            request = new HttpGet(fitsUri);
+        if (shouldProcessWithWebService(stagedPath)) {
+            return extractUsingWebService(objPid, stagedPath);
         } else {
-            // Files are to be processed remotely, so upload them via a post request
-            HttpEntity entity;
-            try {
-                entity = MultipartEntityBuilder.create()
-                        .addPart("datafile", new InputStreamBody(Files.newInputStream(stagedPath),
-                                stagedPath.getFileName().toString()))
-                        .build();
-            } catch (IOException e) {
-                failJob(e, "Unable to read file {0}", stagedPath);
-                return null;
-            }
+            return extractUsingCLI(objPid, stagedPath);
+        }
+    }
 
-            HttpPost postRequest = new HttpPost(fitsExamineUri);
-            postRequest.setEntity(entity);
-            request = postRequest;
+    private boolean shouldProcessWithWebService(Path path) {
+        // FITS cannot currently handle file paths that contain unicode characters
+        if (!CharMatcher.ascii().matchesAllOf(path.toString())) {
+            return false;
+        }
+        try {
+            return Files.size(path) <= maxFileSizeForWebService;
+        } catch (IOException e) {
+            failJob(e, "Unable to inspect file");
+        }
+        return false;
+    }
 
-            log.debug("Requesting FITS document for {} using remote file from {}", objPid, stagedPath);
+    private Document extractUsingWebService(PID objPid, Path stagedPath) {
+        // Files are available locally to FITS, so just pass along path
+        URI fitsUri = null;
+        try {
+            URIBuilder builder = new URIBuilder(fitsExamineUri);
+            builder.addParameter("file", stagedPath.toString());
+            fitsUri = builder.build();
+
+            log.debug("Requesting FITS document for {} using local file via URI {}", objPid, fitsUri);
+        } catch (URISyntaxException e) {
+            failJob(e, "Failed to construct FITs report uri for {0}", objPid);
         }
 
+        HttpUriRequest request = new HttpGet(fitsUri);
+
         try (CloseableHttpResponse resp = httpClient.execute(request)) {
             // Write the report response to file
             InputStream respBodyStream = resp.getEntity().getContent();
@@ -380,32 +372,22 @@ private Document getFitsDocument(PID objPid, String stagedUriString) {
         } catch (IOException | JDOMException e) {
             failJob(e, "Failed to stream report for {0} from server to report document",
                     objPid);
-        } finally {
-            // Cleanup symbolic link if one was created
-            if (sanitizedPath != null) {
-                try {
-                    Files.deleteIfExists(sanitizedPath);
-                } catch (IOException e) {
-                    log.warn("Failed to cleanup sanitized path {}: {}", sanitizedPath, e.getMessage());
-                }
-            }
         }
         return null;
     }
 
-    private Path sanitizePath(Path path) throws IOException {
-        if (CharMatcher.ascii().matchesAllOf(path.toString())) {
-            return null;
-        }
-        String ext = FilenameUtils.getExtension(path.getFileName().toString());
-        if (!ext.equals("")) {
-            ext = "." + ext;
+    private Document extractUsingCLI(PID objPid, Path stagedPath) {
+        try {
+            Process process = Runtime.getRuntime().exec(fitsCommandPath + " -i " + stagedPath);
+            if (process.waitFor() != 0) {
+                failJob(null, "Failed to generate report for {0}, script returned {1} with output:\n{2}",
+                        objPid, process.exitValue(), IOUtils.toString(process.getInputStream(), UTF_8));
+            }
+            return createSAXBuilder().build(process.getInputStream());
+        } catch (IOException | JDOMException | InterruptedException e) {
+            failJob(e, "Failed to generate report for {0}", objPid);
         }
-        // Get a temp path for the symbolic link to be created at, using the same extension as the original
-        Path linkPath = Files.createTempFile("extract", ext);
-        Files.delete(linkPath);
-        Files.createSymbolicLink(linkPath, path);
-        return linkPath;
+        return null;
     }
 
     /**
@@ -577,7 +559,11 @@ public void setBaseFitsUri(String baseFitsUri) {
         this.baseFitsUri = baseFitsUri;
     }
 
-    public void setProcessFilesLocally(boolean processFilesLocally) {
-        this.processFilesLocally = processFilesLocally;
+    public void setFitsHomePath(String fitsHomePath) {
+        this.fitsHomePath = fitsHomePath;
+    }
+
+    public void setMaxFileSizeForWebService(int maxFileSizeForWebService) {
+        this.maxFileSizeForWebService = maxFileSizeForWebService;
     }
 }

deposit/src/main/webapp/WEB-INF/deposit-jobs-context.xml

@@ -265,7 +265,8 @@
         scope="prototype">
         <property name="baseFitsUri" value="${fits.baseUri}" />
         <property name="httpClient" ref="fitsPooledHttpClient" />
-        <property name="processFilesLocally" value="${fits.processFilesLocally:true}" />
+        <property name="maxFileSizeForWebService" value="${fits.maxFileSizeForWebService:128000000}" />
+        <property name="fitsHomePath" value="${fits.homePath}" />
         <property name="executorService" ref="extractTechnicalMetadataExecutor" />
         <property name="flushRate" value="${job.extractTechnicalMetadata.flushRate:2000}" />
         <property name="maxQueuedJobs" value="${job.extractTechnicalMetadata.maxQueuedJobs:6}" />