Reordered datastreams added to new FOXML documents for ingest so that the DATA_FILE will come after other uploaded datastreams. Added more breakpoints in ingest jobs to allow pausing to interrupt

Author: bbpennel

deposit/src/main/java/edu/unc/lib/deposit/fcrepo3/IngestDeposit.java

@@ -29,7 +29,6 @@
 
 import edu.unc.lib.deposit.work.AbstractDepositJob;
 import edu.unc.lib.deposit.work.DepositGraphUtils;
-import edu.unc.lib.deposit.work.JobInterruptedException;
 import edu.unc.lib.dl.acl.util.AccessGroupSet;
 import edu.unc.lib.dl.acl.util.GroupsThreadStore;
 import edu.unc.lib.dl.fedora.AccessClient;
@@ -50,7 +49,6 @@
 import edu.unc.lib.dl.util.PremisEventLogger;
 import edu.unc.lib.dl.util.PremisEventLogger.Type;
 import edu.unc.lib.dl.util.RedisWorkerConstants.DepositField;
-import edu.unc.lib.dl.util.RedisWorkerConstants.DepositState;
 import edu.unc.lib.dl.xml.FOXMLJDOMUtil;
 
 /**
@@ -247,11 +245,7 @@ public void runJob() {
 					statusFactory.incrIngestedObjects(getDepositUUID(), 1);
 
 					// Verify that the job has not been interrupted before continuing
-					DepositState state = statusFactory.getState(getDepositUUID());
-					if (!DepositState.running.equals(state)) {
-						throw new JobInterruptedException("State for job " + getDepositUUID()
-								+ " is no longer running, interrupting");
-					}
+					verifyRunning();
 				}
 			} catch (DepositException e) {
 				failJob(e, Type.INGESTION, "Ingest of object {0} failed", ingestPid);
@@ -261,6 +255,7 @@ public void runJob() {
 			// listen to Fedora JMS to see when all objects are ingested
 			try {
 				while (ingestsAwaitingConfirmation.size() > 0) {
+					verifyRunning();
 					Thread.sleep(COMPLETE_CHECK_DELAY);
 				}
 

deposit/src/main/java/edu/unc/lib/deposit/fcrepo3/MakeFOXML.java

@@ -159,29 +159,6 @@ public void runJob() {
 
 			// TODO translate default web object, already copied, check it
 
-			// add DATA_FILE
-			Property fileLocation = dprop(m, DepositRelationship.stagingLocation);
-			if(o.hasProperty(fileLocation)) {
-				addDatastream(m, o, foxml);
-
-				// add sourceData and defaultWebData properties
-				Resource dataFileResource = m.createResource(new DatastreamPID(p.getPid() + "/DATA_FILE")
-						.getDatastreamURI());
-				relsExt.add(o, cdrprop(m, CDRProperty.sourceData), dataFileResource);
-				relsExt.add(o, cdrprop(m, CDRProperty.defaultWebData), dataFileResource);
-			}
-
-			// add any other datastreams
-			Property hasDatastream = dprop(m, DepositRelationship.hasDatastream);
-			if (o.hasProperty(hasDatastream)) {
-				StmtIterator dsIt = o.listProperties(hasDatastream);
-
-				while (dsIt.hasNext()) {
-					Statement dsStmt = dsIt.next();
-					addDatastream(m, dsStmt.getResource(), foxml);
-				}
-			}
-
 			// Add in invalid term triples
 			Property invalidTermProp = cdrprop(m, CDRProperty.invalidTerm);
 			if (o.hasProperty(invalidTermProp)) {
@@ -198,9 +175,6 @@ public void runJob() {
 				relsExt.add(o, publishedProperty, "no");
 			}
 
-			// add RELS-EXT
-			saveRELSEXTtoFOXMl(relsExt, foxml);
-
 			// add MD_EVENTS
 			addEventsDS(p, foxml);
 
@@ -233,6 +207,32 @@ public void runJob() {
 				foxml.getRootElement().addContent(el);
 			}
 
+			// Adding other datastreams before DATA_FILE to avoid temporary files being deleted before ingesting
+			Property hasDatastream = dprop(m, DepositRelationship.hasDatastream);
+			if (o.hasProperty(hasDatastream)) {
+				StmtIterator dsIt = o.listProperties(hasDatastream);
+
+				while (dsIt.hasNext()) {
+					Statement dsStmt = dsIt.next();
+					addDatastream(m, dsStmt.getResource(), foxml);
+				}
+			}
+
+			// add DATA_FILE
+			Property fileLocation = dprop(m, DepositRelationship.stagingLocation);
+			if (o.hasProperty(fileLocation)) {
+				addDatastream(m, o, foxml);
+
+				// add sourceData and defaultWebData properties
+				Resource dataFileResource = m.createResource(new DatastreamPID(p.getPid() + "/DATA_FILE")
+						.getDatastreamURI());
+				relsExt.add(o, cdrprop(m, CDRProperty.sourceData), dataFileResource);
+				relsExt.add(o, cdrprop(m, CDRProperty.defaultWebData), dataFileResource);
+			}
+
+			// add RELS-EXT
+			saveRELSEXTtoFOXMl(relsExt, foxml);
+
 			writeFOXML(p, foxml);
 			addClicks(1);
 		}
@@ -303,12 +303,12 @@ private void writeFOXML(PID p, Document foxml) {
 	private void addEventsDS(PID p, Document foxml) {
 		// Add locator datastream for events, referencing the events file identified by p
 		File events = getEventsFile(p);
-		
+
 		// Use a path relative to the deposit directory
 		Path absolute = Paths.get(events.getAbsolutePath());
 		Path base = Paths.get(getDepositDirectory().getAbsolutePath());
 		Path relative = base.relativize(absolute);
-		
+
 		Element el = FOXMLJDOMUtil.makeLocatorDatastream(Datastream.MD_EVENTS.getName(), "M", relative.toString(),
 				"text/xml", "URL", Datastream.MD_EVENTS.getLabel(), false, null);
 		foxml.getRootElement().addContent(el);

deposit/src/main/java/edu/unc/lib/deposit/validate/VirusScanJob.java

@@ -92,6 +92,8 @@ public void runJob() {
 		int scannedObjects = 0;
 
 		for (Entry<PID, String> href : hrefs.entrySet()) {
+			verifyRunning();
+
 			URI manifestURI;
 			URI storageURI = null;
 			try {

deposit/src/main/java/edu/unc/lib/deposit/work/AbstractDepositJob.java

@@ -33,6 +33,7 @@
 import edu.unc.lib.dl.util.JobStatusFactory;
 import edu.unc.lib.dl.util.PremisEventLogger;
 import edu.unc.lib.dl.util.PremisEventLogger.Type;
+import edu.unc.lib.dl.util.RedisWorkerConstants.DepositState;
 import edu.unc.lib.dl.xml.JDOMNamespaceUtil;
 
 /**
@@ -225,6 +226,15 @@ public void failJob(Throwable throwable, Type type, String messageformat,
 		appendDepositEvent(getDepositPID(), event);
 		throw new JobFailedException(message, throwable);
 	}
+	
+	protected void verifyRunning() {
+		DepositState state = getDepositStatusFactory().getState(getDepositUUID());
+		
+		if (!DepositState.running.equals(state)) {
+			throw new JobInterruptedException("State for job " + getDepositUUID()
+					+ " is no longer running, interrupting");
+		}
+	}
 
 	/**
 	 * Returns the PREMIS events file for the given PID