Merge pull request #372 from UNC-Libraries/is-applicable-fedora

getDatastream to determine if enhancements applicable

Author: daines

services-worker/src/main/java/edu/unc/lib/dl/cdr/services/AbstractDatastreamEnhancementService.java

@@ -15,22 +15,25 @@
  */
 package edu.unc.lib.dl.cdr.services;
 
-import java.util.List;
-import java.util.Map;
+import static edu.unc.lib.dl.util.JMSMessageUtil.ServicesActions.APPLY_SERVICE;
+import static edu.unc.lib.dl.util.JMSMessageUtil.ServicesActions.APPLY_SERVICE_STACK;
+
+import java.util.regex.Pattern;
 
 import edu.unc.lib.dl.cdr.services.exception.EnhancementException;
-import edu.unc.lib.dl.cdr.services.model.EnhancementApplication;
 import edu.unc.lib.dl.cdr.services.model.EnhancementMessage;
 import edu.unc.lib.dl.cdr.services.model.FedoraEventMessage;
+import edu.unc.lib.dl.fedora.FedoraException;
 import edu.unc.lib.dl.fedora.PID;
 import edu.unc.lib.dl.util.ContentModelHelper;
+import edu.unc.lib.dl.util.ContentModelHelper.Datastream;
 import edu.unc.lib.dl.util.JMSMessageUtil;
+import edu.unc.lib.dl.util.JMSMessageUtil.FedoraActions;
 
 public abstract class AbstractDatastreamEnhancementService extends AbstractIrodsObjectEnhancementService {
 
-	protected String lastAppliedQuery;
-	protected String applicableNoDSQuery;
-	protected String applicableStaleDSQuery;
+	protected String derivativeDatastream;
+	protected Pattern mimetypePattern;
 
 	@Override
 	public boolean prefilterMessage(EnhancementMessage message) throws EnhancementException {
@@ -55,32 +58,54 @@ public boolean prefilterMessage(EnhancementMessage message) throws EnhancementEx
 
 		return ContentModelHelper.Datastream.DATA_FILE.equals(datastream);
 	}
-
-	@Override
-	public boolean isStale(PID pid) throws EnhancementException {
-		return false;
+	
+	protected boolean isDatastreamApplicable(PID pid) throws FedoraException {
+		edu.unc.lib.dl.fedora.types.Datastream dataDoc
+				= managementClient.getDatastream(pid, Datastream.DATA_FILE.getName());
+		
+		// Don't process if there is no original data
+		if (dataDoc == null) {
+			return false;
+		}
+		
+		// Filter out objects with non-applicable mimetypes
+		if (mimetypePattern != null && !mimetypePattern.matcher(dataDoc.getMIMEType()).matches()){
+			return false;
+		}
+		
+		edu.unc.lib.dl.fedora.types.Datastream derivDoc
+				= managementClient.getDatastream(pid, derivativeDatastream);
+		
+		// No derivative present
+		if (derivDoc == null) {
+			return true;
+		}
+		
+		// Derivative is older than the original data, need to reperform the enhancement
+		// Dates are in iso8601/UTC format, so lexographic string comparison is sufficient
+		return dataDoc.getCreateDate().compareTo(derivDoc.getCreateDate()) > 0;
 	}
-
-	@SuppressWarnings("rawtypes")
+	
 	@Override
-	public EnhancementApplication getLastApplied(PID pid) throws EnhancementException {
-		// replace model URI and PID tokens
-		String query = String.format(this.lastAppliedQuery, this.getTripleStoreQueryService().getResourceIndexModelUri(),
-				pid.getURI());
-
-		@SuppressWarnings("unchecked")
-		List<Map> bindings = (List<Map>) ((Map) this.getTripleStoreQueryService().sendSPARQL(query).get("results"))
-				.get("bindings");
-		if (bindings.size() == 0)
-			return null;
-
-		EnhancementApplication lastApplied = new EnhancementApplication();
-		String lastModified = (String) ((Map) bindings.get(0).get("lastModified")).get("value");
-		lastApplied.setLastAppliedFromISO8601(lastModified);
-		lastApplied.setPid(pid);
-		lastApplied.setEnhancementClass(this.getClass());
-
-		return lastApplied;
+	public boolean isApplicable(EnhancementMessage message) throws EnhancementException {
+		String action = message.getQualifiedAction();
+		
+		// Only need to check further if this is an ingest message or the DATA_FILE was changed
+		if (!(FedoraActions.INGEST.equals(action)
+				|| FedoraActions.MODIFY_DATASTREAM_BY_REFERENCE.equals(action)
+				|| FedoraActions.ADD_DATASTREAM.equals(action)
+				|| FedoraActions.MODIFY_DATASTREAM_BY_VALUE.equals(action)
+				|| APPLY_SERVICE_STACK.equals(action)
+				|| (APPLY_SERVICE.equals(action) && getClass().getName()
+						.equals(message.getServiceName())))) {
+			return false;
+		}
+		
+		try {
+			return isDatastreamApplicable(message.getPid());
+		} catch (FedoraException e) {
+			throw new EnhancementException("Failed to check if enhancement was applicable for " + message.getPid(), e);
+		}
 	}
 
 }

services-worker/src/main/java/edu/unc/lib/dl/cdr/services/AbstractFedoraEnhancementService.java

@@ -15,24 +15,15 @@
  */
 package edu.unc.lib.dl.cdr.services;
 
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.io.IOUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.ApplicationContextAware;
 
 import edu.unc.lib.dl.cdr.services.exception.EnhancementException;
 import edu.unc.lib.dl.cdr.services.model.EnhancementMessage;
-import edu.unc.lib.dl.cdr.services.model.LabeledPID;
 import edu.unc.lib.dl.cdr.services.processing.MessageDirector;
 import edu.unc.lib.dl.fedora.ManagementClient;
-import edu.unc.lib.dl.fedora.PID;
 import edu.unc.lib.dl.util.JMSMessageUtil;
 import edu.unc.lib.dl.util.TripleStoreQueryService;
 
@@ -42,9 +33,6 @@ public abstract class AbstractFedoraEnhancementService implements ObjectEnhancem
 	protected TripleStoreQueryService tripleStoreQueryService = null;
 	protected ManagementClient managementClient = null;
 	protected boolean active = false;
-	protected List<String> findCandidatesQueries;
-	protected String findStaleCandidatesQuery;
-	protected List<String> isApplicableQueries;
 
 	private ApplicationContext applicationContext;
 
@@ -94,153 +82,4 @@ public ManagementClient getManagementClient() {
 	public void setManagementClient(ManagementClient managementClient) {
 		this.managementClient = managementClient;
 	}
-
-	@SuppressWarnings("unchecked")
-	@Override
-	public List<PID> findStaleCandidateObjects(int maxResults, String priorToDate) throws EnhancementException {
-		return (List<PID>) this.findCandidateObjects(maxResults, 0, priorToDate, false);
-	}
-
-	@SuppressWarnings("unchecked")
-	@Override
-	public List<PID> findCandidateObjects(int maxResults, int offset) throws EnhancementException {
-		return (List<PID>) this.findCandidateObjects(maxResults, offset, null, false);
-	}
-
-	@Override
-	public int countCandidateObjects() throws EnhancementException {
-		return (Integer) this.findCandidateObjects(-1, 0, null, true);
-	}
-
-	public Object findCandidateObjects(int maxResults, int offset, String priorToDate, boolean countQuery)
-			throws EnhancementException {
-		if (priorToDate == null) {
-			return this.executeCandidateQueries(this.findCandidatesQueries, countQuery, maxResults, offset);
-		} else {
-			String limitClause = "";
-			if (maxResults >= 0 && !countQuery)
-				limitClause = "LIMIT " + maxResults;
-			return this.executeCandidateQuery(String.format(this.findStaleCandidatesQuery, this.getTripleStoreQueryService()
-					.getResourceIndexModelUri(), priorToDate, limitClause) + limitClause, countQuery);
-		}
-	}
-
-	@SuppressWarnings("unchecked")
-	protected Object executeCandidateQueries(List<String> queries, boolean count, int limit, int offset) {
-		int resultCount = 0;
-		List<PID> results = new MaxSizeList<PID>(limit);
-		for (String queryOriginal: queries) {
-			String query = queryOriginal;
-			if (!count)
-				query += " LIMIT " + limit;
-			query += " OFFSET " + offset;
-			Object result = this.executeCandidateQuery(query, count);
-			if (count) {
-				resultCount += ((Integer)result).intValue();
-			} else {
-				List<PID> queryResults = (List<PID>) result;
-				results.addAll(queryResults);
-				if (results.size() >= limit)
-					return results;
-			}
-		}
-
-		if (count)
-			return resultCount;
-		return results;
-	}
-
-	@SuppressWarnings({ "unchecked", "rawtypes" })
-	protected Object executeCandidateQuery(String query, boolean countQuery) {
-		String format = "json";//((countQuery) ? "count/json" : "json");
-		Map results = this.getTripleStoreQueryService().sendSPARQL(query, format);
-		List<Map> bindings = (List<Map>) ((Map) results.get("results")).get("bindings");
-
-		if (LOG.isDebugEnabled())
-			LOG.debug(results.toString());
-		if (countQuery) {
-			// TODO Mulgara doesn't support count queries in SPARQL, will need to redo for other triple stores
-			return bindings.size();
-			/*Map binding = bindings.get(0);
-			int count = Integer.parseInt((String) ((Map) binding.get("count")).get("value"));
-			return count;*/
-		} else {
-			List<PID> result = new ArrayList<PID>();
-			for (Map binding : bindings) {
-				String pidURI = (String) ((Map) binding.get("pid")).get("value");
-				String label = (String) ((Map) binding.get("label")).get("value");
-				result.add(new LabeledPID(pidURI, label));
-			}
-
-			return result;
-		}
-	}
-
-	@Override
-	public boolean isApplicable(EnhancementMessage message) throws EnhancementException {
-		// Automatically isApplicable if the message is specifically asking for this service.
-		String action = message.getQualifiedAction();
-		if ((JMSMessageUtil.ServicesActions.APPLY_SERVICE_STACK.equals(action) || JMSMessageUtil.ServicesActions.APPLY_SERVICE.equals(action))
-				&& this.getClass().getName().equals(message.getServiceName()))
-			return true;
-
-		return askQueries(this.isApplicableQueries, message);
-	}
-
-	protected boolean askQueries(List<String> queries, EnhancementMessage message) {
-		for (String query: queries)
-			if (askQuery(query, message))
-				return true;
-		return false;
-	}
-
-	@SuppressWarnings("unchecked")
-	protected boolean askQuery(String query, EnhancementMessage message) {
-		query = String.format(query,
-				this.tripleStoreQueryService.getResourceIndexModelUri(), message.getPid().getURI());
-		Map<String, Object> result = this.getTripleStoreQueryService().sendSPARQL(query);
-		return (Boolean.TRUE.equals(result.get("boolean")));
-	}
-
-	/**
-	 * @param filePath
-	 *           name of file to open. The file can reside anywhere in the classpath
-	 */
-	protected String readFileAsString(String filePath) throws java.io.IOException {
-		return IOUtils.toString(this.getClass().getResourceAsStream(filePath), "UTF-8");
-	}
-
-	protected class MaxSizeList<E> extends ArrayList<E> {
-		private static final long serialVersionUID = 1L;
-		private int limit = 10;
-
-		public MaxSizeList(int limit) {
-			this.limit = limit;
-		}
-
-		@Override
-		public boolean add(E element) {
-			if (this.size() >= limit) return true;
-			return super.add(element);
-		}
-
-		@Override
-		public void add(int index, E element) {
-			if (this.size() >= limit) return;
-			super.add(index, element);
-		}
-
-		@Override
-		public boolean addAll(Collection<? extends E> c) {
-			if (c.size() + this.size() < limit)
-				return super.addAll(c);
-			Iterator<? extends E> it = c.iterator();
-			while (it.hasNext()) {
-				this.add(it.next());
-				if (this.size() == limit)
-					return true;
-			}
-			return false;
-		}
-	}
 }

services-worker/src/main/java/edu/unc/lib/dl/cdr/services/ObjectEnhancementService.java

@@ -15,47 +15,16 @@
  */
 package edu.unc.lib.dl.cdr.services;
 
-import java.util.List;
-
 import org.jdom2.Element;
 
 import edu.unc.lib.dl.cdr.services.exception.EnhancementException;
-import edu.unc.lib.dl.cdr.services.model.EnhancementApplication;
 import edu.unc.lib.dl.cdr.services.model.EnhancementMessage;
-import edu.unc.lib.dl.fedora.PID;
 
 /**
  * @author Gregory Jansen
  * 
  */
 public interface ObjectEnhancementService {
-	/**
-	 * Returns a list of candidate objects to which this service may apply. This method is allowed to return some false
-	 * positives. However, if the maxResults is equal or greater than the number of objects in the repository, then it
-	 * must include all applicable objects.
-	 * 
-	 * @return
-	 */
-	public List<PID> findCandidateObjects(int maxResults, int offset) throws EnhancementException;
-
-	/**
-	 * Returns the total number of objects that would be returned by findCandidateObjects.
-	 * 
-	 * @return
-	 * @throws EnhancementException
-	 */
-	public int countCandidateObjects() throws EnhancementException;
-
-	/**
-	 * Returns a list of candidate objects to which this service may apply to, including objects that it has applied to
-	 * in the past but which are now stale.
-	 * 
-	 * @param maxResults
-	 * @param priorToDate
-	 * @return
-	 * @throws EnhancementException
-	 */
-	public List<PID> findStaleCandidateObjects(int maxResults, String priorToDate) throws EnhancementException;
 
 	/**
 	 * Creates a task for running this service on the object in question.
@@ -82,24 +51,6 @@ public interface ObjectEnhancementService {
 	 */
 	public boolean prefilterMessage(EnhancementMessage pid) throws EnhancementException;
 
-	/**
-	 * Checks to see if the enhancement should be re-applied. Generally a comparison of timestamps or software agent
-	 * strings. Returns true if this enhancement has never been run or will provide greater enhancement by running again.
-	 * 
-	 * @param pid
-	 * @return true if the object is stale w/respect to this enhancement
-	 */
-	public boolean isStale(PID pid) throws EnhancementException;
-
-	/**
-	 * Determines the last date on which this service was applied to the object represented by pid.
-	 * 
-	 * @param pid
-	 * @return the most recent date this service was applied to object pid, or null if it has never been applied.
-	 * @throws EnhancementException
-	 */
-	public EnhancementApplication getLastApplied(PID pid) throws EnhancementException;
-
 	/**
 	 * @return true if this service is currently active
 	 */