Merge pull request #350 from UNC-Libraries/xml-import-2

Bulk Metadata Importing

Author: daines

fcrepo-clients/src/main/java/edu/unc/lib/dl/util/VocabularyHelperManager.java

@@ -15,7 +15,9 @@
  */
 package edu.unc.lib.dl.util;
 
+import static edu.unc.lib.dl.util.ContentModelHelper.CDRProperty.invalidTerm;
 import static edu.unc.lib.dl.util.ContentModelHelper.CDRProperty.replaceInvalidTerms;
+import static edu.unc.lib.dl.util.ContentModelHelper.Datastream.RELS_EXT;
 import static edu.unc.lib.dl.util.ContentModelHelper.Model.COLLECTION;
 
 import java.net.URI;
@@ -37,10 +39,14 @@
 import org.springframework.beans.factory.annotation.Autowired;
 
 import edu.unc.lib.dl.fedora.AccessClient;
+import edu.unc.lib.dl.fedora.DatastreamDocument;
 import edu.unc.lib.dl.fedora.FedoraException;
 import edu.unc.lib.dl.fedora.ManagementClient;
+import edu.unc.lib.dl.fedora.OptimisticLockException;
 import edu.unc.lib.dl.fedora.PID;
 import edu.unc.lib.dl.util.ContentModelHelper.CDRProperty;
+import edu.unc.lib.dl.util.ContentModelHelper.Datastream;
+import edu.unc.lib.dl.xml.JDOMNamespaceUtil;
 import edu.unc.lib.dl.xml.VocabularyHelper;
 
 /**
@@ -196,57 +202,45 @@ public void updateInvalidTermsRelations(PID pid, Element docElement) throws Fedo
 		if (helpers == null)
 			return;
 
-		String invalidTermPred = CDRProperty.invalidTerm.toString();
-		List<String> allExistingTerms = queryService.fetchBySubjectAndPredicate(pid, invalidTermPred);
-
-		// Decompose triple values and group terms by vocabulary prefix
-		Map<String, List<String>> termMap = new HashMap<>();
-		for (String term : allExistingTerms) {
-			String parts[] = term.split("\\|", 2);
-
-			List<String> terms = termMap.get(parts[0]);
-			if (terms == null) {
-				terms = new ArrayList<>();
-				termMap.put(parts[0], terms);
-			}
-
-			terms.add(term);
-		}
+		DatastreamDocument relsDs = managementClient.getXMLDatastreamIfExists(pid, Datastream.RELS_EXT.getName());
+		
+		Element descEl = relsDs.getDocument().getRootElement().getChild("Description", JDOMNamespaceUtil.RDF_NS);
+		
+		// Remove all existing invalid term predicates
+		boolean termsChanged =
+				descEl.removeChildren(invalidTerm.getPredicate(), invalidTerm.getNamespace());
 
 		for (VocabularyHelper helper : helpers) {
-			List<String> existingTerms = termMap.get(helper.getInvalidTermPrefix());
-
 			Set<String> invalidTerms;
 			try {
 				invalidTerms = helper.getInvalidTermsWithPrefix(docElement);
+				
+				if (invalidTerms != null && invalidTerms.size() > 0) {
+					termsChanged = true;
+					
+					for (String term : invalidTerms) {
+						Element invTermEl = new Element(invalidTerm.getPredicate(), invalidTerm.getNamespace());
+						invTermEl.setText(term);
+						descEl.addContent(invTermEl);
+					}
+				}
 			} catch (JDOMException e) {
 				log.error("Failed to extract invalid terms from {}", pid.getPid(), e);
 				continue;
 			}
-
-			if (existingTerms != null && invalidTerms.size() == existingTerms.size()
-					&& invalidTerms.containsAll(existingTerms)) {
-				continue;
-			}
-
-			if (existingTerms != null) {
-				// Remove any terms which are no longer present
-				List<String> removeTerms = new ArrayList<String>(existingTerms);
-				removeTerms.removeAll(invalidTerms);
-
-				for (String term : removeTerms) {
-					managementClient.purgeLiteralStatement(pid, invalidTermPred, term, null);
-				}
-
-				// Calculate the set of newly invalid terms which need to be added
-				invalidTerms.removeAll(existingTerms);
-			}
-
-			if (invalidTerms.size() > 0) {
-				for (String term : invalidTerms) {
-					managementClient.addLiteralStatement(pid, invalidTermPred, term, null);
+		}
+		
+		// If any terms changed, then update RELS-EXT with optimistic locking
+		if (termsChanged) {
+			do {
+				try {
+					managementClient.modifyDatastream(pid, RELS_EXT.getName(), "Setting invalid vocabulary terms",
+							relsDs.getLastModified(), relsDs.getDocument());
+					return;
+				} catch (OptimisticLockException e) {
+					log.debug("Unable to update RELS-EXT for {}, retrying", pid);
 				}
-			}
+			} while (true);
 		}
 	}
 

fcrepo-clients/src/test/java/edu/unc/lib/dl/util/VocabularyHelperManagerTest.java

@@ -17,35 +17,41 @@
 
 import static edu.unc.lib.dl.test.TestHelpers.setField;
 import static edu.unc.lib.dl.util.ContentModelHelper.CDRProperty.indexValidTerms;
+import static edu.unc.lib.dl.util.ContentModelHelper.CDRProperty.invalidTerm;
 import static edu.unc.lib.dl.util.ContentModelHelper.CDRProperty.replaceInvalidTerms;
 import static edu.unc.lib.dl.util.ContentModelHelper.CDRProperty.warnInvalidTerms;
 import static edu.unc.lib.dl.util.ContentModelHelper.Datastream.DATA_FILE;
+import static edu.unc.lib.dl.util.ContentModelHelper.Datastream.RELS_EXT;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyString;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 import static org.mockito.MockitoAnnotations.initMocks;
 
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import org.jdom2.Document;
 import org.jdom2.Element;
 import org.junit.Before;
 import org.junit.Test;
+import org.mockito.ArgumentCaptor;
 import org.mockito.Mock;
 
 import edu.unc.lib.dl.fedora.AccessClient;
+import edu.unc.lib.dl.fedora.DatastreamDocument;
 import edu.unc.lib.dl.fedora.ManagementClient;
 import edu.unc.lib.dl.fedora.PID;
 import edu.unc.lib.dl.fedora.types.MIMETypedStream;
+import edu.unc.lib.dl.xml.JDOMNamespaceUtil;
 import edu.unc.lib.dl.xml.VocabularyHelper;
 
 /**
@@ -151,14 +157,26 @@ public void updateInvalidTermsTest() throws Exception {
 		helper.setInvalidTerms(new HashSet<>(Arrays.asList("term", "term2")));
 		helper.setPrefix(VOCAB_TYPE);
 
-		when(queryService.fetchBySubjectAndPredicate(any(PID.class), anyString())).thenReturn(
-				Arrays.asList(VOCAB_TYPE + "|term"));
+		Document relsDoc = new Document()
+			.addContent(new Element("RDF", JDOMNamespaceUtil.RDF_NS)
+			.addContent(new Element("Description", JDOMNamespaceUtil.RDF_NS)
+			.addContent(new Element(invalidTerm.getPredicate(), invalidTerm.getNamespace()))
+			.setText(VOCAB_TYPE + "|term")));
+		
+		when(managementClient.getXMLDatastreamIfExists(any(PID.class), eq(RELS_EXT.getName())))
+			.thenReturn(new DatastreamDocument(relsDoc, "2015-07-29"));
 
 		Element doc = mock(Element.class);
 		manager.updateInvalidTermsRelations(new PID(ITEM_PID), doc);
 
-		verify(managementClient, never()).purgeLiteralStatement(any(PID.class), anyString(), anyString(), anyString());
-
-		verify(managementClient).addLiteralStatement(any(PID.class), anyString(), anyString(), anyString());
+		ArgumentCaptor<Document> captor = ArgumentCaptor.forClass(Document.class);
+		verify(managementClient).modifyDatastream(any(PID.class), eq(RELS_EXT.getName()),
+				anyString(), anyString(), captor.capture());
+		
+		Document modified = captor.getValue();
+		List<Element> invTerms = modified.getRootElement().getChild("Description", JDOMNamespaceUtil.RDF_NS)
+			.getChildren(invalidTerm.getPredicate(), invalidTerm.getNamespace());
+		
+		assertEquals("Incorrect number of invalid terms after update", 2, invTerms.size());
 	}
 }

metadata/src/main/java/edu/unc/lib/dl/util/RedisWorkerConstants.java

@@ -8,6 +8,9 @@ public class RedisWorkerConstants {
 	public static final String INGESTS_UPLOADED_PREFIX = "ingests-uploaded:";
 	public static final String DEPOSIT_TO_JOBS_PREFIX = "deposit-to-jobs:";
 	public static final String JOB_STATUS_PREFIX = "job-status:";
+	public static final String BULK_UPDATE_PREFIX = "bulk-update:";
+	public static final String BULK_RESUME_PREFIX = "bulk-resume:";
+	public static final String BULK_UPDATE_QUEUE = "bulk-md-update";
 
 	public static enum DepositField {
 		uuid, state, actionRequest, contactName, depositorName, intSenderIdentifier, intSenderDescription,

persistence/pom.xml

@@ -48,14 +48,17 @@
 			<version>1.3.4</version>
 		</dependency>
 		<dependency>
-			<groupId>commons-logging</groupId>
-			<artifactId>commons-logging</artifactId>
-			<version>1.1.1</version>
+			<groupId>org.slf4j</groupId>
+			<artifactId>slf4j-api</artifactId>
+			<version>1.6.1</version>
+			<type>jar</type>
+			<scope>compile</scope>
 		</dependency>
 		<dependency>
-			<groupId>log4j</groupId>
-			<artifactId>log4j</artifactId>
-			<version>1.2.16</version>
+			<groupId>org.slf4j</groupId>
+			<artifactId>slf4j-log4j12</artifactId>
+			<version>1.6.1</version>
+			<scope>compile</scope>
 		</dependency>
 		<dependency>
 			<groupId>xerces</groupId>
@@ -187,5 +190,11 @@
 			<artifactId>staging-areas</artifactId>
 			<version>0.0.1-SNAPSHOT</version>
 		</dependency>
+		<dependency>
+			<groupId>com.samskivert</groupId>
+			<artifactId>jmustache</artifactId>
+			<version>1.9</version>
+			<type>jar</type>
+		</dependency>
 	</dependencies>
 </project>

persistence/src/main/java/edu/unc/lib/dl/update/AtomDCToMODSFilter.java

@@ -24,7 +24,7 @@
 import edu.unc.lib.dl.util.ContentModelHelper;
 import edu.unc.lib.dl.xml.ModsXmlHelper;
 
-public class AtomDCToMODSFilter extends MODSUIPFilter {
+public class AtomDCToMODSFilter extends MODSValidationUIPFilter {
 	private static Logger log = Logger.getLogger(AtomDCToMODSFilter.class);
 	private final String datastreamName = AtomPubMetadataParserUtil.ATOM_DC_DATASTREAM;
 

persistence/src/main/java/edu/unc/lib/dl/update/BulkMetadataDatastreamUIP.java

@@ -0,0 +1,59 @@
+/**
+ * Copyright 2008 The University of North Carolina at Chapel Hill
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.unc.lib.dl.update;
+
+import org.jdom2.Document;
+
+import edu.unc.lib.dl.fedora.PID;
+
+/**
+ * @author bbpennel
+ * @date Jul 13, 2015
+ */
+public class BulkMetadataDatastreamUIP extends MetadataUIP {
+	private String lastModified;
+	private String datastream;
+	
+	/**
+	 * @param pid
+	 * @param user
+	 * @param operation
+	 */
+	public BulkMetadataDatastreamUIP(PID pid, String user, UpdateOperation operation,
+			String datastream, String lastModified, Document content) {
+		super(pid, user, operation);
+		this.lastModified = lastModified;
+		this.datastream = datastream;
+		
+		getIncomingData().put(datastream, content.getRootElement());
+	}
+
+	public String getLastModified() {
+		return lastModified;
+	}
+
+	public void setLastModified(String lastModified) {
+		this.lastModified = lastModified;
+	}
+
+	public String getDatastream() {
+		return datastream;
+	}
+
+	public void setDatastream(String datastream) {
+		this.datastream = datastream;
+	}
+}