Skip to content

Commit

Permalink
Merge pull request #1422 from UNC-Libraries/bxc-3676-primary-tombstone
Browse files Browse the repository at this point in the history 
BXC-3676 - Fix error when indexing work with tombstone primary object
  • Loading branch information
@sharonluong
sharonluong authored Jun 13, 2022
2 parents af612b7 + 8159ac0 commit bb63d76
Show file tree
Hide file tree
Showing 8 changed files with 307 additions and 68 deletions.
29 changes: 29 additions & 0 deletions model-api/src/main/java/edu/unc/lib/boxc/model/api/exceptions/TombstoneFoundException.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/**
* Copyright 2008 The University of North Carolina at Chapel Hill
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package edu.unc.lib.boxc.model.api.exceptions;

/**
* Exception indicating that the requested resource has been destroyed and replaced by a tombstone object
*
* @author bbpennel
*/
public class TombstoneFoundException extends RepositoryException {
private static final long serialVersionUID = 1L;

public TombstoneFoundException(String message) {
super(message);
}
}
7 changes: 3 additions & 4 deletions model-api/src/main/java/edu/unc/lib/boxc/model/api/objects/WorkObject.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,14 +71,13 @@ FileObject addDataFile(URI storageUri, String filename, String mimetype, String
* original file, using the provided pid as the identifier for the new
* FileObject.
*
*
* @param contentStream
* Inputstream containing the binary content for the data file. Required.
* @param filePid
* @param storageUri
* @param filename
* @param mimetype
* @param sha1Checksum
* @param md5Checksum
* @param model
* model containing properties for the new fileObject
* @return
*/
FileObject addDataFile(PID filePid, URI storageUri, String filename, String mimetype, String sha1Checksum,
Expand Down
12 changes: 11 additions & 1 deletion model-fcrepo/src/main/java/edu/unc/lib/boxc/model/fcrepo/objects/WorkObjectImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

import java.net.URI;

import edu.unc.lib.boxc.model.api.exceptions.NotFoundException;
import edu.unc.lib.boxc.model.api.exceptions.TombstoneFoundException;
import org.apache.commons.lang3.StringUtils;
import org.apache.jena.rdf.model.Model;
import org.apache.jena.rdf.model.ModelFactory;
Expand All @@ -39,6 +41,8 @@
import edu.unc.lib.boxc.model.api.services.RepositoryObjectFactory;
import edu.unc.lib.boxc.model.fcrepo.ids.PIDs;
import edu.unc.lib.boxc.model.fcrepo.services.RepositoryObjectDriver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
* A repository object which represents a single work, and should contain one or
Expand All @@ -51,6 +55,7 @@
*
*/
public class WorkObjectImpl extends AbstractContentContainerObject implements WorkObject {
private static final Logger log = LoggerFactory.getLogger(WorkObjectImpl.class);

protected WorkObjectImpl(PID pid, RepositoryObjectDriver driver, RepositoryObjectFactory repoObjFactory) {
super(pid, driver, repoObjFactory);
Expand Down Expand Up @@ -116,7 +121,12 @@ public FileObject getPrimaryObject() {
}

PID primaryPid = PIDs.get(primaryStmt.getResource().getURI());
return driver.getRepositoryObject(primaryPid, FileObject.class);
try {
return driver.getRepositoryObject(primaryPid, FileObject.class);
} catch (TombstoneFoundException e) {
log.debug("Cannot retrieve primary object for {}", getPid().getId(), e);
}
return null;
}

@Override
Expand Down
5 changes: 5 additions & 0 deletions ...l-fcrepo/src/main/java/edu/unc/lib/boxc/model/fcrepo/services/RepositoryObjectDriver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
import java.util.ArrayList;
import java.util.List;

import edu.unc.lib.boxc.model.api.exceptions.TombstoneFoundException;
import edu.unc.lib.boxc.model.api.objects.Tombstone;
import org.apache.http.HttpStatus;
import org.apache.jena.query.QueryExecution;
import org.apache.jena.query.QuerySolution;
Expand Down Expand Up @@ -162,6 +164,9 @@ public RepositoryObject getRepositoryObject(PID pid) {
public <T extends RepositoryObject> T getRepositoryObject(PID pid, Class<T> type)
throws ObjectTypeMismatchException {
RepositoryObject repoObj = repositoryObjectLoader.getRepositoryObject(pid);
if (repoObj instanceof Tombstone) {
throw new TombstoneFoundException("Tombstone found, requested object " + pid + " no longer exists");
}
if (!type.isInstance(repoObj)) {
throw new ObjectTypeMismatchException("Requested object " + pid + " is not a " + type.getName());
}
Expand Down
9 changes: 9 additions & 0 deletions operations/src/main/java/edu/unc/lib/boxc/operations/impl/destroy/DestroyObjectsJob.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import java.util.Collections;
import java.util.List;

import edu.unc.lib.boxc.model.api.objects.WorkObject;
import org.apache.jena.rdf.model.Model;
import org.apache.jena.rdf.model.ModelFactory;
import org.apache.jena.rdf.model.NodeIterator;
Expand Down Expand Up @@ -95,6 +96,14 @@ public void run() {

if (!repoObj.getResource(true).hasProperty(RDF.type, Cdr.Tombstone)) {
RepositoryObject parentObj = repoObj.getParent();
// If the object being deleted is the primary object of a work, then clear the relation
if (parentObj instanceof WorkObject && repoObj instanceof FileObject) {
var parentWork = (WorkObject) parentObj;
var primaryObj = parentWork.getPrimaryObject();
if (primaryObj != null && primaryObj.getPid().equals(repoObj.getPid())) {
parentWork.clearPrimaryObject();
}
}

// purge tree with repoObj as root from repository
// Add the root of the tree to delete
Expand Down
182 changes: 122 additions & 60 deletions operations/src/test/java/edu/unc/lib/boxc/operations/impl/destroy/DestroyObjectsJobIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,57 +15,6 @@
*/
package edu.unc.lib.boxc.operations.impl.destroy;

import static edu.unc.lib.boxc.model.api.rdf.CdrAcl.markedForDeletion;
import static edu.unc.lib.boxc.model.api.sparql.SparqlUpdateHelper.createSparqlReplace;
import static edu.unc.lib.boxc.model.api.xml.JDOMNamespaceUtil.CDR_MESSAGE_NS;
import static edu.unc.lib.boxc.model.fcrepo.ids.RepositoryPaths.getContentRootPid;
import static edu.unc.lib.boxc.operations.jms.indexing.IndexingActionType.DELETE_SOLR_TREE;
import static java.util.Arrays.asList;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyString;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.mockito.MockitoAnnotations.initMocks;

import java.io.File;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.io.FileUtils;
import org.apache.jena.rdf.model.Model;
import org.apache.jena.rdf.model.Resource;
import org.apache.jena.vocabulary.RDF;
import org.fcrepo.client.FcrepoClient;
import org.jdom2.Document;
import org.jdom2.Element;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.mockito.Mock;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.ContextHierarchy;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

import edu.unc.lib.boxc.auth.api.models.AccessGroupSet;
import edu.unc.lib.boxc.auth.api.models.AgentPrincipals;
import edu.unc.lib.boxc.auth.api.services.AccessControlService;
Expand All @@ -75,6 +24,7 @@
import edu.unc.lib.boxc.fcrepo.utils.TransactionManager;
import edu.unc.lib.boxc.model.api.ResourceType;
import edu.unc.lib.boxc.model.api.ids.PID;
import edu.unc.lib.boxc.model.api.ids.PIDMinter;
import edu.unc.lib.boxc.model.api.objects.AdminUnit;
import edu.unc.lib.boxc.model.api.objects.BinaryObject;
import edu.unc.lib.boxc.model.api.objects.CollectionObject;
Expand Down Expand Up @@ -104,6 +54,57 @@
import edu.unc.lib.boxc.persist.impl.storage.StorageLocationManagerImpl;
import edu.unc.lib.boxc.search.api.models.ObjectPath;
import edu.unc.lib.boxc.search.solr.services.ObjectPathFactory;
import org.apache.commons.io.FileUtils;
import org.apache.jena.rdf.model.Model;
import org.apache.jena.rdf.model.RDFNode;
import org.apache.jena.rdf.model.Resource;
import org.apache.jena.vocabulary.RDF;
import org.fcrepo.client.FcrepoClient;
import org.jdom2.Document;
import org.jdom2.Element;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.mockito.Mock;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.ContextHierarchy;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

import java.io.File;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static edu.unc.lib.boxc.model.api.rdf.CdrAcl.markedForDeletion;
import static edu.unc.lib.boxc.model.api.sparql.SparqlUpdateHelper.createSparqlReplace;
import static edu.unc.lib.boxc.model.api.xml.JDOMNamespaceUtil.CDR_MESSAGE_NS;
import static edu.unc.lib.boxc.model.fcrepo.ids.RepositoryPaths.getContentRootPid;
import static edu.unc.lib.boxc.operations.jms.indexing.IndexingActionType.DELETE_SOLR_TREE;
import static java.util.Arrays.asList;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyString;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.mockito.MockitoAnnotations.initMocks;

/**
*
Expand Down Expand Up @@ -161,6 +162,8 @@ public class DestroyObjectsJobIT {
private StorageLocationManagerImpl locationManager;
@Autowired
private BinaryTransferService transferService;
@Autowired
private PIDMinter pidMinter;

private AgentPrincipals agent;

Expand Down Expand Up @@ -224,6 +227,59 @@ public void destroySingleFileObjectTest() throws Exception {
assertMessagePresent(docCaptor.getAllValues(), filesToCleanup, null);
}

@Test
public void destroyPrimaryObjectInMultiFileWorkTest() throws Exception {
PID fileObjPid = objsToDestroy.get(2);
var workObj = repoObjLoader.getWorkObject(objsToDestroy.get(1));
workObj.setPrimaryObject(fileObjPid);
// Add a second file
var fileObj2 = addFileToWork(workObj);
treeIndexer.indexAll(baseAddress);

initializeJob(asList(fileObjPid));

FileObject fileObj = repoObjLoader.getFileObject(fileObjPid);
Map<URI, Map<String, String>> filesToCleanup = derivativesToCleanup(fileObj);

URI contentUri = fileObj.getOriginalFile().getContentUri();
assertTrue(Files.exists(Paths.get(contentUri)));

job.run();

Model logParentModel = fileObj.getParent().getPremisLog().getEventsModel();
assertTrue(logParentModel.contains(null, RDF.type, Premis.Deletion));
assertTrue(logParentModel.contains(null, Premis.note,
"1 object(s) were destroyed"));

Tombstone stoneFile = repoObjLoader.getTombstone(fileObjPid);
Resource stoneResc = stoneFile.getResource();
assertTrue(stoneResc.hasProperty(RDF.type, Cdr.Tombstone));
assertTrue(stoneResc.hasProperty(RDF.type, Cdr.FileObject));
// check to make sure metadata from binary was retained by file obj's tombstone
assertTrue(stoneResc.hasProperty(Ebucore.filename));
assertTrue(stoneResc.hasProperty(Cdr.hasMessageDigest));
assertTrue(stoneResc.hasProperty(Ebucore.hasMimeType));
assertTrue(stoneResc.hasProperty(Cdr.hasSize));

assertFalse("Original file must be deleted", Files.exists(Paths.get(contentUri)));

verify(indexingMessageSender).sendIndexingOperation(anyString(), eq(fileObjPid), eq(DELETE_SOLR_TREE));

verify(binaryDestroyedMessageSender).sendMessage(docCaptor.capture());

assertMessagePresent(docCaptor.getAllValues(), filesToCleanup, null);

workObj.shouldRefresh();

assertNull("Primary object of work must now be null", workObj.getPrimaryObject());
var members = workObj.getMembers();
assertTrue("Second file must still be present", members.stream()
.anyMatch(m -> m.getPid().equals(fileObj2.getPid())));
assertEquals(2, members.size());
assertFalse("Primary object must be unset",
workObj.getModel().contains(null, Cdr.primaryObject, (RDFNode) null));
}

@Test
public void destroyObjectsInSameTreeTest() throws Exception {
//remove unrelated folder obj before running job
Expand Down Expand Up @@ -383,15 +439,7 @@ private List<PID> createContentTree() throws Exception {
collection.addMember(folder2);
WorkObject work = repoObjFactory.createWorkObject(null);
folder.addMember(work);
String bodyString = "Content";
String mimetype = "text/plain";
Path storagePath = Paths.get(locationManager.getStorageLocationById(LOC1_ID).getNewStorageUri(work.getPid()));
Files.createDirectories(storagePath);
File contentFile = Files.createTempFile(storagePath, "file", ".txt").toFile();
String sha1 = "4f9be057f0ea5d2ba72fd2c810e8d7b9aa98b469";
String filename = contentFile.getName();
FileUtils.writeStringToFile(contentFile, bodyString, "UTF-8");
FileObject file = work.addDataFile(contentFile.toPath().toUri(), filename, mimetype, sha1, null);
var file = addFileToWork(work);

treeIndexer.indexAll(baseAddress);

Expand All @@ -404,6 +452,20 @@ private List<PID> createContentTree() throws Exception {
return objsToDestroy;
}

private FileObject addFileToWork(WorkObject workObj) throws Exception {
String bodyString = "Content";
String mimetype = "text/plain";
PID filePid = pidMinter.mintContentPid();
Path storagePath = Paths.get(locationManager.getStorageLocationById(LOC1_ID)
.getNewStorageUri(filePid));
Files.createDirectories(storagePath);
File contentFile = Files.createTempFile(storagePath, "file", ".txt").toFile();
String sha1 = "4f9be057f0ea5d2ba72fd2c810e8d7b9aa98b469";
String filename = contentFile.getName();
FileUtils.writeStringToFile(contentFile, bodyString, "UTF-8");
return workObj.addDataFile(filePid, contentFile.toPath().toUri(), filename, mimetype, sha1, null, null);
}

private void initializeJob(List<PID> objsToDestroy) {
DestroyObjectsRequest request = new DestroyObjectsRequest("jobid", agent,
objsToDestroy.stream().map(PID::getId).toArray(String[]::new));
Expand Down
13 changes: 11 additions & 2 deletions ...camel-app/src/test/java/edu/unc/lib/boxc/services/camel/solr/AbstractSolrProcessorIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,13 @@
import java.io.File;
import java.io.InputStream;
import java.net.URI;
import java.nio.file.Paths;
import java.util.UUID;

import edu.unc.lib.boxc.indexing.solr.test.RepositoryObjectSolrIndexer;
import edu.unc.lib.boxc.model.fcrepo.ids.PIDs;
import edu.unc.lib.boxc.persist.api.storage.StorageLocationManager;
import edu.unc.lib.boxc.persist.impl.storage.StorageLocationTestHelper;
import org.apache.camel.Exchange;
import org.apache.camel.Message;
import org.apache.camel.test.spring.CamelSpringRunner;
Expand Down Expand Up @@ -104,6 +109,8 @@ public abstract class AbstractSolrProcessorIT {
protected RepositoryObjectTreeIndexer treeIndexer;
@Autowired
protected RepositoryObjectSolrIndexer repositoryObjectSolrIndexer;
@Autowired
protected StorageLocationManager locManager;

protected ContentRootObject rootObj;
protected AdminUnit unitObj;
Expand Down Expand Up @@ -143,10 +150,12 @@ protected void indexObjectsInTripleStore() throws Exception {
}

protected URI makeContentUri(String content) throws Exception {
File contentFile = File.createTempFile("test", ".txt");
var loc1 = locManager.getStorageLocationById(StorageLocationTestHelper.LOC1_ID);
var storageUri = loc1.getNewStorageUri(PIDs.get(UUID.randomUUID().toString()));
var contentFile = new File(storageUri);
contentFile.deleteOnExit();
FileUtils.write(contentFile, content, UTF_8);
return contentFile.toPath().toUri();
return storageUri;
}

protected InputStream streamResource(String resourcePath) throws Exception {
Expand Down
Loading

0 comments on commit bb63d76

Please sign in to comment.