Skip to content

Commit 10bf2bf

Browse files
bbpennelbbpennel
authored
BXC-3178 - Support migration SIP ingest (#1281)
* Add normalization job for preconstructed deposits * Add agent for cdm to boxc migration
1 parent c802a28 commit 10bf2bf

File tree

7 files changed

+320
-25
lines changed

7 files changed

+320
-25
lines changed

‎deposit-utils/src/main/java/edu/unc/lib/boxc/deposit/api/DepositMethod.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@ public enum DepositMethod {
2727
SWORD20("SWORD 2.0"),
2828
CDRAPI1("CDR API 1.0"),
2929
CDRCollector("CDR Collector 1.0"),
30-
BXC3_TO_5_MIGRATION_UTIL("BXC3 To BXC5 Migration Utility");
30+
BXC3_TO_5_MIGRATION_UTIL("BXC3 To BXC5 Migration Utility"),
31+
CDM_TO_BXC_MIGRATION("CDM To Box-c Migration");
3132

3233
private String label;
3334

‎deposit-utils/src/main/java/edu/unc/lib/boxc/deposit/impl/model/DepositDirectoryManager.java

+8
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
* @author bbpennel
4646
*/
4747
public class DepositDirectoryManager {
48+
public static final String MODEL_FILENAME = "model.n3";
4849

4950
private Path depositDir;
5051
private Path descriptionDir;
@@ -156,6 +157,13 @@ private Path makeMetadataFilePath(Path parentPath, PID pid, String extension) {
156157
return mdPath.resolve(pid.getId() + extension);
157158
}
158159

160+
/**
161+
* @return Path of triples file containing the deposit model
162+
*/
163+
public Path getModelPath() {
164+
return depositDir.resolve(MODEL_FILENAME);
165+
}
166+
159167
public Path getDepositDir() {
160168
return depositDir;
161169
}

‎deposit/src/main/java/edu/unc/lib/deposit/normalize/PreconstructedDepositJob.java

+87
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
/**
2+
* Copyright 2008 The University of North Carolina at Chapel Hill
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package edu.unc.lib.deposit.normalize;
17+
18+
import static org.slf4j.LoggerFactory.getLogger;
19+
20+
import java.io.IOException;
21+
import java.net.URI;
22+
import java.nio.file.Files;
23+
import java.nio.file.Path;
24+
import java.nio.file.Paths;
25+
import java.util.Map;
26+
27+
import org.apache.commons.io.FileUtils;
28+
import org.apache.jena.rdf.model.Model;
29+
import org.slf4j.Logger;
30+
31+
import edu.unc.lib.boxc.deposit.api.RedisWorkerConstants.DepositField;
32+
import edu.unc.lib.boxc.deposit.impl.model.DepositDirectoryManager;
33+
import edu.unc.lib.boxc.model.api.rdf.RDFModelUtil;
34+
import edu.unc.lib.deposit.work.AbstractDepositJob;
35+
36+
/**
37+
* Normalization job to import a preconstructed deposit directory
38+
*
39+
* @author bbpennel
40+
*/
41+
public class PreconstructedDepositJob extends AbstractDepositJob {
42+
private static final Logger log = getLogger(PreconstructedDepositJob.class);
43+
44+
public PreconstructedDepositJob() {
45+
super();
46+
}
47+
48+
public PreconstructedDepositJob(String uuid, String depositUUID) {
49+
super(uuid, depositUUID);
50+
}
51+
52+
@Override
53+
public void runJob() {
54+
try {
55+
DepositDirectoryManager dirManager = new DepositDirectoryManager(
56+
depositPID, getDepositsDirectory().toPath(), true);
57+
Map<String, String> depositStatus = getDepositStatus();
58+
// Determine if we are importing an external deposit dir, or starting from one already in place
59+
String sourceUriProp = depositStatus.get(DepositField.sourceUri.name());
60+
if (sourceUriProp != null) {
61+
URI sourceUri = URI.create(sourceUriProp);
62+
Path sourcePath = Paths.get(sourceUri);
63+
// Check to see if the path is within the deposits directory or external
64+
if (!sourcePath.startsWith(getDepositsDirectory().toPath())) {
65+
// external path, so move into the deposits directory
66+
log.info("Importing external deposit directory {}", sourcePath);
67+
FileUtils.copyDirectory(sourcePath.toFile(), getDepositDirectory());
68+
}
69+
}
70+
// Check to see if there is a model file to import
71+
Path modelPath = dirManager.getModelPath();
72+
if (Files.exists(modelPath)) {
73+
log.info("Importing preconstructed model file included in deposit directory");
74+
Model importModel = RDFModelUtil.createModel(Files.newInputStream(modelPath), "N3");
75+
Model depositModel = getReadOnlyModel();
76+
commit(() -> {
77+
depositModel.removeAll();
78+
depositModel.add(importModel);
79+
});
80+
} else {
81+
log.info("No model file provided, skipping import of additional deposit model properties");
82+
}
83+
} catch (IOException e) {
84+
failJob(e, "Failed to import deposit directory");
85+
}
86+
}
87+
}

‎deposit/src/main/java/edu/unc/lib/deposit/work/DepositSupervisor.java

+19-23
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@
6666
import edu.unc.lib.deposit.normalize.CDRMETS2N3BagJob;
6767
import edu.unc.lib.deposit.normalize.DirectoryToBagJob;
6868
import edu.unc.lib.deposit.normalize.NormalizeFileObjectsJob;
69+
import edu.unc.lib.deposit.normalize.PreconstructedDepositJob;
6970
import edu.unc.lib.deposit.normalize.Simple2N3BagJob;
7071
import edu.unc.lib.deposit.normalize.UnpackDepositJob;
7172
import edu.unc.lib.deposit.transfer.TransferBinariesToStorageJob;
@@ -743,29 +744,24 @@ private Job getNextJob(Job job, String depositUUID, Map<String, String> status,
743744
}
744745
}
745746

746-
// Deposit package type may be converted to N3
747-
if (!packagingType.equals(PackagingType.BAG_WITH_N3.getUri())) {
748-
Job conversion = null;
749-
// we need to add N3 packaging to this bag
750-
if (packagingType.equals(PackagingType.METS_CDR.getUri())) {
751-
conversion = makeJob(CDRMETS2N3BagJob.class, depositUUID);
752-
} else if (packagingType.equals(PackagingType.SIMPLE_OBJECT.getUri())) {
753-
conversion = makeJob(Simple2N3BagJob.class, depositUUID);
754-
} else if (packagingType.equals(PackagingType.BAGIT.getUri())) {
755-
conversion = makeJob(BagIt2N3BagJob.class, depositUUID);
756-
} else if (packagingType.equals(PackagingType.DIRECTORY.getUri())) {
757-
conversion = makeJob(DirectoryToBagJob.class, depositUUID);
758-
}
759-
760-
if (conversion == null) {
761-
String msg = MessageFormat
762-
.format("Cannot convert deposit package to N3 BagIt."
763-
+ " No converter for this packaging type(s): {0}",
764-
packagingType);
765-
throw new DepositFailedException(msg);
766-
} else if (!successfulJobs.contains(conversion.getClassName())) {
767-
return conversion;
768-
}
747+
Class<?> conversionClass = null;
748+
if (packagingType.equals(PackagingType.BAG_WITH_N3.getUri())) {
749+
conversionClass = PreconstructedDepositJob.class;
750+
} else if (packagingType.equals(PackagingType.METS_CDR.getUri())) {
751+
conversionClass = CDRMETS2N3BagJob.class;
752+
} else if (packagingType.equals(PackagingType.SIMPLE_OBJECT.getUri())) {
753+
conversionClass = Simple2N3BagJob.class;
754+
} else if (packagingType.equals(PackagingType.BAGIT.getUri())) {
755+
conversionClass = BagIt2N3BagJob.class;
756+
} else if (packagingType.equals(PackagingType.DIRECTORY.getUri())) {
757+
conversionClass = DirectoryToBagJob.class;
758+
}
759+
if (conversionClass == null) {
760+
String msg = MessageFormat.format("Cannot convert deposit package to N3 BagIt."
761+
+ " No converter for this packaging type(s): {0}", packagingType);
762+
throw new DepositFailedException(msg);
763+
} else if (!successfulJobs.contains(conversionClass.getName())) {
764+
return makeJob(conversionClass, depositUUID);
769765
}
770766

771767
// Normalize all fileObjects into Works

‎deposit/src/main/webapp/WEB-INF/deposit-jobs-context.xml

+4
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,10 @@
148148
scope="prototype">
149149
</bean>
150150

151+
<bean id="PreconstructedDepositJob" class="edu.unc.lib.deposit.normalize.PreconstructedDepositJob"
152+
scope="prototype">
153+
</bean>
154+
151155
<bean id="bagitValidationExecutor" class="java.util.concurrent.Executors"
152156
factory-method="newFixedThreadPool" destroy-method="shutdownNow">
153157
<constructor-arg value="${job.bagitValidation.workers:50}"/>

‎deposit/src/test/java/edu/unc/lib/deposit/normalize/PreconstructedDepositJobTest.java

+198
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
/**
2+
* Copyright 2008 The University of North Carolina at Chapel Hill
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package edu.unc.lib.deposit.normalize;
17+
18+
import static edu.unc.lib.boxc.common.test.TestHelpers.setField;
19+
import static org.junit.Assert.assertEquals;
20+
import static org.junit.Assert.assertTrue;
21+
import static org.junit.Assert.fail;
22+
import static org.mockito.Matchers.anyString;
23+
import static org.mockito.Mockito.when;
24+
25+
import java.io.FileNotFoundException;
26+
import java.io.Writer;
27+
import java.nio.file.Files;
28+
import java.nio.file.Path;
29+
import java.util.HashMap;
30+
import java.util.Map;
31+
32+
import org.apache.jena.rdf.model.Model;
33+
import org.apache.jena.rdf.model.ModelFactory;
34+
import org.apache.jena.rdf.model.Resource;
35+
import org.apache.jena.vocabulary.DC;
36+
import org.junit.Before;
37+
import org.junit.Test;
38+
39+
import edu.unc.lib.boxc.deposit.api.RedisWorkerConstants.DepositField;
40+
import edu.unc.lib.boxc.deposit.impl.model.DepositDirectoryManager;
41+
import edu.unc.lib.deposit.fcrepo4.AbstractDepositJobTest;
42+
import edu.unc.lib.deposit.work.JobFailedException;
43+
44+
/**
45+
* @author bbpennel
46+
*/
47+
public class PreconstructedDepositJobTest extends AbstractDepositJobTest {
48+
private PreconstructedDepositJob job;
49+
private Map<String, String> status;
50+
51+
@Before
52+
public void setup() throws Exception {
53+
status = new HashMap<>();
54+
when(depositStatusFactory.get(anyString())).thenReturn(status);
55+
56+
job = new PreconstructedDepositJob();
57+
job.setDepositUUID(depositUUID);
58+
job.setDepositDirectory(depositDir);
59+
setField(job, "pidMinter", pidMinter);
60+
setField(job, "depositModelManager", depositModelManager);
61+
setField(job, "depositsDirectory", depositsDirectory);
62+
setField(job, "depositStatusFactory", depositStatusFactory);
63+
}
64+
65+
@Test
66+
public void noSourceNoModelFilePrepopulatedModel() throws Exception {
67+
Model writeModel = job.getWritableModel();
68+
Resource preResc = writeModel.getResource(depositPid.getRepositoryPath());
69+
preResc.addLiteral(DC.title, "Test value");
70+
job.closeModel();
71+
72+
job.run();
73+
74+
Model model = job.getReadOnlyModel();
75+
assertEquals(1, model.listStatements().toList().size());
76+
Resource postResc = model.getResource(depositPid.getRepositoryPath());
77+
assertTrue(postResc.hasLiteral(DC.title, "Test value"));
78+
}
79+
80+
@Test
81+
public void withExternalSourceNoModelFile() throws Exception {
82+
Path externalBasePath = tmpFolder.newFolder().toPath();
83+
DepositDirectoryManager extDirManager = new DepositDirectoryManager(depositPid, externalBasePath, true);
84+
status.put(DepositField.sourceUri.name(), extDirManager.getDepositDir().toUri().toString());
85+
Files.createFile(extDirManager.getPremisPath(depositPid));
86+
87+
job.run();
88+
89+
Model model = job.getReadOnlyModel();
90+
assertEquals(0, model.listStatements().toList().size());
91+
92+
DepositDirectoryManager intDirManager = new DepositDirectoryManager(
93+
depositPid, depositsDirectory.toPath(), true, false);
94+
Path intPremisPath = intDirManager.getPremisPath(depositPid);
95+
assertTrue(Files.exists(intPremisPath));
96+
}
97+
98+
@Test
99+
public void withInternalSourceNoModelFile() throws Exception {
100+
DepositDirectoryManager preDirManager = new DepositDirectoryManager(
101+
depositPid, depositsDirectory.toPath(), true);
102+
status.put(DepositField.sourceUri.name(), preDirManager.getDepositDir().toUri().toString());
103+
Files.createFile(preDirManager.getPremisPath(depositPid));
104+
105+
job.run();
106+
107+
Model model = job.getReadOnlyModel();
108+
assertEquals(0, model.listStatements().toList().size());
109+
110+
DepositDirectoryManager postDirManager = new DepositDirectoryManager(
111+
depositPid, depositsDirectory.toPath(), true, false);
112+
Path intPremisPath = postDirManager.getPremisPath(depositPid);
113+
assertTrue(Files.exists(intPremisPath));
114+
}
115+
116+
@Test
117+
public void noSourceWithModelFile() throws Exception {
118+
DepositDirectoryManager preDirManager = new DepositDirectoryManager(
119+
depositPid, depositsDirectory.toPath(), true);
120+
Model importModel = ModelFactory.createDefaultModel();
121+
Resource preResc = importModel.getResource(depositPid.getRepositoryPath());
122+
preResc.addLiteral(DC.title, "Import value");
123+
Writer writer = Files.newBufferedWriter(preDirManager.getModelPath());
124+
importModel.write(writer, "N3");
125+
126+
job.run();
127+
128+
Model model = job.getReadOnlyModel();
129+
assertEquals(1, model.listStatements().toList().size());
130+
Resource postResc = model.getResource(depositPid.getRepositoryPath());
131+
assertTrue(postResc.hasLiteral(DC.title, "Import value"));
132+
}
133+
134+
@Test
135+
public void withSourceWithModelFile() throws Exception {
136+
Path externalBasePath = tmpFolder.newFolder().toPath();
137+
DepositDirectoryManager extDirManager = new DepositDirectoryManager(
138+
depositPid, externalBasePath, true);
139+
Model importModel = ModelFactory.createDefaultModel();
140+
Resource extResc = importModel.getResource(depositPid.getRepositoryPath());
141+
extResc.addLiteral(DC.title, "Import me");
142+
Writer writer = Files.newBufferedWriter(extDirManager.getModelPath());
143+
importModel.write(writer, "N3");
144+
145+
status.put(DepositField.sourceUri.name(), extDirManager.getDepositDir().toUri().toString());
146+
Files.createFile(extDirManager.getPremisPath(depositPid));
147+
148+
job.run();
149+
150+
Model model = job.getReadOnlyModel();
151+
assertEquals(1, model.listStatements().toList().size());
152+
Resource postResc = model.getResource(depositPid.getRepositoryPath());
153+
assertTrue(postResc.hasLiteral(DC.title, "Import me"));
154+
155+
DepositDirectoryManager intDirManager = new DepositDirectoryManager(
156+
depositPid, depositsDirectory.toPath(), true, false);
157+
Path intPremisPath = intDirManager.getPremisPath(depositPid);
158+
assertTrue(Files.exists(intPremisPath));
159+
}
160+
161+
@Test
162+
public void noSourceWithModelFilePrepopulatedModel() throws Exception {
163+
Model writeModel = job.getWritableModel();
164+
Resource preResc = writeModel.getResource(depositPid.getRepositoryPath());
165+
preResc.addLiteral(DC.title, "Pre-existing Condition");
166+
job.closeModel();
167+
168+
DepositDirectoryManager preDirManager = new DepositDirectoryManager(
169+
depositPid, depositsDirectory.toPath(), true);
170+
Model importModel = ModelFactory.createDefaultModel();
171+
Resource importResc = importModel.getResource(depositPid.getRepositoryPath());
172+
importResc.addLiteral(DC.title, "Imported Titles");
173+
Writer writer = Files.newBufferedWriter(preDirManager.getModelPath());
174+
importModel.write(writer, "N3");
175+
176+
job.run();
177+
178+
Model model = job.getReadOnlyModel();
179+
assertEquals(1, model.listStatements().toList().size());
180+
Resource postResc = model.getResource(depositPid.getRepositoryPath());
181+
assertTrue(postResc.hasLiteral(DC.title, "Imported Titles"));
182+
}
183+
184+
@Test
185+
public void sourceDirDoesNotExistTest() throws Exception {
186+
Path externalBasePath = tmpFolder.newFolder().toPath();
187+
Files.delete(externalBasePath);
188+
DepositDirectoryManager extDirManager = new DepositDirectoryManager(depositPid, externalBasePath, true, false);
189+
status.put(DepositField.sourceUri.name(), extDirManager.getDepositDir().toUri().toString());
190+
191+
try {
192+
job.run();
193+
fail();
194+
} catch (JobFailedException e) {
195+
assertTrue(e.getCause() instanceof FileNotFoundException);
196+
}
197+
}
198+
}

‎model-api/src/main/java/edu/unc/lib/boxc/model/api/SoftwareAgentConstants.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,8 @@ public enum SoftwareAgent {
4242
servicesAPI("bxc-services", true),
4343
fixityCheckingService("fixity", "3.0"),
4444
embargoUpdateService("embargo-update", "3.0"),
45-
migrationUtil("bxc-migration-util", true),
45+
bxc3ToBxc5MigrationUtil("bxc-migration-util", true),
46+
cdmToBxcMigrationUtil("cdm-to-boxc-migration-util", "1.0"),
4647
clamav("clamav", "0.99"),
4748
depositBxc3("deposit", "3.0"),
4849
FITS("fits", "1.0.6"),

0 commit comments

Comments
 (0)