Skip to content

Commit 54f5544

Browse files
wowasawowasa
authored
Merge pull request #219 from clarin-eric/dev
Dev
2 parents 0c8113e + ea7e376 commit 54f5544

File tree

36 files changed

+457
-426
lines changed

36 files changed

+457
-426
lines changed

‎CHANGES.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
# 6.3.1
2+
- adapting concept registry access
3+
14
# 6.3.0
25
- upgrade to Spring Boot 3.2.2
36
- adding origin to duplicated mdSelflinks and rendering list in HTML view (issue #213)

‎curation-api/src/main/java/eu/clarin/cmdi/curation/api/subprocessor/instance/InstanceFacetProcessor.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import java.util.Map;
77
import java.util.stream.Collectors;
88

9+
import eu.clarin.cmdi.curation.cr.exception.CRServiceStorageException;
910
import org.apache.commons.lang3.StringUtils;
1011
import org.springframework.beans.factory.annotation.Autowired;
1112
import org.springframework.stereotype.Component;
@@ -190,6 +191,9 @@ public void process(CMDInstance instance, CMDInstanceReport instanceReport) {
190191
instanceReport.details.add(new Detail(Severity.FATAL, "file", "can't parse file '" + instance.getPath().getFileName() + "'"));
191192
instanceReport.isProcessable = false;
192193
}
194+
catch (CRServiceStorageException e) {
195+
throw new RuntimeException(e);
196+
}
193197
}
194198
}
195199
}

‎curation-api/src/main/java/eu/clarin/cmdi/curation/api/subprocessor/instance/XmlValidator.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import javax.xml.parsers.SAXParserFactory;
88
import javax.xml.validation.ValidatorHandler;
99

10+
import eu.clarin.cmdi.curation.cr.exception.CRServiceStorageException;
1011
import org.springframework.beans.factory.annotation.Autowired;
1112
import org.springframework.stereotype.Component;
1213
import org.xml.sax.Attributes;
@@ -62,8 +63,12 @@ public void process(CMDInstance instance, CMDInstanceReport report) {
6263

6364
return;
6465
}
65-
66-
final int messageCount = report.details.size();
66+
catch (CRServiceStorageException e) {
67+
68+
throw new RuntimeException(e);
69+
}
70+
71+
final int messageCount = report.details.size();
6772

6873
schemaValidator.setErrorHandler(new CMDErrorHandler(report));
6974
schemaValidator.setContentHandler(new CMDInstanceContentHandler(instance, report));

‎curation-api/src/main/java/eu/clarin/cmdi/curation/api/subprocessor/profile/ProfileConceptHandler.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import java.util.Map;
88
import java.util.Map.Entry;
99

10+
import eu.clarin.cmdi.curation.cr.exception.CRServiceStorageException;
1011
import org.springframework.beans.factory.annotation.Autowired;
1112
import org.springframework.context.annotation.Scope;
1213
import org.springframework.context.annotation.ScopedProxyMode;
@@ -54,8 +55,11 @@ public void process(CMDProfile profile, CMDProfileReport report) {
5455
return;
5556

5657
}
57-
58-
report.conceptReport = new ConceptReport();
58+
catch (CRServiceStorageException e) {
59+
throw new RuntimeException(e);
60+
}
61+
62+
report.conceptReport = new ConceptReport();
5963

6064
final Map<String, ConceptReport.Concept> conceptMap = new HashMap<String, ConceptReport.Concept>();
6165

‎curation-api/src/main/java/eu/clarin/cmdi/curation/api/subprocessor/profile/ProfileFacetHandler.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
import java.util.Map;
77

8+
import eu.clarin.cmdi.curation.cr.exception.CRServiceStorageException;
89
import org.springframework.beans.factory.annotation.Autowired;
910
import org.springframework.stereotype.Component;
1011

@@ -80,7 +81,10 @@ public void process(CMDProfile profile, CMDProfileReport report) {
8081
report.details.add(new Detail(Severity.FATAL,"facet" , "no ParsedProfile for profile id " + header.getId()));
8182

8283
}
83-
report.facetReport.percProfileCoverage = (double) report.facetReport.numOfFacetsCoveredByProfile/report.facetReport.numOfFacets;
84+
catch (CRServiceStorageException e) {
85+
throw new RuntimeException(e);
86+
}
87+
report.facetReport.percProfileCoverage = (double) report.facetReport.numOfFacetsCoveredByProfile/report.facetReport.numOfFacets;
8488
report.facetReport.score = report.facetReport.percProfileCoverage;
8589
report.score+=report.facetReport.score;
8690
}

‎curation-api/src/main/java/eu/clarin/cmdi/curation/api/subprocessor/profile/ProfileHeaderHandler.java‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import eu.clarin.cmdi.curation.api.subprocessor.AbstractSubprocessor;
1010
import eu.clarin.cmdi.curation.cr.CRService;
1111
import eu.clarin.cmdi.curation.pph.PPHService;
12+
import eu.clarin.cmdi.curation.pph.exception.PPHServiceNotAvailableException;
1213
import lombok.extern.slf4j.Slf4j;
1314

1415
import org.springframework.beans.factory.annotation.Autowired;
@@ -54,17 +55,23 @@ public void process(CMDProfile profile, CMDProfileReport report) {
5455
report.score += report.headerReport.score;
5556
}
5657

57-
pphService.getProfileHeaders().stream()
58-
.filter(profileHeader -> profileHeader.getName().equals(report.headerReport.getName())
59-
&& !profileHeader.getId().equals(report.headerReport.getId()))
60-
.findAny().ifPresent(profileReport -> report.details.add(new Detail(Severity.WARNING,"header",
61-
"The name '" + report.headerReport.getName() + "' of the profile is not unique")));
6258

63-
pphService.getProfileHeaders().stream()
64-
.filter(profileHeader -> profileHeader.getDescription().equals(report.headerReport.getDescription())
65-
&& !profileHeader.getId().equals(report.headerReport.getId()))
66-
.findAny().ifPresent(profileReport -> report.details.add(new Detail(Severity.WARNING,"header",
67-
"The description '" + report.headerReport.getDescription() + "' of the profile is not unique")));
59+
try {
60+
pphService.getProfileHeaders().stream()
61+
.filter(profileHeader -> profileHeader.getName().equals(report.headerReport.getName())
62+
&& !profileHeader.getId().equals(report.headerReport.getId()))
63+
.findAny().ifPresent(profileReport -> report.details.add(new Detail(Severity.WARNING,"header",
64+
"The name '" + report.headerReport.getName() + "' of the profile is not unique")));
6865

66+
pphService.getProfileHeaders().stream()
67+
.filter(profileHeader -> profileHeader.getDescription().equals(report.headerReport.getDescription())
68+
&& !profileHeader.getId().equals(report.headerReport.getId()))
69+
.findAny().ifPresent(profileReport -> report.details.add(new Detail(Severity.WARNING,"header",
70+
"The description '" + report.headerReport.getDescription() + "' of the profile is not unique")));
71+
}
72+
catch (PPHServiceNotAvailableException e) {
73+
74+
throw new RuntimeException(e);
75+
}
6976
}
7077
}

‎curation-api/src/main/java/eu/clarin/cmdi/curation/api/vlo_extension/FacetsMappingCacheFactory.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import java.util.*;
44

5+
import eu.clarin.cmdi.curation.cr.exception.CRServiceStorageException;
56
import org.springframework.beans.factory.annotation.Autowired;
67
import org.springframework.stereotype.Service;
78

@@ -105,8 +106,11 @@ public Map<String, List<Pattern>> createConceptLinkPathMapping() throws NavExcep
105106
log.error("no ProfileCacheEntry object for profile id '{}'", header.getId());
106107

107108
}
108-
109-
return result;
109+
catch (CRServiceStorageException e) {
110+
throw new RuntimeException(e);
111+
}
112+
113+
return result;
110114
}
111115

112116
@Override

‎curation-api/src/test/resources/application.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@ curation:
1616
restApi: "https://catalog.clarin.eu/ds/ComponentRegistry/rest/registry/1.x/profiles"
1717
query: "?registrySpace=published&status=production&status=development&&status=deprecated"
1818
ccr-service:
19-
restApi: "https://openskos.meertens.knaw.nl/ccr/api/find-concepts"
20-
query: "?q=status:[*%20TO%20*]&fl=status,uri,prefLabel@en&format=json&rows=5000"
19+
restApi: "https://vocabularies.clarin.eu/clavas/rest/v1/data"
20+
query: "?uri=${conceptURI}"
2121
facets:
2222
- languageCode
2323
- collection

‎curation-app/src/main/java/eu/clarin/cmdi/curation/app/CurationApp.java‎

Lines changed: 39 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,16 @@
22

33
import java.io.IOException;
44
import java.net.MalformedURLException;
5+
import java.nio.file.DirectoryStream;
56
import java.nio.file.Files;
67
import java.nio.file.Path;
78
import java.time.LocalDateTime;
9+
import java.util.stream.Stream;
810

11+
import eu.clarin.cmdi.curation.pph.exception.PPHServiceNotAvailableException;
912
import org.springframework.beans.factory.annotation.Autowired;
1013
import org.springframework.boot.CommandLineRunner;
1114
import org.springframework.boot.SpringApplication;
12-
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
1315
import org.springframework.boot.autoconfigure.SpringBootApplication;
1416
import org.springframework.boot.autoconfigure.domain.EntityScan;
1517
import org.springframework.boot.context.properties.EnableConfigurationProperties;
@@ -41,7 +43,6 @@
4143
@EntityScan(basePackages = "eu.clarin.linkchecker.persistence.model")
4244
@EnableCaching
4345
@EnableConfigurationProperties
44-
@EnableAutoConfiguration
4546
@Slf4j
4647
public class CurationApp {
4748

@@ -101,29 +102,34 @@ public CommandLineRunner commandLineRunner(ApplicationContext ctx) {
101102
if("all".equalsIgnoreCase(conf.getMode()) || "profile".equalsIgnoreCase(conf.getMode())) {
102103

103104
final AllProfileReport allProfileReport = new AllProfileReport();
104-
105-
106-
pphService.getProfileHeaders().forEach(header -> {
107-
108-
log.info("start processing profile '{}'", header.getId());
109-
110-
try {
111-
CMDProfileReport profileReport = curation.processCMDProfile(header.getId());
112-
113-
allProfileReport.addReport(profileReport);
114-
115-
storage.saveReport(profileReport, CurationEntityType.PROFILE, false);
116-
117-
}
118-
catch (MalformedURLException e1) {
119-
120-
log.error("malformed URL for id '{}' - check the setting for curation.pph-service.restApi");
121-
}
122-
123-
log.info("done processing profile '{}'", header.getId());
124-
});
125-
126-
storage.saveReport(allProfileReport, CurationEntityType.PROFILE, false);
105+
106+
107+
try {
108+
pphService.getProfileHeaders().forEach(header -> {
109+
110+
log.info("start processing profile '{}'", header.getId());
111+
112+
try {
113+
CMDProfileReport profileReport = curation.processCMDProfile(header.getId());
114+
115+
allProfileReport.addReport(profileReport);
116+
117+
storage.saveReport(profileReport, CurationEntityType.PROFILE, false);
118+
119+
}
120+
catch (MalformedURLException e1) {
121+
122+
log.error("malformed URL for id '{}' - check the setting for curation.pph-service.restApi", header.getId());
123+
}
124+
125+
log.info("done processing profile '{}'", header.getId());
126+
});
127+
128+
storage.saveReport(allProfileReport, CurationEntityType.PROFILE, false);
129+
}
130+
catch (PPHServiceNotAvailableException e) {
131+
throw new RuntimeException(e);
132+
}
127133

128134
}
129135

@@ -161,19 +167,19 @@ private void processCollection(Path path, AllCollectionReport allCollectionRepor
161167

162168
log.info("start processing collection from path '{}'", path);
163169

164-
CollectionReport colectionReport = curation.processCollection(path);
170+
CollectionReport collectionReport = curation.processCollection(path);
165171

166-
allCollectionReport.addReport(colectionReport);
167-
allLinkcheckerReport.addReport(colectionReport);
172+
allCollectionReport.addReport(collectionReport);
173+
allLinkcheckerReport.addReport(collectionReport);
168174

169-
storage.saveReport(colectionReport, CurationEntityType.COLLECTION, true);
175+
storage.saveReport(collectionReport, CurationEntityType.COLLECTION, true);
170176

171177
log.info("done processing collection from path '{}'", path);
172178

173179
}
174180
else {
175-
try {
176-
Files.newDirectoryStream(path).forEach(dir -> processCollection(dir, allCollectionReport, allLinkcheckerReport));
181+
try (DirectoryStream<Path> directoryStream = Files.newDirectoryStream(path)){
182+
directoryStream.forEach(dir -> processCollection(dir, allCollectionReport, allLinkcheckerReport));
177183
}
178184
catch (IOException e) {
179185
log.error("can't create directory stream for path '{}'", path);
@@ -184,8 +190,8 @@ private void processCollection(Path path, AllCollectionReport allCollectionRepor
184190

185191
private boolean isCollectionRoot(Path path) {
186192

187-
try {
188-
return Files.walk(path, 1).anyMatch(fileOrDirectory -> Files.isRegularFile(fileOrDirectory));
193+
try (Stream<Path> stream = Files.walk(path, 1)){
194+
return stream.anyMatch(Files::isRegularFile);
189195
}
190196
catch (IOException e) {
191197

‎curation-app/src/main/resources/application.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@ curation:
3131
restApi: "https://catalog.clarin.eu/ds/ComponentRegistry/rest/registry/1.x/profiles"
3232
query: "?registrySpace=published&status=production&status=development&&status=deprecated"
3333
ccr-service:
34-
restApi: "https://openskos.meertens.knaw.nl/ccr/api/find-concepts"
35-
query: "?q=status:[*%20TO%20*]&fl=status,uri,prefLabel@en&format=json&rows=5000"
34+
restApi: "https://vocabularies.clarin.eu/clavas/rest/v1/data"
35+
query: "?uri=${conceptURI}"
3636
mode: all
3737
directory:
3838
home: ${APPLICATION_HOME_DIRECTORY}

0 commit comments

Comments
 (0)