Merge pull request #156 from clarin-eric/6.0.0-rc1

6.0.0 rc1

Author: wowasa

CHANGES.md

@@ -1,3 +1,10 @@
+# 6.0.0
+- converting the project to Spring boot project
+- using Spring data JPA for persistence (see [linkchecker-persistence](https://github.com/clarin-eric/linkchecker-persistence), Spring caching and processor Beans)
+- modularization of the project (app, web, shared api and services)
+- reorganization of processors, sub-processors and reports 
+- separation of business-logic (in processors and sub-processors) and presentation layer (report instances)   
+
 # 5.4.x
 - adding deactivation and deletion of non confirmed links
 - externalising help and faq page as markdown

curation-api/pom.xml

@@ -2,7 +2,7 @@
    <parent>
       <groupId>eu.clarin.cmdi</groupId>
       <artifactId>curation-dashboard</artifactId>
-      <version>6.0.0-SNAPSHOT</version>
+      <version>${global.version}</version>
    </parent>
    <modelVersion>4.0.0</modelVersion>
    <artifactId>curation-api</artifactId>
@@ -60,7 +60,7 @@
       <dependency>
          <groupId>eu.clarin.cmdi</groupId>
          <artifactId>vlo-importer</artifactId>
-         <version>4.11.2</version>
+         <version>${vlo.version}</version>
          <exclusions>
             <exclusion>
                <groupId>net.sf.saxon</groupId>
@@ -90,7 +90,7 @@
       <dependency>
          <groupId>eu.clarin.cmdi</groupId>
          <artifactId>curation-cr</artifactId>
-         <version>${revision}</version>
+         <version>${global.version}</version>
       </dependency>
       <dependency>
          <groupId>com.h2database</groupId>

curation-api/src/main/java/eu/clarin/cmdi/curation/api/CurationModuleImpl.java

@@ -141,15 +141,31 @@ public CollectionReport processCollection(Path path) {
    public Collection<LinkcheckerDetailReport> getLinkcheckerDetailReports() {
 
       final Collection<LinkcheckerDetailReport> linkcheckerDetailReports = new ArrayList<LinkcheckerDetailReport>();
+      final LinkcheckerDetailReport overallReport = new LinkcheckerDetailReport("Overall");
+      linkcheckerDetailReports.add(overallReport);
 
       final LinkcheckerDetailReport[] lastLinkcheckerDetailReport = new LinkcheckerDetailReport[1];
       final CategoryReport[] lastCategoryReport = new CategoryReport[1];
 
 
-      try(Stream<StatusDetail> stream = sdRep.findByOrderNrLessThanEqual(100l)){
+      try(Stream<StatusDetail> stream = sdRep.findByOrderNrLessThanEqual(50l)){
 
          stream.forEach(statusDetail -> {
 
+            StatusDetailReport statusDetailReport =  new StatusDetailReport(
+                  statusDetail.getUrlname(),
+                  statusDetail.getOrigin(),
+                  statusDetail.getMethod(), 
+                  statusDetail.getStatusCode(),
+                  statusDetail.getMessage(), 
+                  statusDetail.getCheckingDate(), 
+                  statusDetail.getContentType(), 
+                  statusDetail.getExpectedMimeType(),
+                  statusDetail.getContentLength(), 
+                  statusDetail.getDuration(), 
+                  statusDetail.getRedirectCount()
+               );
+            
             if(lastLinkcheckerDetailReport[0] == null || !lastLinkcheckerDetailReport[0].getName().equals(statusDetail.getProvidergroupname())) {
 
                lastLinkcheckerDetailReport[0] = new LinkcheckerDetailReport(statusDetail.getProvidergroupname());
@@ -165,26 +181,25 @@ public Collection<LinkcheckerDetailReport> getLinkcheckerDetailReports() {
 
                lastLinkcheckerDetailReport[0].getCategoryReports().add(lastCategoryReport[0]);
 
+               overallReport.getCategoryReports()
+                  .stream()
+                  .filter(report -> report.getCategory() == statusDetail.getCategory())
+                  .findFirst()
+                  .orElseGet(() -> {
+                     CategoryReport categoryReport = new CategoryReport(statusDetail.getCategory());
+                     overallReport.getCategoryReports().add(categoryReport);
+                     
+                     return categoryReport;
+                  })
+                  .getStatusDetails()
+                  .add(statusDetailReport);
+
+              
             }
 
-            lastCategoryReport[0].getStatusDetails().add(
-                  new StatusDetailReport(
-                     statusDetail.getUrlname(),
-                     statusDetail.getOrigin(),
-                     statusDetail.getMethod(), 
-                     statusDetail.getStatusCode(),
-                     statusDetail.getMessage(), 
-                     statusDetail.getCheckingDate(), 
-                     statusDetail.getContentType(), 
-                     statusDetail.getExpectedMimeType(),
-                     statusDetail.getContentLength(), 
-                     statusDetail.getDuration(), 
-                     statusDetail.getRedirectCount()
-                  )
-               );
+            lastCategoryReport[0].getStatusDetails().add(statusDetailReport);      
          });         
-      }
-      
+      }      
 
       return linkcheckerDetailReports;
 

curation-api/src/main/java/eu/clarin/cmdi/curation/api/conf/ApiConfig.java

@@ -32,7 +32,7 @@ public class ApiConfig {
 
    private Directory directory = new Directory();
 
-   private int threadPoolSize;
+   private int threadpoolSize;
 
    private String vloConfigLocation;
 

curation-api/src/main/java/eu/clarin/cmdi/curation/api/processor/CMDInstanceProcessor.java

@@ -54,7 +54,7 @@ public CMDInstanceReport process(CMDInstance instance){
 
       this.subprocessors
          .stream()
-         .takeWhile(p -> instanceReport.isValidReport)
+         .takeWhile(p -> instanceReport.isProcessable)
          .forEach(subprocessor -> subprocessor.process(instance, instanceReport));
 
 

curation-api/src/main/java/eu/clarin/cmdi/curation/api/report/collection/AllCollectionReport.java

@@ -67,7 +67,7 @@ public long getNumOfFiles() {
       }
       @XmlElement
       public int getNumOfProfiles() {
-         return collectionReport.headerReport.totNumOfProfiles;
+         return collectionReport.profileReport.totNumOfProfiles;
       }
       @XmlElement
       public int getNumOfLinks() {

curation-api/src/main/java/eu/clarin/cmdi/curation/api/report/collection/CollectionReport.java

@@ -7,6 +7,7 @@
 import eu.clarin.cmdi.curation.api.report.collection.sec.FileReport;
 import eu.clarin.cmdi.curation.api.report.collection.sec.HeaderReport;
 import eu.clarin.cmdi.curation.api.report.collection.sec.LinkcheckerReport;
+import eu.clarin.cmdi.curation.api.report.collection.sec.ProfileReport;
 import eu.clarin.cmdi.curation.api.report.collection.sec.ResProxyReport;
 import eu.clarin.cmdi.curation.api.report.collection.sec.XmlPopulationReport;
 import eu.clarin.cmdi.curation.api.report.collection.sec.XmlValidityReport;
@@ -37,17 +38,13 @@ public class CollectionReport implements NamedReport{
    @XmlAttribute
    public double aggregatedMaxScore;
    @XmlAttribute
-   public double aggregatedMaxScoreValid;
-   @XmlAttribute
    public double scorePercentage;
    @XmlAttribute
-   public double avgScore;
-   @XmlAttribute
-   public double avgScoreValid;   
+   public double avgScore; 
    @XmlAttribute
    public double minScore = Double.MAX_VALUE;
    @XmlAttribute
-   public double maxScore = Double.MAX_VALUE;
+   public double maxScore;
    @XmlAttribute
    public double insMaxScore = eu.clarin.cmdi.curation.api.report.instance.CMDInstanceReport.maxScore +1;
 
@@ -57,6 +54,9 @@ public class CollectionReport implements NamedReport{
 
    @XmlElement
    public final FileReport fileReport = new FileReport();
+   
+   @XmlElement
+   public final ProfileReport profileReport = new ProfileReport();
 
    @XmlElement
    public final HeaderReport headerReport = new HeaderReport();

curation-api/src/main/java/eu/clarin/cmdi/curation/api/report/collection/sec/FacetReport.java

@@ -29,11 +29,9 @@ public class FacetReport {
    @XmlAttribute
    public double aggregatedMaxScore;
    @XmlAttribute
-   public double aggregatedMaxScoreValid;
+   public double scorePercentage;
    @XmlAttribute
    public double avgScore;
-   @XmlAttribute
-   public double avgScoreValid;  
    @XmlElement
    public double percCoverageNonZero;
    @XmlElementWrapper(name = "facets")

curation-api/src/main/java/eu/clarin/cmdi/curation/api/report/collection/sec/FileReport.java

@@ -22,17 +22,17 @@ public class FileReport {
    @XmlAttribute
    public double aggregatedMaxScore;
    @XmlAttribute
-   public double aggregatedMaxScoreValid;
+   public double scorePercentage;
    @XmlAttribute
    public double avgScore;
-   @XmlAttribute
-   public double avgScoreValid; 
    @XmlElement
    public String provider;
    @XmlElement
    public long numOfFiles;
    @XmlElement
-   public long numOfValidFiles;
+   public long numOfFilesProcessable;
+   @XmlElement
+   public long numOfFilesNonProcessable;
    @XmlElement
    public long size;
    @XmlElement

curation-api/src/main/java/eu/clarin/cmdi/curation/api/report/collection/sec/HeaderReport.java

@@ -4,7 +4,6 @@
  */
 package eu.clarin.cmdi.curation.api.report.collection.sec;
 
-import java.util.ArrayList;
 import java.util.Collection;
 
 import javax.xml.bind.annotation.XmlAccessType;
@@ -14,10 +13,6 @@
 import javax.xml.bind.annotation.XmlElementWrapper;
 import javax.xml.bind.annotation.XmlRootElement;
 
-import lombok.NoArgsConstructor;
-import lombok.RequiredArgsConstructor;
-
-
 
 /**
  *
@@ -30,32 +25,23 @@ public class HeaderReport {
    @XmlAttribute
    public double aggregatedMaxScore;
    @XmlAttribute
-   public double aggregatedMaxScoreValid;
+   public double scorePercentage;
    @XmlAttribute
    public double avgScore;
-   @XmlAttribute
-   public double avgScoreValid;  
+   
    @XmlElement
-   public int totNumOfProfiles;
+   public long numWithSchemaLocation;
+   @XmlElement
+   public long numSchemaCRResident;
+   @XmlElement
+   public long numWithMdProfile;
+   @XmlElement
+   public long numWithMdSelflink;
+   @XmlElement   
+   public long numWithMdCollectionDisplayName;
 
    @XmlElementWrapper(name = "duplicatedMDSelfLinks")
    @XmlElement(name = "duplicatedMDSelfLink")
    public Collection<String> duplicatedMDSelfLink;
-   @XmlElementWrapper(name = "profiles")
-   @XmlElement(name = "profile")
-   public Collection<Profile> profiles = new ArrayList<Profile>();  
-
 
-   @XmlRootElement
-   @XmlAccessorType(XmlAccessType.FIELD)
-   @RequiredArgsConstructor
-   @NoArgsConstructor(force = true)
-   public static class Profile {
-      @XmlAttribute
-      public final String profileId;
-      @XmlAttribute
-      public final double score;
-      @XmlAttribute
-      public int count = 1;
-   }
 }