Merge pull request #42 from clarin-eric/vaadin_replacement

Vaadin replacement

Author: Wolfgang Walter SAUER

curation-module-core/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<artifactId>curation-module</artifactId>
 		<groupId>eu.clarin.cmdi</groupId>
-		<version>2.4</version>
+		<version>3.0</version>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>
     <artifactId>curation-module-core</artifactId>
@@ -93,7 +93,7 @@
 		<dependency>
 			<groupId>eu.clarin.cmdi</groupId>
 			<artifactId>linkChecker</artifactId>
-			<version>2.4</version>
+			<version>3.0</version>
 		</dependency>
 		<!--<dependency> -->
 		<!--<groupId>com.github.mfornos</groupId> -->
@@ -143,5 +143,5 @@
             </plugin>
         </plugins>
     </build>
-	<version>2.4</version>
+	<version>3.0</version>
 </project>

curation-module-core/src/main/java/eu/clarin/cmdi/curation/cr/CRService.java

@@ -1,5 +1,10 @@
 package eu.clarin.cmdi.curation.cr;
 
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.CharBuffer;
 import java.util.Collection;
 import java.util.concurrent.ExecutionException;
 import java.util.regex.Matcher;
@@ -60,11 +65,35 @@ public ProfileHeader createProfileHeader(String schemaLocation, String cmdiVersi
 
 		if(header == null){
 			header = new ProfileHeader();
-			header.setId(getIdFromSchemaLocation(schemaLocation));
 			header.setSchemaLocation(schemaLocation);
-			header.setCmdiVersion(cmdiVersion);
-			header.setPublic(false);
-
+			header.setId(getIdFromSchemaLocation(schemaLocation));
+	        header.setCmdiVersion(cmdiVersion);
+	        header.setPublic(false);
+	        
+			if(header.getId() == null) { // when the id can't be extracted from the schema location we have to get it from the file content
+		        CharBuffer buffer = CharBuffer.allocate(1000);
+		        
+		        InputStreamReader reader;
+                try {
+                    reader = new InputStreamReader(new URL(schemaLocation).openStream());
+                    reader.read(buffer);
+                    String content = buffer.rewind().toString();
+                    
+                    Matcher matcher = PROFILE_ID_PATTERN.matcher(content);
+                    
+                    if(matcher.find())
+                        header.setId(matcher.group());
+                    
+                    if(!content.contains("http://www.clarin.eu/cmd/1"))
+                        header.setCmdiVersion("1.1");
+                }
+                catch (MalformedURLException ex) {
+                    _logger.error("schema location "  + schemaLocation + " is no valid URL", ex);
+                }
+                catch (IOException ex) {
+                    _logger.error("couldn't read from schema location " + schemaLocation, ex);
+                }  
+			}
 		}
 		header.setLocalFile(isLocalFile);		
 		return header;		
@@ -117,5 +146,4 @@ public String getIdFromSchemaLocation(String schemaLocation) {
 
 	    return matcher.find()? matcher.group():null;
 	}
-
 }

curation-module-core/src/main/java/eu/clarin/cmdi/curation/cr/ProfileCacheFactory.java

@@ -21,8 +21,12 @@
 import javax.xml.validation.SchemaFactory;
 import java.io.File;
 import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
 import java.util.concurrent.TimeUnit;
 
 class ProfileCacheFactory {
@@ -74,7 +78,7 @@ public ProfileCacheLoader(boolean isPublicProfilesCache) {
 
 
         @Override
-        public ProfileCacheEntry load(ProfileHeader header) throws IOException, VTDException, SAXException {
+        public ProfileCacheEntry load(ProfileHeader header) throws IOException, VTDException, SAXException, URISyntaxException {
 
 
             _logger.info("Profile {} is not in the cache, it will be loaded", header.getId());
@@ -103,6 +107,7 @@ public ProfileCacheEntry load(ProfileHeader header) throws IOException, VTDExcep
                 }
 
             } else {//non-public profiles are not cached on disk
+                
                 _logger.debug("schema {} is not public. Schema will be downloaded in temp folder", header.getId());
 
 
@@ -125,8 +130,13 @@ public ProfileCacheEntry load(ProfileHeader header) throws IOException, VTDExcep
 
 
                     _logger.info("XSD for the {} is not in the local cache, it will be downloaded", header.getId());
-                    new HTTPLinkChecker(15000, 5, Configuration.USERAGENT).download(header.getSchemaLocation(), xsd.toFile());
-
+                    
+                    if(header.getSchemaLocation().startsWith("file:")) {
+                        Files.move(Paths.get(new URI(header.getSchemaLocation())), xsd, StandardCopyOption.REPLACE_EXISTING);
+                    }
+                    else {
+                        new HTTPLinkChecker(15000, 5, Configuration.USERAGENT).download(header.getSchemaLocation(), xsd.toFile());
+                    }
                 }
             }
 

curation-module-core/src/main/java/eu/clarin/cmdi/curation/cr/ProfileHeader.java

@@ -16,7 +16,7 @@ public class ProfileHeader {
 
 	private String id;
 	private String schemaLocation;
-	private String url;
+//	private String url;
 	private String name;
 	private String description;
 	private String cmdiVersion;
@@ -122,11 +122,9 @@ public String toString() {
 				.toString();
 	}
 
-    public String getUrl() {
-        return url;
-    }
-
-    public void setUrl(String url) {
-        this.url = url;
-    }
+    /*
+     * public String getUrl() { return url; }
+     * 
+     * public void setUrl(String url) { this.url = url; }
+     */
 }

curation-module-core/src/main/java/eu/clarin/cmdi/curation/entities/CMDCollection.java

@@ -1,23 +1,24 @@
 package eu.clarin.cmdi.curation.entities;
 
 import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.ArrayDeque;
+import java.util.Deque;
+
 
 import eu.clarin.cmdi.curation.processor.AbstractProcessor;
 import eu.clarin.cmdi.curation.processor.CollectionProcessor;
 
 public class CMDCollection extends CurationEntity {
 
-	List<CurationEntity> children;
+	Deque<CurationEntity> children;
 
 	long numOfFiles;
 	long maxFileSize = 0;
 	long minFileSize = Long.MAX_VALUE;
 
 	public CMDCollection(Path path) {
 		super(path);
-		children = new ArrayList<CurationEntity>();
+		children = new ArrayDeque<CurationEntity>();
 	}
 
 	@Override
@@ -53,7 +54,7 @@ private void aggregateWithDir(CMDCollection child) {
 			minFileSize = child.minFileSize;
 	}
 
-	public List<CurationEntity> getChildren() {
+	public Deque<CurationEntity> getChildren() {
 		return children;
 	}
 

curation-module-core/src/main/java/eu/clarin/cmdi/curation/entities/CMDProfile.java

@@ -1,5 +1,7 @@
 package eu.clarin.cmdi.curation.entities;
 
+import java.nio.file.Path;
+
 import eu.clarin.cmdi.curation.processor.AbstractProcessor;
 import eu.clarin.cmdi.curation.processor.CMDProfileProcessor;
 
@@ -13,6 +15,10 @@ public class CMDProfile extends CurationEntity {
 
 	private String schemaLocation;
 	private String cmdiVersion;
+	
+	public CMDProfile(Path path) {
+	    super(path);
+	}
 
 	public CMDProfile(String schemaLocation, String cmdiVersion) {
 		super(null);

curation-module-core/src/main/java/eu/clarin/cmdi/curation/instance_parser/InstanceParser.java

@@ -31,7 +31,7 @@ public class InstanceParser {
 
 	static{
 		TransformerFactory factory = TransformerFactory.newInstance();
-		Source xslt = new StreamSource(InstanceParser.class.getResourceAsStream("/instanceTransformer.xsl"));		
+		Source xslt = new StreamSource(InstanceParser.class.getResourceAsStream("/xslt/instanceTransformer.xsl"));		
 		try {
 			tranformer = factory.newTransformer(xslt);
 		} catch (TransformerConfigurationException e) {

curation-module-core/src/main/java/eu/clarin/cmdi/curation/main/Configuration.java

@@ -30,10 +30,11 @@ public class Configuration {
     public static Path OUTPUT_DIRECTORY = null;
     public static Path CACHE_DIRECTORY = null;
     public static Path COLLECTION_HTML_DIRECTORY = null;
+    public static int THREAD_POOL_SIZE=100;
     public static Collection<String> FACETS = null;
     public static int REDIRECT_FOLLOW_LIMIT;
     public static int TIMEOUT;
-    private static final int TIMEOUTDEFAULT = 5000;//in ms(if config file doesnt have it)
+    private static int TIMEOUTDEFAULT = 5000;//in ms(if config file doesnt have it)
 
     public static VloConfig VLO_CONFIG;
     public static boolean DATABASE;
@@ -78,7 +79,7 @@ private static void readProperties(Properties config) throws IOException {
         } else {
             TIMEOUT = Integer.parseInt(timeout);
         }
-
+        THREAD_POOL_SIZE = Integer.valueOf(config.getProperty("THREAD_POOL_SIZE","100"));
 
         String[] facets = config.getProperty("FACETS").split(",");
         FACETS = Arrays.asList(facets).stream().map(f -> f.trim()).collect(Collectors.toList());

curation-module-core/src/main/java/eu/clarin/cmdi/curation/main/CurationModule.java

@@ -10,6 +10,7 @@
 import eu.clarin.curation.linkchecker.httpLinkChecker.HTTPLinkChecker;
 
 import java.io.IOException;
+import java.net.MalformedURLException;
 import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -30,9 +31,14 @@ public Report<?> processCMDProfile(URL schemaLocation) {
 
         return new CMDProfile(schemaLocation.toString(), "1.x").generateReport(null);
     }
+    
+    @Override    
+    public Report<?> processCMDProfile(Path path) throws MalformedURLException {
 
-    @Override
+            return processCMDProfile(path.toUri().toURL());
+    }
 
+    @Override
     public Report<?> processCMDInstance(Path path) throws IOException {
         if (Files.notExists(path))
             throw new IOException(path.toString() + " doesn't exist!");
@@ -45,10 +51,10 @@ public Report<?> processCMDInstance(Path path) throws IOException {
     @Override
     public Report<?> processCMDInstance(URL url) throws IOException {
         String path = FileNameEncoder.encode(url.toString()) + ".xml";
-        Path cmdiFile = Paths.get(System.getProperty("java.io.tmpdir"), path);
-        new HTTPLinkChecker(15000, 5, Configuration.USERAGENT).download(url.toString(), cmdiFile.toFile());
-        long size = Files.size(cmdiFile);
-        CMDInstance cmdInstance = new CMDInstance(cmdiFile, size);
+        Path cmdiFilePath = Paths.get(System.getProperty("java.io.tmpdir"), path);
+        new HTTPLinkChecker(15000, 5, Configuration.USERAGENT).download(url.toString(), cmdiFilePath.toFile());
+        long size = Files.size(cmdiFilePath);
+        CMDInstance cmdInstance = new CMDInstance(cmdiFilePath, size);
         cmdInstance.setUrl(url.toString());
 
         Report<?> report = cmdInstance.generateReport(null);

curation-module-core/src/main/java/eu/clarin/cmdi/curation/main/CurationModuleInterface.java

@@ -1,6 +1,7 @@
 package eu.clarin.cmdi.curation.main;
 
 import java.io.IOException;
+import java.net.MalformedURLException;
 import java.net.URL;
 import java.nio.file.Path;
 import java.util.Collection;
@@ -12,9 +13,11 @@
 public interface CurationModuleInterface {
 
     public Report processCMDProfile(String profileId);
+    
+    public Report processCMDProfile(Path path) throws MalformedURLException, IOException;
 
 
-	public Report processCMDProfile(URL schemaLocation);
+	public Report processCMDProfile(URL schemaLocation) throws IOException;
 
 	/*
 	 * throws Exception if file doesn't exist or is invalid