Parse all datasets in ESRF record

Fixes #22
EGI-Federation · May 13, 2023 · 17ada09 · 17ada09
1 parent 93e042c
commit 17ada09
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 18 deletions.
diff --git a/src/main/java/eosc/eu/model/StorageElement.java b/src/main/java/eosc/eu/model/StorageElement.java
@@ -111,6 +111,9 @@ public StorageElement(EsrfDataFile ef, String baseUrl, String sessionId) {
 
         if(null != ef.Datafile.modTime)
             this.modifiedAt = ef.Datafile.modTime;
+
+        if(null != ef.Datafile.dataset && null != ef.Datafile.dataset.name)
+            this.collection = ef.Datafile.dataset.name;
     }
 
     /**

diff --git a/src/main/java/parser/b2share/B2ShareParser.java b/src/main/java/parser/b2share/B2ShareParser.java
@@ -200,6 +200,9 @@ public Uni<StorageContent> parseDOI(String auth, String doi, int level) {
                 return Uni.createFrom().failure(new TransferServiceException("noFilesLink"));
             })
             .chain(bucket -> {
+                // Got bucket content
+                log.info("Got B2Share bucket");
+
                 // Build list of source files
                 StorageContent srcFiles = new StorageContent(bucket.contents.size());
                 for(var file : bucket.contents) {

diff --git a/src/main/java/parser/esrf/EsrfParser.java b/src/main/java/parser/esrf/EsrfParser.java
@@ -1,7 +1,7 @@
 package parser.esrf;
 
-import eosc.eu.PortConfig;
 import io.smallrye.mutiny.Uni;
+import io.smallrye.mutiny.Multi;
 import io.smallrye.mutiny.tuples.Tuple2;
 import org.eclipse.microprofile.rest.client.RestClientBuilder;
 import org.eclipse.microprofile.rest.client.RestClientDefinitionException;
@@ -15,6 +15,7 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import eosc.eu.PortConfig;
 import eosc.eu.ParsersConfig.ParserConfig;
 import eosc.eu.ParserService;
 import eosc.eu.TransferServiceException;
@@ -200,34 +201,42 @@ public Uni<StorageContent> parseDOI(String auth, String doi, int level) {
 
                 sessionId.set(session.sessionId);
 
-                // Get the dataset
+                // Get the datasets
                 return parser.getDataSetsAsync(sessionId.get(), this.authority, this.recordId);
             })
-            .chain(datasets -> {
+            .onItem().transformToMulti(datasets -> {
                 // Got dataset(s)
-                log.infof("Found %d datasets at DOI %s", datasets.size(), doi);
+                MDC.put("datasetCount", datasets.size());
+                log.info("Found datasets");
 
-                // Handle the first dataset, ignore the rest
-                var dataset = datasets.get(0);
-                log.infof("First dataset has ID %s", dataset.id);
+                return Multi.createFrom().iterable(datasets);
+            })
+            .onItem().transformToUniAndConcatenate(dataset -> {
+                // Got a dataset
+                MDC.put("datasetId", dataset.id);
+                MDC.put("datasetName", dataset.name);
+                log.infof("Got dataset");
 
+                // Fetch the files in the dataset
                 return parser.getDataFilesAsync(sessionId.get(), dataset.id);
             })
-            .chain(files -> {
-                // Got the files in the dataset
+            .onFailure().invoke(e -> {
+                log.error("Failed to parse ESRF DOI");
+            })
+            .collect()
+            .in(StorageContent::new, (sc, files) -> {
+                // Got the files from a dataset
+                MDC.put("fileCount", files.size());
+                log.info("Got dataset files");
+
                 var session = sessionId.get();
-                StorageContent srcFiles = new StorageContent(files.size());
                 for(var file : files) {
-                    srcFiles.elements.add(new StorageElement(file, baseUrl, session));
+                    sc.elements.add(new StorageElement(file, baseUrl, session));
                 }
-
-                srcFiles.count = srcFiles.elements.size();
-
-                // Success
-                return Uni.createFrom().item(srcFiles);
             })
-            .onFailure().invoke(e -> {
-                log.error("Failed to parse ESRF DOI");
+            .chain(sc -> {
+                sc.count = sc.elements.size();
+                return Uni.createFrom().item(sc);
             });
 
         return result;

diff --git a/src/main/java/parser/esrf/model/EsrfDataFile.java b/src/main/java/parser/esrf/model/EsrfDataFile.java
@@ -38,5 +38,6 @@ public class Info {
         public Date modTime;
         public long fileSize;
         public String location;
+        public EsrfDataSet dataset;
     }
 }