Enable NanoporeParser to validate and parse pod5 based nanopore structures (#127)

* Enable NanoporeParser to check for pod5 and dorado basecaller generated files

* Add Full data structure example

* Add JD

Co-authored-by: Tobias Koch <[email protected]>
Co-authored-by: wow-such-code <[email protected]>

Author: 3 people

pom.xml

@@ -128,7 +128,7 @@
     <dependency>
       <artifactId>data-model-lib</artifactId>
       <groupId>life.qbic</groupId>
-      <version>2.25.0</version>
+      <version>2.27.0</version>
     </dependency>
     <dependency>
       <groupId>org.mockito</groupId>

src/main/groovy/life/qbic/utils/NanoporeParser.groovy

@@ -3,6 +3,7 @@ package life.qbic.utils
 import com.fasterxml.jackson.databind.ObjectMapper
 import groovy.json.JsonSlurper
 import groovy.util.logging.Log4j2
+import life.qbic.datamodel.instruments.OxfordNanoporeInstrumentOutputDoradoMinimal
 import life.qbic.datamodel.instruments.OxfordNanoporeInstrumentOutputMinimal
 import net.jimblackler.jsonschemafriend.Schema
 import net.jimblackler.jsonschemafriend.SchemaStore
@@ -14,7 +15,6 @@ import java.nio.file.Path
 import java.nio.file.Paths
 import java.text.ParseException
 import life.qbic.datamodel.datasets.OxfordNanoporeExperiment
-
 import java.util.stream.Collectors
 
 @Log4j2
@@ -95,8 +95,8 @@ class NanoporeParser {
                 jsonStarted = true
             }
             if (jsonStarted) {
-                def split = line.replaceAll("\\s+","").split(":")
-                if(split.size() == 2 && split[1].replaceAll('"',"").size() <= 1){
+                def split = line.replaceAll("\\s+", "").split(":")
+                if (split.size() == 2 && split[1].replaceAll('"', "").size() <= 1) {
                     log.info("Metadata value ${split[0]} missing in ${reportFile["path"]}")
                 }
                 buffer.append(line)
@@ -110,12 +110,11 @@ class NanoporeParser {
         new File(Paths.get(root.toString(), summaryFile["path"].toString()) as String)
                 .readLines().each { line ->
             def split = line.split("=")
-            if(split.size() > 1){
+            if (split.size() > 1) {
                 finalMetaData[split[0]] = split[1]
-            }
-            else {
+            } else {
                 log.info("Metadata value ${split[0]} missing in ${summaryFile["path"]}, defaulting to empty value")
-                finalMetaData[split[0]] =  ""
+                finalMetaData[split[0]] = ""
             }
         }
         return finalMetaData
@@ -178,18 +177,27 @@ class NanoporeParser {
      * @throws net.jimblackler.jsonschemafriend.ValidationException
      */
     private static void validateJson(String json) throws ValidationException {
-        // Step 1: load schema
+        // Step 1: load json
         ObjectMapper objectMapper = new ObjectMapper()
         Object jsonObject = objectMapper.readValue(json, Object)
+
         SchemaStore schemaStore = new SchemaStore()
-        Schema schema = schemaStore.loadSchema(OxfordNanoporeInstrumentOutputMinimal.getSchemaAsStream())
         Validator validator = new Validator()
-        validator.validate(schema, jsonObject)
+        try {
+            //Validate against Fast5 Based Oxford Measurement
+            Schema schema = schemaStore.loadSchema(OxfordNanoporeInstrumentOutputMinimal.getSchemaAsStream())
+            validator.validate(schema, jsonObject)
+        } catch (ValidationException ignored) {
+            //Validate against Pod5 Based Oxford Measurement
+            Schema schema = schemaStore.loadSchema(OxfordNanoporeInstrumentOutputDoradoMinimal.getSchemaAsStream())
+            validator.validate(schema, jsonObject)
+        }
     }
 
     /*
      * Converts a file tree into a json object.
      */
+
     private static class DirectoryConverter {
         private static final PREDEFINED_EXTENSIONS = ["fastq.gz"]
         private static final IGNORED_FOLDERNAMES = ["qc"]
@@ -239,11 +247,11 @@ class NanoporeParser {
             List<File> children = currentDirectory.listFiles()
 
             List<File> visibleChildren = children.stream()
-                    .filter(file -> !file.isHidden()).collect(Collectors.toList());
+                    .filter(file -> !file.isHidden()).collect(Collectors.toList())
 
             for (File file : children) {
                 if (!visibleChildren.contains(file)) {
-                    hiddenFiles.add(file);
+                    hiddenFiles.add(file)
                 }
             }
 
@@ -252,11 +260,11 @@ class NanoporeParser {
                 return !IGNORED_FOLDERNAMES.contains(currentFolderName)
             }.collect {
                 file ->
-                        if (file.isFile()) {
-                            convertFile(file.toPath())
-                        } else if (file.isDirectory()) {
-                            convertDirectory(file.toPath())
-                        }
+                    if (file.isFile()) {
+                        convertFile(file.toPath())
+                    } else if (file.isDirectory()) {
+                        convertDirectory(file.toPath())
+                    }
             }
 
             def convertedDirectory = [

src/test/groovy/life/qbic/utils/NanoporeParserSpec.groovy

@@ -125,6 +125,41 @@ class NanoporeParserSpec extends Specification {
     thrown(ValidationException)
   }
 
+  def "parsing a valid minimal file structure for dorado based basecalling containing additional unknown files and folder still returns an OxfordNanoporeExperiment Object"() {
+    given:
+    def pathToDirectory = Paths.get(exampleDirectoriesRoot, "validates/QABCD001AB_E12A345a01_PAE12345_nanopore_valid_dorado_minimal")
+    when:
+    def experiment = NanoporeParser.parseFileStructure(pathToDirectory)
+    then:
+    assert experiment instanceof OxfordNanoporeExperiment
+    // Check that the metadata from the report file has been retrieved
+    assert experiment.getMeasurements().get(0).getMachineHost() == "PCT0094"
+    // Check that the metadata from the summary file has been retrieved
+    assert experiment.getMeasurements().get(0).getLibraryPreparationKit() == "SQK-LSK109-XL"
+  }
+
+  def "parsing a valid file structure for dorado based basecalling containing additional unknown files and folder still returns an OxfordNanoporeExperiment Object"() {
+    given:
+    def pathToDirectory = Paths.get(exampleDirectoriesRoot, "validates/QABCD001AB_E12A345a01_PAE12345_nanopore_valid_dorado_example")
+    when:
+    def experiment = NanoporeParser.parseFileStructure(pathToDirectory)
+    then:
+    assert experiment instanceof OxfordNanoporeExperiment
+    // Check that the metadata from the report file has been retrieved
+    assert experiment.getMeasurements().get(0).getMachineHost() == "PCT0094"
+    // Check that the metadata from the summary file has been retrieved
+    assert experiment.getMeasurements().get(0).getLibraryPreparationKit() == "SQK-LSK109-XL"
+  }
+
+  def "parsing an invalid minimal file structure for dorado based basecalling leads to a ValidationException"() {
+    given:
+    def pathToDirectory = Paths.get(exampleDirectoriesRoot, "fails/QABCD001AB_E12A345a01_PAE12345_missing_skip_folder")
+    when:
+    def experiment = NanoporeParser.parseFileStructure(pathToDirectory)
+    then:
+    thrown(ValidationException)
+  }
+
   def "parsing the alternative valid file structure with metadata missing returns an OxfordNanoporeExperiment Object"() {
     given:
     def pathToDirectory = Paths.get(exampleDirectoriesRoot, "validates/QABCD001AB_E12A345a01_PAE12345_nanopore_new_minimal")

src/test/resources/dummyFileSystem/nanopore-instrument-output/fails/QABCD001AB_E12A345a01_PAE12345_missing_skip_folder/20200122_1217_1-A1-B1-PAE12345_1234567a/barcode_alignment_.tsv

@@ -0,0 +1 @@
+This is some text

src/test/resources/dummyFileSystem/nanopore-instrument-output/validates/QABCD001AB_E12A345a01_PAE12345_nanopore_valid_minimal/20200122_1217_1-A1-B1-PAE12345_1234567a/fastq_fail/myfile.fastq.gz renamed to src/test/resources/dummyFileSystem/nanopore-instrument-output/fails/QABCD001AB_E12A345a01_PAE12345_missing_skip_folder/20200122_1217_1-A1-B1-PAE12345_1234567a/basecalling/fastq_fail/myfile.fastq.gz

@@ -0,0 +1 @@
+This is some text

src/test/resources/dummyFileSystem/nanopore-instrument-output/fails/QABCD001AB_E12A345a01_PAE12345_missing_skip_folder/20200122_1217_1-A1-B1-PAE12345_1234567a/basecalling/fastq_fail/myfile2.fastq.gz

@@ -0,0 +1 @@
+This is some text

src/test/resources/dummyFileSystem/nanopore-instrument-output/fails/QABCD001AB_E12A345a01_PAE12345_missing_skip_folder/20200122_1217_1-A1-B1-PAE12345_1234567a/basecalling/fastq_fail/myfile3.fastq.gz

@@ -0,0 +1 @@
+This is some text