feat (add logical source): improve code structure

.github/gradle.yml → .github/workflows/gradle.yml

@@ -8,7 +8,14 @@ jobs:
     steps:
       - name: Checkout sources
         uses: actions/checkout@v4
+      - name: Setup Java
+        uses: actions/setup-java@v4
+        with:
+          java-version: '17'
+          distribution: 'temurin'
       - name: Setup Gradle
         uses: gradle/actions/setup-gradle@v3
       - name: Build with Gradle
         run: ./gradlew build
+      - name: Run tests with Gradle
+        run: ./gradlew test

src/main/java/fi/csc/mscr/tranformation/rml/RMLGenerator.java

@@ -13,6 +13,7 @@
 import org.topbraid.shacl.vocabulary.SH;
 
 import java.io.InputStream;
+import java.util.Optional;
 
 //import fi.vm.yti.datamodel.api.v2.dto.MSCR;
 
@@ -25,26 +26,97 @@ public class RMLGenerator {
 	String nsFNML = "http://semweb.mmlab.be/ns/fnml#";
 	String nsGREL = "http://users.ugent.be/~bjdmeest/function/grel.ttl#";
 
+	enum ReferenceFormulation {
+		JSONPath,
+		XPath,
+		CSV
+	}
+
 	public String testMe() {
 		return "test";
 	}
 
-	private String getSourceIteratorForTargetShape(Model model, String targetShapeUri) throws Exception {
+	private Optional<String> getSubjectMapTemplate(Model model, String mappingIri) {
+		String q = String.format("""
+				PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>				
+				PREFIX : <http://uri.suomi.fi/datamodel/ns/mscr#>				
+
+				select ?template
+				where {
+				    ?mappingURI a :Mapping .
+				    ?mappingURI :target/rdf:_1 ?target .
+				    ?target :uri <subject:https://shacl-play.sparna.fr/shapes/Person> .
+
+				    ?mappingURI :processing ?proc .
+				    ?proc ?p ?o .
+				    ?o rdf:_2 ?oo. # TODO: is this general?
+				    ?oo :value ?template
+
+				}
+
+				""", mappingIri);
+
+		QueryExecution qe = QueryExecutionFactory.create(q, model);
+		ResultSet results = qe.execSelect();
+
+		if (results.hasNext()) {
+			QuerySolution res = results.next();
+			return Optional.of(res.getLiteral("template").toString());
+		} else {
+			return Optional.empty();
+		}
+
+	}
+
+	private Optional<ReferenceFormulation> getSourceSchema(Model model, String crosswalkIri) {
+
+		String q = String.format("""
+				PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>				
+				PREFIX : <http://uri.suomi.fi/datamodel/ns/mscr#>	
+
+				select ?format where {
+				    <%s> a :Crosswalk ;
+				     :sourceSchema ?sourceIri .
+
+				    ?sourceIri :format ?format	   
+				}		
+				""", crosswalkIri);
+
+		QueryExecution qe = QueryExecutionFactory.create(q, model);
+		ResultSet results = qe.execSelect();
+
+		if (results.hasNext()) {
+			QuerySolution res = results.next();
+			var ref = res.getLiteral("format");
+
+			switch (ref.toString()) {
+				case "JSONSCHEMA": return Optional.of(ReferenceFormulation.JSONPath);
+				case "XSD": return Optional.of(ReferenceFormulation.XPath);
+				case "CSV": return Optional.of(ReferenceFormulation.CSV);
+
+				default: throw new Error("No matching reference formulation found for " + ref);
+			}
+		} else {
+			return Optional.empty();
+		}
+
+	}
+
+	private Optional<String> getSourceIteratorForTargetShape(Model model, String targetShapeUri) {
 
 		String q = String.format("""
 				PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>				
 				PREFIX : <http://uri.suomi.fi/datamodel/ns/mscr#>				
 				select ?iterator
 				where {
-				  ?mappingURI a <http://uri.suomi.fi/datamodel/ns/mscr#Mapping> .
+				  ?mappingURI a :Mapping .
 				  ?mappingURI :target/rdf:_1 ?target .
 				  ?target :uri <%s> .
 
 				  ?mappingURI :source/rdf:_1 ?source .
 				  ?source :uri ?sourceShapeId .
 
-				  ?sourceShapeId <http://uri.suomi.fi/datamodel/ns/mscr#instancePath> ?iterator .
-
+				  ?sourceShapeId :instancePath ?iterator .
 				}						
 				""", targetShapeUri);
 
@@ -54,14 +126,67 @@ private String getSourceIteratorForTargetShape(Model model, String targetShapeUr
 
 		if (results.hasNext()) {
 			QuerySolution res = results.next();
-			return res.getLiteral("iterator").toString();
+			return Optional.of(res.getLiteral("iterator").toString());
 		} else {
-			throw new Exception("Source iterator could not be found for " + targetShapeUri);
+			return Optional.empty();
+		}
+
+	}
+
+	private Model getPredicatesForShape(Model model, String targetShapeUri) {
+
+		String q = String.format("""
+				PREFIX sh: <http://www.w3.org/ns/shacl#>
+				PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>				
+				PREFIX : <http://uri.suomi.fi/datamodel/ns/mscr#>				
+
+				select ?sourceShapeUri ?schemaPath ?predicate
+				where {
+				    BIND(IRI("%s") as ?nodeShape)
+
+				    ?mappingURI a :Mapping .
+				    ?mappingURI :target ?targetSeq .
+				    ?targetSeq ?p ?target .
+				    ?target :uri ?propShape .
+
+				    ?nodeShape sh:property ?propShape  .
+				    ?propShape sh:path ?predicate .
+
+				    ?mappingURI :source ?seqBlankNode .
+				    ?seqBlankNode ?pp ?sourceBlankNode .
+
+				    ?sourceBlankNode :uri ?sourceShapeUri .
+
+					?sourceShapeUri :schemaPath ?schemaPath
+				}
+				""", targetShapeUri);
+
+		QueryExecution qe = QueryExecutionFactory.create(q, model);
+		ResultSet results = qe.execSelect();
+
+		Model outputModel = ModelFactory.createDefaultModel();
+
+		while (results.hasNext()) {
+			QuerySolution res = results.next();
+
+			Resource predObjMap = outputModel.createResource();
+			Resource objectMap = outputModel.createResource();
+
+			Resource predicate = res.get("predicate").asResource();
+			Literal reference = res.get("schemaPath").asLiteral();
+			predObjMap.addProperty(outputModel.createProperty(nsRR + "predicate"), predicate);
+			predObjMap.addProperty(outputModel.createProperty(nsRR + "objectMap"), objectMap);
+			objectMap.addProperty(outputModel.createProperty(nsRML + "reference"), reference);
+
 		}
 
+		return outputModel;
+
 	}
 
-	public Resource addLogicalSource(String logicalSourceURI, Model m) throws Exception {
+	public Resource addLogicalSource(String logicalSourceURI, Model inputModel, String targetShapeUri, String crosswalkIri) throws Exception {
+
+		Model outputModel = ModelFactory.createDefaultModel();
 
 		/*
 		You can now get the iterator source for a specific shape by querying along the lines:
@@ -74,25 +199,54 @@ public Resource addLogicalSource(String logicalSourceURI, Model m) throws Except
 		 */
 
 		String iterator = "";
-		Resource logicalSource = m.createResource(logicalSourceURI);
+		ReferenceFormulation refFormulation = null;
+		Resource logicalSource = outputModel.createResource(logicalSourceURI);
+
 		try {
-			iterator = getSourceIteratorForTargetShape(m, "iterator:https://shacl-play.sparna.fr/shapes/Person");
+			var ref = this.getSourceSchema(inputModel, crosswalkIri);
+			refFormulation = ref.orElseThrow(Exception::new);
 		} catch(Exception e) {
-			System.out.println(e.toString());
+			System.out.println("Format could not be found");
 			throw e;
 		}
+
+		logicalSource.addProperty(RDF.type, outputModel.createResource(nsRML + "BaseSource"));
+		Resource referenceFormulation = outputModel.createResource(nsQL + refFormulation);
+		logicalSource.addProperty(outputModel.createProperty(nsRML + "referenceFormulation"), referenceFormulation);
+		//logicalSource.addProperty(m.createProperty(nsRML + "source"), m.createLiteral("data/person.json"));
 
-		logicalSource.addProperty(m.createProperty(nsRML + "iterator"), iterator);
-
-		/*logicalSource.addProperty(RDF.type, m.createResource(nsRML + "BaseSource"));
-		Resource referenceFormulation = m.createResource(nsQL + "JSONPath");
-		logicalSource.addProperty(m.createProperty(nsRML + "referenceFormulation"), referenceFormulation);
-		logicalSource.addProperty(m.createProperty(nsRML + "source"), m.createLiteral("data/person.json"));
+		try {
+			iterator = this.getSourceIteratorForTargetShape(inputModel, targetShapeUri).orElseThrow(Exception::new);
+		} catch(Exception e) {
+			System.out.println("Iterator could not be found");
+			throw e;
+		}
 
-		Resource sourceProperty = inputModel.getResource(sourcePropertyURI);
-		Literal iteratorPath = sourceProperty.getProperty(MSCR.instancePath).getLiteral();
-		logicalSource.addProperty(m.createProperty(nsRML + "iterator"), iteratorPath);*/
+		logicalSource.addProperty(outputModel.createProperty(nsRML + "iterator"), iterator);
 
 		return logicalSource;
 	}
+
+	public Model addSubjectMap(Model inputModel, String mappingIri) throws Exception {
+
+		Model outputModel = ModelFactory.createDefaultModel();
+
+		Resource triplesMap = outputModel.createResource("#Mapping"); // TODO: make this unique
+
+		Optional<String> template = getSubjectMapTemplate(inputModel, mappingIri);
+
+		Resource subjMap = outputModel.createResource();
+		subjMap.addProperty(outputModel.createProperty(nsRR + "template"), outputModel.createLiteral(template.orElseThrow(Exception::new)));
+		subjMap.addProperty(outputModel.createProperty(nsRR + "class"), outputModel.createResource("http://schema.org/Person")); // TODO: get this from target
+
+		triplesMap.addProperty(outputModel.createProperty(nsRR + "subjectMap"), subjMap);
+
+		return outputModel;
+	}
+
+	public Model addPredicateObjectMap(Model inputModel, String targetShapeUri) {
+
+		return getPredicatesForShape(inputModel, targetShapeUri);
+
+	}
 }

src/test/java/fi/csc/mscr/tranformation/rml/RMLGeneratorTest.java

@@ -5,43 +5,74 @@
 import org.apache.jena.rdf.model.Model;
 import org.apache.jena.rdf.model.ModelFactory;
 import org.apache.jena.rdf.model.Resource;
+import org.apache.jena.rdf.model.StmtIterator;
 import org.apache.jena.riot.RDFDataMgr;
 import org.junit.jupiter.api.Test;
+import java.util.Arrays;
+import java.util.List;
 
 class RMLGeneratorTest {
 
-	Model loadPersonTestData() {
-		Model model = ModelFactory.createDefaultModel();
-
-		String[] inputFileNames = {"src/test/resources/data/person_json2shacl/crosswalk_content.ttl",
-				"src/test/resources/data/person_json2shacl/crosswalk_metadata.ttl",
-				"src/test/resources/data/person_json2shacl/source_metadata.ttl",
-				"src/test/resources/data/person_json2shacl/source_content.ttl",
-				"src/test/resources/data/person_json2shacl/target_metadata.ttl",
-				"src/test/resources/data/person_json2shacl/target_content.ttl"
-		};
-
-		for (String inputFileName : inputFileNames) {
-			model.add(RDFDataMgr.loadModel(inputFileName));
-		}
-
-		return model;
-	}
-
-	@Test
-	void test() {
-		RMLGenerator r = new RMLGenerator();
-		assertEquals("test", r.testMe());
-	}
-
-	@Test
-	void testAddLogicalSource() throws Exception {
-		Model model = loadPersonTestData();
-		RMLGenerator r = new RMLGenerator();
-		Resource logicalSource = r.addLogicalSource("http://example.com/1", model);
-
-		assertEquals("http://example.com/1", logicalSource.getURI());
+    String[] personJsonTestData = {"src/test/resources/data/person_json2shacl/crosswalk_content.ttl",
+            "src/test/resources/data/person_json2shacl/crosswalk_metadata.ttl",
+            "src/test/resources/data/person_json2shacl/source_metadata.ttl",
+            "src/test/resources/data/person_json2shacl/source_content.ttl",
+            "src/test/resources/data/person_json2shacl/target_metadata.ttl",
+            "src/test/resources/data/person_json2shacl/target_content.ttl"
+    };
+
+    Model loadPersonTestData(List<String> inputFileNames) {
+        Model model = ModelFactory.createDefaultModel();
+
+        inputFileNames.forEach(inputFileName -> model.add(RDFDataMgr.loadModel(inputFileName)));
+
+        return model;
+    }
+
+    @Test
+    void test() {
+        RMLGenerator r = new RMLGenerator();
+        assertEquals("test", r.testMe());
+    }
+
+    @Test
+    void testAddLogicalSourceForPersonJSON() throws Exception {
+        Model model = loadPersonTestData(Arrays.asList(this.personJsonTestData));
+        RMLGenerator r = new RMLGenerator();
+        Resource logicalSource = r.addLogicalSource("http://example.com/1", model, "iterator:https://shacl-play.sparna.fr/shapes/Person", "mscr:crosswalk:653b47f8-0bad-4c0e-86e9-f4ff13b5d8e3");
+
+        /*StmtIterator props = logicalSource.listProperties();
+        props.forEach(
+                System.out::println
+        );*/
+
+        assertEquals("http://example.com/1", logicalSource.getURI());
+        assertEquals("http://semweb.mmlab.be/ns/rml#BaseSource", logicalSource.getProperty(model.createProperty("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")).getObject().toString());
 		assertEquals("$", logicalSource.getProperty(model.createProperty("http://semweb.mmlab.be/ns/rml#iterator")).getObject().asLiteral().toString());
-	}
+        assertEquals("http://semweb.mmlab.be/ns/ql#JSONPath", logicalSource.getProperty(model.createProperty("http://semweb.mmlab.be/ns/rml#referenceFormulation")).getObject().toString());
+    }
+
+    @Test
+    void testAddSubjectMap() throws Exception {
+        Model model = loadPersonTestData(Arrays.asList(this.personJsonTestData));
+        RMLGenerator r = new RMLGenerator();
+
+        Model subjMap = r.addSubjectMap(model, "subject:https://shacl-play.sparna.fr/shapes/Person");
+
+        subjMap.write(System.out, "TTL");
+
+    }
+
+    @Test
+    void testAddPredicateObjectMap() throws Exception {
+
+        Model model = loadPersonTestData(Arrays.asList(this.personJsonTestData));
+        RMLGenerator r = new RMLGenerator();
+
+        Model predObjMaps = r.addPredicateObjectMap(model,"https://shacl-play.sparna.fr/shapes/Person");
+
+        predObjMaps.write(System.out, "TTL");
+
+    }
 
 }

src/test/resources/data/person_json2shacl/crosswalk_content.ttl

@@ -53,7 +53,7 @@
                 <http://www.w3.org/2004/02/skos/core#exactMatch> ;
         <http://uri.suomi.fi/datamodel/ns/mscr#processing>
                 [ <http://uri.suomi.fi/datamodel/ns/mscr#id>
-                          "http://uri.suomi.fi/datamodel/ns/mscr#replace2Func" ;
+                          "http://uri.suomi.fi/datamodel/ns/mscr#template" ; # TODO: adapt in MSCR
                   <http://uri.suomi.fi/datamodel/ns/mscr#processingParams>
                           [ a       <http://www.w3.org/1999/02/22-rdf-syntax-ns#Bag> ;
                             <http://www.w3.org/1999/02/22-rdf-syntax-ns#_1>
@@ -64,9 +64,9 @@
                                     ] ;
                             <http://www.w3.org/1999/02/22-rdf-syntax-ns#_2>
                                     [ <http://uri.suomi.fi/datamodel/ns/mscr#key>
-                                              "pattern" ;
+                                              <http://uri.suomi.fi/datamodel/ns/mscr#fno/param/value> ; # TODO: adapt in MSCR
                                       <http://uri.suomi.fi/datamodel/ns/mscr#value>
-                                              "https://example.com/{value_0}{value_1}"
+                                              "https://example.com/{firstName}{lastName}" # TODO: adapt in MSCR
                                     ]
                           ]
                 ] ;