Merge pull request #160 from project-sunbird/validation_abstraction

Validation abstraction
Sunbird-RC · Jun 21, 2018 · c63d8a8 · c63d8a8
2 parents fbdd7a7 + f684b85
commit c63d8a8
Show file tree

Hide file tree

Showing 10 changed files with 125 additions and 32 deletions.
diff --git a/java/middleware-commons/src/main/java/io/opensaber/registry/middleware/util/RDFUtil.java b/java/middleware-commons/src/main/java/io/opensaber/registry/middleware/util/RDFUtil.java
@@ -34,7 +34,7 @@ public class RDFUtil {
     private static String registryContext;
 
 
-    public static Model getRdfModelFromJsonld(String jsonldData, String format){
+    public static Model getRdfModelBasedOnFormat(String jsonldData, String format){
     	Model m = ModelFactory.createDefaultModel();
     	StringReader reader = new StringReader(jsonldData);
     	return m.read(reader, null, format);

diff --git a/...-conversion/src/test/java/io/opensaber/registry/middleware/impl/JSONLDConversionTest.java b/...-conversion/src/test/java/io/opensaber/registry/middleware/impl/JSONLDConversionTest.java
@@ -43,7 +43,7 @@ private void setup() throws IOException, URISyntaxException{
     		String jsonLDData = Paths.get(getPath(TEACHER_JSONLD)).toString();
     		Path filePath = Paths.get(jsonLDData);
     		String jsonld = new String(Files.readAllBytes(filePath), StandardCharsets.UTF_8);
-    		org.apache.jena.rdf.model.Model model = RDFUtil.getRdfModelFromJsonld(jsonld, FORMAT);
+    		org.apache.jena.rdf.model.Model model = RDFUtil.getRdfModelBasedOnFormat(jsonld, FORMAT);
     		rdfModel = JenaRDF4J.asRDF4JModel(model);
     }
 

diff --git a/...ware/rdf-conversion/src/main/java/io/opensaber/registry/middleware/impl/RDFConverter.java b/...ware/rdf-conversion/src/main/java/io/opensaber/registry/middleware/impl/RDFConverter.java
@@ -23,7 +23,7 @@ public Map<String, Object> execute(Map<String, Object> mapData) throws IOExcepti
 		if (jsonld == null) {
 			throw new MiddlewareHaltException(JSONLD_DATA_IS_MISSING);
 		} else if (jsonld instanceof String) {
-			Model rdfModel = RDFUtil.getRdfModelFromJsonld(jsonld.toString(), FORMAT);
+			Model rdfModel = RDFUtil.getRdfModelBasedOnFormat(jsonld.toString(), FORMAT);
 			mapData.put(Constants.RDF_OBJECT, rdfModel);
 		} else {
 			throw new MiddlewareHaltException(this.getClass().getName() + "JSONLD data is invalid!");

diff --git a/...-mapping/src/test/java/io/opensaber/registry/middleware/impl/RDFValidationMapperTest.java b/...-mapping/src/test/java/io/opensaber/registry/middleware/impl/RDFValidationMapperTest.java
@@ -78,7 +78,7 @@ public void loadSchemaForValidation(String validationFile) throws IOException {
 	}
 
 	public void loadValidationConfigModel(){
-		validationConfig = RDFUtil.getRdfModelFromJsonld(schema.serialize(FORMAT).right().get(), FORMAT);
+		validationConfig = RDFUtil.getRdfModelBasedOnFormat(schema.serialize(FORMAT).right().get(), FORMAT);
 	}
 
 	private void setJsonld(String filename){
@@ -117,7 +117,7 @@ private URI getPath(String file) throws URISyntaxException {
 
 	private Model getNewValidRdf(String fileName){
 		setJsonld(fileName);
-		Model model = RDFUtil.getRdfModelFromJsonld(jsonld, FORMAT);
+		Model model = RDFUtil.getRdfModelBasedOnFormat(jsonld, FORMAT);
 		return model;
 	}
 
@@ -127,7 +127,7 @@ private Model getRdfWithDifferentType(String fileName){
 		JsonObject jsonObject = p.parse(jsonld).getAsJsonObject();
 		jsonObject.addProperty("@type", "School1");
 		String dataString = new Gson().toJson(jsonObject);
-		Model model = RDFUtil.getRdfModelFromJsonld(dataString, FORMAT);
+		Model model = RDFUtil.getRdfModelBasedOnFormat(dataString, FORMAT);
 		return model;
 	}
 
@@ -137,7 +137,7 @@ private Model getRdfWithComplexNodeRemoved(String fileName){
 		JsonObject jsonObject = p.parse(jsonld).getAsJsonObject();
 		jsonObject.remove("sample:address");
 		String dataString = new Gson().toJson(jsonObject);
-		Model model = RDFUtil.getRdfModelFromJsonld(dataString, FORMAT);
+		Model model = RDFUtil.getRdfModelBasedOnFormat(dataString, FORMAT);
 		return model;
 	}
 

diff --git a/...ware/rdf-validation/src/main/java/io/opensaber/registry/middleware/impl/RDFValidator.java b/...ware/rdf-validation/src/main/java/io/opensaber/registry/middleware/impl/RDFValidator.java
@@ -1,13 +1,23 @@
 package io.opensaber.registry.middleware.impl;
 
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import org.apache.jena.rdf.model.Model;
+import org.apache.jena.rdf.model.ModelFactory;
+import org.apache.jena.rdf.model.Property;
+import org.apache.jena.rdf.model.RDFNode;
+import org.apache.jena.rdf.model.Resource;
+import org.apache.jena.rdf.model.ResourceFactory;
+import org.apache.jena.vocabulary.RDF;
+
 import es.weso.schema.Schema;
 import io.opensaber.registry.middleware.Middleware;
 import io.opensaber.registry.middleware.MiddlewareHaltException;
 import io.opensaber.registry.middleware.Validator;
 import io.opensaber.registry.middleware.util.Constants;
+import io.opensaber.registry.middleware.util.RDFUtil;
 import io.opensaber.validators.shex.shaclex.ShaclexValidator;
 import io.opensaber.pojos.ValidationResponse;
 
@@ -21,40 +31,54 @@ public class RDFValidator implements Middleware{
 	private static final String INVALID_REQUEST_PATH = "Request URL is invalid";
 	private static final String ADD_REQUEST_PATH = "/add";
 
+	private static final String VALIDATION_IS_MISSING = "Validation is missing";
+	private static final String VALIDATION_MISSING_FOR_TYPE = "Validation missing for type";
+	private static final String SX_SHAPE_IRI = "http://shex.io/ns/shex#Shape";
+	private static final String SX_EXPRESSION_IRI = "http://shex.io/ns/shex#expression";
+	private static final String SX_EXPRESSIONS_IRI = "http://shex.io/ns/shex#expressions";
+	private static final String SX_VALUES_IRI = "http://shex.io/ns/shex#values";
+	private static final String SX_VALUE_EXPR_IRI = "http://shex.io/ns/shex#valueExpr";
+	private static final String FORMAT = "JSON-LD";
+
+	private Map<String,String> shapeTypeMap;
 	private Schema schemaForCreate;
 	private Schema schemaForUpdate;
 
 	public RDFValidator(Schema schemaForCreate, Schema schemaForUpdate) {
 		this.schemaForCreate = schemaForCreate;
 		this.schemaForUpdate = schemaForUpdate;
+		this.shapeTypeMap = getShapeMap(RDF.type, SX_SHAPE_IRI);
+	}
+
+	public Map<String,String> getShapeTypeMap(){
+		return shapeTypeMap;
 	}
 
 	public Map<String, Object> execute(Map<String, Object> mapData) throws IOException, MiddlewareHaltException {
 		Object RDF = mapData.get(Constants.RDF_OBJECT);
 		Object method = mapData.get(Constants.METHOD_ORIGIN);
-		Object validationRDF = mapData.get(Constants.RDF_VALIDATION_MAPPER_OBJECT);
+		//Object validationRDF = mapData.get(Constants.RDF_VALIDATION_MAPPER_OBJECT);
 		if (RDF == null) {
 			throw new MiddlewareHaltException(RDF_DATA_IS_MISSING);
-		} else if (validationRDF == null) {
-			throw new MiddlewareHaltException(RDF_VALIDATION_MAPPING_MISSING);
-		} else if (!(RDF instanceof Model)) {
+		}else if (!(RDF instanceof Model)) {
 			throw new MiddlewareHaltException(RDF_DATA_IS_INVALID);
-		} else if (!(validationRDF instanceof Model)) {
-			throw new MiddlewareHaltException(RDF_VALIDATION_MAPPING_IS_INVALID);
 		}else if (method == null){
 			throw new MiddlewareHaltException(INVALID_REQUEST_PATH);
 		}else if (schemaForCreate == null || schemaForUpdate == null) {
 			throw new MiddlewareHaltException(SCHEMA_IS_NULL);
+		}else if(shapeTypeMap == null){
+			throw new MiddlewareHaltException(this.getClass().getName()+VALIDATION_IS_MISSING); 
 		} else {
 			Schema schema = null;
-			mergeModels((Model) RDF, (Model) validationRDF);
+			Model validationRdf = generateShapeModel((Model)RDF);
+			mergeModels((Model) RDF, (Model) validationRdf);
 			ValidationResponse validationResponse = null;
 			if(ADD_REQUEST_PATH.equals((String)method)){
 				schema = schemaForCreate;
 			} else {
 				schema = schemaForUpdate;
 			}
-			Validator validator = new ShaclexValidator(schema, (Model) validationRDF);
+			Validator validator = new ShaclexValidator(schema, (Model) validationRdf);
 			validationResponse = validator.validate();
 			mapData.put(Constants.RDF_VALIDATION_OBJECT, validationResponse);
 			return mapData;
@@ -71,5 +95,73 @@ private void mergeModels(Model RDF, Model validationRDF){
 			validationRDF.add(RDF.listStatements());
 		}
 	}
+
+
+	private Model generateShapeModel(Model inputRdf) throws MiddlewareHaltException {
+		Model model = ModelFactory.createDefaultModel();
+		List<Resource> labelNodes = RDFUtil.getRootLabels(inputRdf);
+		if (labelNodes.size() != 1) {
+			throw new MiddlewareHaltException(this.getClass().getName() + RDF_DATA_IS_INVALID);
+		}
+		Resource target = labelNodes.get(0);
+		List<String> typeList = RDFUtil.getTypeForSubject(inputRdf, target);
+		if (typeList.size() != 1) {
+			throw new MiddlewareHaltException(this.getClass().getName() + RDF_DATA_IS_INVALID);
+		}
+		String targetType = typeList.get(0);
+		String shapeName = shapeTypeMap.get(targetType);
+		if (shapeName == null) {
+			throw new MiddlewareHaltException(this.getClass().getName() + VALIDATION_MISSING_FOR_TYPE);
+		}
+
+		Resource subjectResource = ResourceFactory.createResource(shapeName);
+		Property predicate = ResourceFactory.createProperty(Constants.TARGET_NODE_IRI);
+		model.add(subjectResource, predicate, target);
+		return model;
+	}
+
+	/**
+	 * This method generates a shapemap which contains mappings between each entity type and the corresponding
+	 * shape that the validations should target. Here we first filter out all the shape resources from the validationConfig.
+	 * Then we iterate through the list of shape resources and do a bunch of filtering from the validationConfig
+	 * based on a few predicates to finally arrive at the type for which the shape is targeted.
+	 * @param predicate
+	 * @param object
+	 * @param validationConfig is the rdf model format of the Schema file used for validations
+	 * @return
+	 */
+	private Map<String,String> getShapeMap(Property predicate, String object){
+		Map<String,String> shapeMap = new HashMap<String, String>();
+		Model validationConfig = getValidationConfigModel();
+		List<Resource> shapeList = RDFUtil.getListOfSubjects(predicate, object, validationConfig);
+		for(Resource shape: shapeList){
+			RDFNode node = getObjectAfterFilter(shape, SX_EXPRESSION_IRI, validationConfig);
+			RDFNode firstNode = getObjectAfterFilter(node, SX_EXPRESSIONS_IRI, validationConfig);
+			RDFNode secondNode = getObjectAfterFilter(firstNode, RDF.first.getURI(), validationConfig);
+			RDFNode thirdNode = getObjectAfterFilter(secondNode, SX_VALUES_IRI, validationConfig);
+			if(thirdNode == null){
+				thirdNode = getObjectAfterFilter(secondNode, SX_VALUE_EXPR_IRI, validationConfig);
+			}
+			RDFNode fourthNode = getObjectAfterFilter(thirdNode, SX_VALUES_IRI, validationConfig);
+			RDFNode typeNode = getObjectAfterFilter(fourthNode, RDF.first.getURI(), validationConfig);
+			if(typeNode!=null){
+				shapeMap.put(typeNode.toString(), shape.toString());
+			}
+		}
+		return shapeMap;
+	}
+
+	private RDFNode getObjectAfterFilter(RDFNode node, String predicate, Model validationConfig){
+			Property property = ResourceFactory.createProperty(predicate);
+			List<RDFNode> nodeList = RDFUtil.getListOfObjectNodes((Resource)node, property,validationConfig);
+			if(nodeList.size() != 0){
+				return nodeList.get(0);
+			}
+			return null;
+	}
+
+	private Model getValidationConfigModel(){
+		return RDFUtil.getRdfModelBasedOnFormat(schemaForUpdate.serialize(FORMAT).right().get(), FORMAT);
+	}
 
 }
diff --git a/...rdf-validation/src/test/java/io/opensaber/registry/middleware/impl/RDFValidationTest.java b/...rdf-validation/src/test/java/io/opensaber/registry/middleware/impl/RDFValidationTest.java
@@ -70,10 +70,11 @@ private boolean setup(String shexFileForCreate, String shexFileForUpdate) {
 		return successfulInitialization;
 	}
 
-	private boolean setup(Schema schema) {
+	private boolean setup(Schema schema, String shexFileForUpdate) {
 		boolean successfulInitialization = true;
 		try {
-			middleware = new RDFValidator(schema, schema);
+			Schema createSchema = readSchema(shexFileForUpdate, SCHEMAFORMAT, PROCESSOR);
+			middleware = new RDFValidator(null, createSchema);
 		} catch (Exception e) {
 			successfulInitialization = false;
 		}
@@ -106,7 +107,7 @@ public void testHaltIfRDFpresentButInvalid() throws IOException, MiddlewareHaltE
 	}
 
 
-	@Test
+	/*@Test
 	public void testHaltIfValidationMappingMissing() throws IOException, MiddlewareHaltException, URISyntaxException{
 		expectedEx.expect(MiddlewareHaltException.class);
 		expectedEx.expectMessage("RDF validation mapping is missing!");
@@ -137,14 +138,14 @@ public void testHaltIfValidationMappingIsNotModel() throws IOException, Middlewa
 		mapData.put(Constants.RDF_VALIDATION_MAPPER_OBJECT, "{}");
 		middleware.execute(mapData);
 		testForSuccessfulResult();
-	}
+	}*/
 
 	@Test
 	public void testHaltIfSchemaIsMissing() throws IOException, MiddlewareHaltException, URISyntaxException{
 		expectedEx.expect(MiddlewareHaltException.class);
 		expectedEx.expectMessage("Schema for validation is missing");
 		Schema schema = null;
-		assertTrue(setup(schema));
+		assertTrue(setup(schema,COMPLEX_UPDATE_SHEX));
 		mapData = new HashMap<String,Object>();
 		mapData.put(Constants.RDF_OBJECT, getValidRdf(COMPLEX_TTL));
 		mapData.put(Constants.METHOD_ORIGIN, ADD_REQUEST_PATH);
@@ -305,13 +306,13 @@ private String readFromFile(String file) throws IOException,FileNotFoundExceptio
 
 	private Model getValidRdf(String filename, String format) {
 		setJsonld(filename);
-		Model model = RDFUtil.getRdfModelFromJsonld(jsonld, format);
+		Model model = RDFUtil.getRdfModelBasedOnFormat(jsonld, format);
 		return model;
 	}
 
 	private Model getValidRdf(String fileName){
 		setJsonld(fileName);
-		Model model = RDFUtil.getRdfModelFromJsonld(jsonld, TTL_FORMAT);
+		Model model = RDFUtil.getRdfModelBasedOnFormat(jsonld, TTL_FORMAT);
 		return model;
 	}
 

diff --git a/java/registry/src/main/java/io/opensaber/registry/config/GenericConfiguration.java b/java/registry/src/main/java/io/opensaber/registry/config/GenericConfiguration.java
@@ -213,10 +213,10 @@ public void addInterceptors(InterceptorRegistry registry) {
         }
 		registry.addInterceptor(rdfConversionInterceptor())
 				.addPathPatterns("/add", "/update").order(2);
-		registry.addInterceptor(rdfValidationMappingInterceptor())
-				.addPathPatterns("/add", "/update").order(3);
+		/*registry.addInterceptor(rdfValidationMappingInterceptor())
+				.addPathPatterns("/add", "/update").order(3);*/
 		registry.addInterceptor(rdfValidationInterceptor())
-				.addPathPatterns("/add", "/update").order(4);
+				.addPathPatterns("/add", "/update").order(3);
 	/*	registry.addInterceptor(new JSONLDConversionInterceptor(jsonldConverter()))
 				.addPathPatterns("/read/{id}").order(2);*/
 	}

diff --git a/java/registry/src/test/java/io/opensaber/registry/controller/RegistryTestBase.java b/java/registry/src/test/java/io/opensaber/registry/controller/RegistryTestBase.java
@@ -93,24 +93,24 @@ public String generateBaseUrl(){
 	public Model getNewValidRdf(String fileName, String contextConstant){
 		setJsonld(fileName);
 		setJsonldWithNewRootLabel(contextConstant+generateRandomId());
-		Model model = RDFUtil.getRdfModelFromJsonld(jsonld, FORMAT);
+		Model model = RDFUtil.getRdfModelBasedOnFormat(jsonld, FORMAT);
 		return model;
 	}
 
 	public Model getNewValidRdf(String fileName){
 		setJsonld(fileName);
 		setJsonldWithNewRootLabel();
-		return RDFUtil.getRdfModelFromJsonld(jsonld, FORMAT);
+		return RDFUtil.getRdfModelBasedOnFormat(jsonld, FORMAT);
 	}
 
 	public Model getNewValidRdfFromJsonString(String json){
-		return RDFUtil.getRdfModelFromJsonld(json, FORMAT);
+		return RDFUtil.getRdfModelBasedOnFormat(json, FORMAT);
 	}
 
 	public Model getNewValidRdf(String fileName, String contextConstant, String rootNodeLabel){
 		setJsonld(fileName);
 		setJsonldWithNewRootLabel(rootNodeLabel);
-		return RDFUtil.getRdfModelFromJsonld(jsonld, FORMAT);
+		return RDFUtil.getRdfModelBasedOnFormat(jsonld, FORMAT);
 	}
 
 	public Model getRdfWithInvalidTpe(){

diff --git a/...a-configuration/src/main/java/io/opensaber/registry/schema/config/SchemaConfigurator.java b/...a-configuration/src/main/java/io/opensaber/registry/schema/config/SchemaConfigurator.java
@@ -50,7 +50,7 @@ private void loadSchemaConfigModel(String schemaFile) throws IOException {
 			throw new IOException(Constants.SCHEMA_CONFIGURATION_MISSING);
 		}
 		String contents = new String(ByteStreams.toByteArray(is));
-		schemaConfig = RDFUtil.getRdfModelFromJsonld(contents, FORMAT);
+		schemaConfig = RDFUtil.getRdfModelBasedOnFormat(contents, FORMAT);
 	}
 
 	private void loadSchemaForValidation(String validationFile, boolean isCreate) throws IOException {
@@ -71,7 +71,7 @@ private void loadSchemaForValidation(String validationFile, boolean isCreate) th
 	}
 
 	private void loadValidationConfigModel(){
-		validationConfig = RDFUtil.getRdfModelFromJsonld(schemaForUpdate.serialize(FORMAT).right().get(), FORMAT);
+		validationConfig = RDFUtil.getRdfModelBasedOnFormat(schemaForUpdate.serialize(FORMAT).right().get(), FORMAT);
 	}
 
 	public boolean isPrivate(String propertyName) {

diff --git a/...shex/shaclex/src/test/java/io/opensaber/validators/shex/shaclex/ShaclexValidatorTest.java b/...shex/shaclex/src/test/java/io/opensaber/validators/shex/shaclex/ShaclexValidatorTest.java
@@ -251,7 +251,7 @@ public void test_validate_nested_enumerated_constants_property() throws Exceptio
 
 	private ValidationResponse validate(String data, String dataFormat, String schemaFile,
 									   String schemaFormat, String processor) throws Exception {
-		Model dataModel = RDFUtil.getRdfModelFromJsonld(data, dataFormat);
+		Model dataModel = RDFUtil.getRdfModelBasedOnFormat(data, dataFormat);
 		Model validationRDF = generateShapeModel(dataModel);
 		mergeModels(dataModel, validationRDF);
 		Schema schema = readSchema(Paths.get(schemaFile), schemaFormat, processor);