From 8b3d667565ba371b69adaa255cbf936dd3d8bf5e Mon Sep 17 00:00:00 2001 From: jyotsna-tarento Date: Mon, 18 Jun 2018 12:57:07 +0530 Subject: [PATCH] Issue #SB-4635 Combining validation and mapping --- .../registry/middleware/util/RDFUtil.java | 2 +- .../middleware/impl/JSONLDConversionTest.java | 2 +- .../middleware/impl/RDFConverter.java | 2 +- .../impl/RDFValidationMapperTest.java | 8 +- .../middleware/impl/RDFValidator.java | 108 ++++++++++++++++-- .../middleware/impl/RDFValidationTest.java | 15 +-- .../registry/config/GenericConfiguration.java | 6 +- .../registry/controller/RegistryTestBase.java | 8 +- .../schema/config/SchemaConfigurator.java | 4 +- .../shex/shaclex/ShaclexValidatorTest.java | 2 +- 10 files changed, 125 insertions(+), 32 deletions(-) diff --git a/java/middleware-commons/src/main/java/io/opensaber/registry/middleware/util/RDFUtil.java b/java/middleware-commons/src/main/java/io/opensaber/registry/middleware/util/RDFUtil.java index e79379060..cc269d08d 100644 --- a/java/middleware-commons/src/main/java/io/opensaber/registry/middleware/util/RDFUtil.java +++ b/java/middleware-commons/src/main/java/io/opensaber/registry/middleware/util/RDFUtil.java @@ -34,7 +34,7 @@ public class RDFUtil { private static String registryContext; - public static Model getRdfModelFromJsonld(String jsonldData, String format){ + public static Model getRdfModelBasedOnFormat(String jsonldData, String format){ Model m = ModelFactory.createDefaultModel(); StringReader reader = new StringReader(jsonldData); return m.read(reader, null, format); diff --git a/java/middleware/registry-middleware/jsonld-conversion/src/test/java/io/opensaber/registry/middleware/impl/JSONLDConversionTest.java b/java/middleware/registry-middleware/jsonld-conversion/src/test/java/io/opensaber/registry/middleware/impl/JSONLDConversionTest.java index 42956ff65..39876a248 100644 --- a/java/middleware/registry-middleware/jsonld-conversion/src/test/java/io/opensaber/registry/middleware/impl/JSONLDConversionTest.java +++ b/java/middleware/registry-middleware/jsonld-conversion/src/test/java/io/opensaber/registry/middleware/impl/JSONLDConversionTest.java @@ -43,7 +43,7 @@ private void setup() throws IOException, URISyntaxException{ String jsonLDData = Paths.get(getPath(TEACHER_JSONLD)).toString(); Path filePath = Paths.get(jsonLDData); String jsonld = new String(Files.readAllBytes(filePath), StandardCharsets.UTF_8); - org.apache.jena.rdf.model.Model model = RDFUtil.getRdfModelFromJsonld(jsonld, FORMAT); + org.apache.jena.rdf.model.Model model = RDFUtil.getRdfModelBasedOnFormat(jsonld, FORMAT); rdfModel = JenaRDF4J.asRDF4JModel(model); } diff --git a/java/middleware/registry-middleware/rdf-conversion/src/main/java/io/opensaber/registry/middleware/impl/RDFConverter.java b/java/middleware/registry-middleware/rdf-conversion/src/main/java/io/opensaber/registry/middleware/impl/RDFConverter.java index 9cc1a39ab..c31e13694 100644 --- a/java/middleware/registry-middleware/rdf-conversion/src/main/java/io/opensaber/registry/middleware/impl/RDFConverter.java +++ b/java/middleware/registry-middleware/rdf-conversion/src/main/java/io/opensaber/registry/middleware/impl/RDFConverter.java @@ -23,7 +23,7 @@ public Map execute(Map mapData) throws IOExcepti if (jsonld == null) { throw new MiddlewareHaltException(JSONLD_DATA_IS_MISSING); } else if (jsonld instanceof String) { - Model rdfModel = RDFUtil.getRdfModelFromJsonld(jsonld.toString(), FORMAT); + Model rdfModel = RDFUtil.getRdfModelBasedOnFormat(jsonld.toString(), FORMAT); mapData.put(Constants.RDF_OBJECT, rdfModel); } else { throw new MiddlewareHaltException(this.getClass().getName() + "JSONLD data is invalid!"); diff --git a/java/middleware/registry-middleware/rdf-validation-mapping/src/test/java/io/opensaber/registry/middleware/impl/RDFValidationMapperTest.java b/java/middleware/registry-middleware/rdf-validation-mapping/src/test/java/io/opensaber/registry/middleware/impl/RDFValidationMapperTest.java index 8d269e166..fcd682875 100644 --- a/java/middleware/registry-middleware/rdf-validation-mapping/src/test/java/io/opensaber/registry/middleware/impl/RDFValidationMapperTest.java +++ b/java/middleware/registry-middleware/rdf-validation-mapping/src/test/java/io/opensaber/registry/middleware/impl/RDFValidationMapperTest.java @@ -78,7 +78,7 @@ public void loadSchemaForValidation(String validationFile) throws IOException { } public void loadValidationConfigModel(){ - validationConfig = RDFUtil.getRdfModelFromJsonld(schema.serialize(FORMAT).right().get(), FORMAT); + validationConfig = RDFUtil.getRdfModelBasedOnFormat(schema.serialize(FORMAT).right().get(), FORMAT); } private void setJsonld(String filename){ @@ -117,7 +117,7 @@ private URI getPath(String file) throws URISyntaxException { private Model getNewValidRdf(String fileName){ setJsonld(fileName); - Model model = RDFUtil.getRdfModelFromJsonld(jsonld, FORMAT); + Model model = RDFUtil.getRdfModelBasedOnFormat(jsonld, FORMAT); return model; } @@ -127,7 +127,7 @@ private Model getRdfWithDifferentType(String fileName){ JsonObject jsonObject = p.parse(jsonld).getAsJsonObject(); jsonObject.addProperty("@type", "School1"); String dataString = new Gson().toJson(jsonObject); - Model model = RDFUtil.getRdfModelFromJsonld(dataString, FORMAT); + Model model = RDFUtil.getRdfModelBasedOnFormat(dataString, FORMAT); return model; } @@ -137,7 +137,7 @@ private Model getRdfWithComplexNodeRemoved(String fileName){ JsonObject jsonObject = p.parse(jsonld).getAsJsonObject(); jsonObject.remove("sample:address"); String dataString = new Gson().toJson(jsonObject); - Model model = RDFUtil.getRdfModelFromJsonld(dataString, FORMAT); + Model model = RDFUtil.getRdfModelBasedOnFormat(dataString, FORMAT); return model; } diff --git a/java/middleware/registry-middleware/rdf-validation/src/main/java/io/opensaber/registry/middleware/impl/RDFValidator.java b/java/middleware/registry-middleware/rdf-validation/src/main/java/io/opensaber/registry/middleware/impl/RDFValidator.java index 16a4d3f55..7888014fb 100644 --- a/java/middleware/registry-middleware/rdf-validation/src/main/java/io/opensaber/registry/middleware/impl/RDFValidator.java +++ b/java/middleware/registry-middleware/rdf-validation/src/main/java/io/opensaber/registry/middleware/impl/RDFValidator.java @@ -1,13 +1,23 @@ package io.opensaber.registry.middleware.impl; import java.io.IOException; +import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; +import org.apache.jena.rdf.model.Property; +import org.apache.jena.rdf.model.RDFNode; +import org.apache.jena.rdf.model.Resource; +import org.apache.jena.rdf.model.ResourceFactory; +import org.apache.jena.vocabulary.RDF; + import es.weso.schema.Schema; import io.opensaber.registry.middleware.Middleware; import io.opensaber.registry.middleware.MiddlewareHaltException; import io.opensaber.registry.middleware.Validator; import io.opensaber.registry.middleware.util.Constants; +import io.opensaber.registry.middleware.util.RDFUtil; import io.opensaber.validators.shex.shaclex.ShaclexValidator; import io.opensaber.pojos.ValidationResponse; @@ -21,40 +31,54 @@ public class RDFValidator implements Middleware{ private static final String INVALID_REQUEST_PATH = "Request URL is invalid"; private static final String ADD_REQUEST_PATH = "/add"; + private static final String VALIDATION_IS_MISSING = "Validation is missing"; + private static final String VALIDATION_MISSING_FOR_TYPE = "Validation missing for type"; + private static final String SX_SHAPE_IRI = "http://shex.io/ns/shex#Shape"; + private static final String SX_EXPRESSION_IRI = "http://shex.io/ns/shex#expression"; + private static final String SX_EXPRESSIONS_IRI = "http://shex.io/ns/shex#expressions"; + private static final String SX_VALUES_IRI = "http://shex.io/ns/shex#values"; + private static final String SX_VALUE_EXPR_IRI = "http://shex.io/ns/shex#valueExpr"; + private static final String FORMAT = "JSON-LD"; + + private Map shapeTypeMap; private Schema schemaForCreate; private Schema schemaForUpdate; public RDFValidator(Schema schemaForCreate, Schema schemaForUpdate) { this.schemaForCreate = schemaForCreate; this.schemaForUpdate = schemaForUpdate; + this.shapeTypeMap = getShapeMap(RDF.type, SX_SHAPE_IRI); + } + + public Map getShapeTypeMap(){ + return shapeTypeMap; } public Map execute(Map mapData) throws IOException, MiddlewareHaltException { Object RDF = mapData.get(Constants.RDF_OBJECT); Object method = mapData.get(Constants.METHOD_ORIGIN); - Object validationRDF = mapData.get(Constants.RDF_VALIDATION_MAPPER_OBJECT); + //Object validationRDF = mapData.get(Constants.RDF_VALIDATION_MAPPER_OBJECT); if (RDF == null) { throw new MiddlewareHaltException(RDF_DATA_IS_MISSING); - } else if (validationRDF == null) { - throw new MiddlewareHaltException(RDF_VALIDATION_MAPPING_MISSING); - } else if (!(RDF instanceof Model)) { + }else if (!(RDF instanceof Model)) { throw new MiddlewareHaltException(RDF_DATA_IS_INVALID); - } else if (!(validationRDF instanceof Model)) { - throw new MiddlewareHaltException(RDF_VALIDATION_MAPPING_IS_INVALID); }else if (method == null){ throw new MiddlewareHaltException(INVALID_REQUEST_PATH); }else if (schemaForCreate == null || schemaForUpdate == null) { throw new MiddlewareHaltException(SCHEMA_IS_NULL); + }else if(shapeTypeMap == null){ + throw new MiddlewareHaltException(this.getClass().getName()+VALIDATION_IS_MISSING); } else { Schema schema = null; - mergeModels((Model) RDF, (Model) validationRDF); + Model validationRdf = generateShapeModel((Model)RDF); + mergeModels((Model) RDF, (Model) validationRdf); ValidationResponse validationResponse = null; if(ADD_REQUEST_PATH.equals((String)method)){ schema = schemaForCreate; } else { schema = schemaForUpdate; } - Validator validator = new ShaclexValidator(schema, (Model) validationRDF); + Validator validator = new ShaclexValidator(schema, (Model) validationRdf); validationResponse = validator.validate(); mapData.put(Constants.RDF_VALIDATION_OBJECT, validationResponse); return mapData; @@ -71,5 +95,73 @@ private void mergeModels(Model RDF, Model validationRDF){ validationRDF.add(RDF.listStatements()); } } + + + private Model generateShapeModel(Model inputRdf) throws MiddlewareHaltException { + Model model = ModelFactory.createDefaultModel(); + List labelNodes = RDFUtil.getRootLabels(inputRdf); + if (labelNodes.size() != 1) { + throw new MiddlewareHaltException(this.getClass().getName() + RDF_DATA_IS_INVALID); + } + Resource target = labelNodes.get(0); + List typeList = RDFUtil.getTypeForSubject(inputRdf, target); + if (typeList.size() != 1) { + throw new MiddlewareHaltException(this.getClass().getName() + RDF_DATA_IS_INVALID); + } + String targetType = typeList.get(0); + String shapeName = shapeTypeMap.get(targetType); + if (shapeName == null) { + throw new MiddlewareHaltException(this.getClass().getName() + VALIDATION_MISSING_FOR_TYPE); + } + + Resource subjectResource = ResourceFactory.createResource(shapeName); + Property predicate = ResourceFactory.createProperty(Constants.TARGET_NODE_IRI); + model.add(subjectResource, predicate, target); + return model; + } + + /** + * This method generates a shapemap which contains mappings between each entity type and the corresponding + * shape that the validations should target. Here we first filter out all the shape resources from the validationConfig. + * Then we iterate through the list of shape resources and do a bunch of filtering from the validationConfig + * based on a few predicates to finally arrive at the type for which the shape is targeted. + * @param predicate + * @param object + * @param validationConfig is the rdf model format of the Schema file used for validations + * @return + */ + private Map getShapeMap(Property predicate, String object){ + Map shapeMap = new HashMap(); + Model validationConfig = getValidationConfigModel(); + List shapeList = RDFUtil.getListOfSubjects(predicate, object, validationConfig); + for(Resource shape: shapeList){ + RDFNode node = getObjectAfterFilter(shape, SX_EXPRESSION_IRI, validationConfig); + RDFNode firstNode = getObjectAfterFilter(node, SX_EXPRESSIONS_IRI, validationConfig); + RDFNode secondNode = getObjectAfterFilter(firstNode, RDF.first.getURI(), validationConfig); + RDFNode thirdNode = getObjectAfterFilter(secondNode, SX_VALUES_IRI, validationConfig); + if(thirdNode == null){ + thirdNode = getObjectAfterFilter(secondNode, SX_VALUE_EXPR_IRI, validationConfig); + } + RDFNode fourthNode = getObjectAfterFilter(thirdNode, SX_VALUES_IRI, validationConfig); + RDFNode typeNode = getObjectAfterFilter(fourthNode, RDF.first.getURI(), validationConfig); + if(typeNode!=null){ + shapeMap.put(typeNode.toString(), shape.toString()); + } + } + return shapeMap; + } + + private RDFNode getObjectAfterFilter(RDFNode node, String predicate, Model validationConfig){ + Property property = ResourceFactory.createProperty(predicate); + List nodeList = RDFUtil.getListOfObjectNodes((Resource)node, property,validationConfig); + if(nodeList.size() != 0){ + return nodeList.get(0); + } + return null; + } + + private Model getValidationConfigModel(){ + return RDFUtil.getRdfModelBasedOnFormat(schemaForUpdate.serialize(FORMAT).right().get(), FORMAT); + } } diff --git a/java/middleware/registry-middleware/rdf-validation/src/test/java/io/opensaber/registry/middleware/impl/RDFValidationTest.java b/java/middleware/registry-middleware/rdf-validation/src/test/java/io/opensaber/registry/middleware/impl/RDFValidationTest.java index 4a4fafb20..f26cac7da 100644 --- a/java/middleware/registry-middleware/rdf-validation/src/test/java/io/opensaber/registry/middleware/impl/RDFValidationTest.java +++ b/java/middleware/registry-middleware/rdf-validation/src/test/java/io/opensaber/registry/middleware/impl/RDFValidationTest.java @@ -70,10 +70,11 @@ private boolean setup(String shexFileForCreate, String shexFileForUpdate) { return successfulInitialization; } - private boolean setup(Schema schema) { + private boolean setup(Schema schema, String shexFileForUpdate) { boolean successfulInitialization = true; try { - middleware = new RDFValidator(schema, schema); + Schema createSchema = readSchema(shexFileForUpdate, SCHEMAFORMAT, PROCESSOR); + middleware = new RDFValidator(null, createSchema); } catch (Exception e) { successfulInitialization = false; } @@ -106,7 +107,7 @@ public void testHaltIfRDFpresentButInvalid() throws IOException, MiddlewareHaltE } - @Test + /*@Test public void testHaltIfValidationMappingMissing() throws IOException, MiddlewareHaltException, URISyntaxException{ expectedEx.expect(MiddlewareHaltException.class); expectedEx.expectMessage("RDF validation mapping is missing!"); @@ -137,14 +138,14 @@ public void testHaltIfValidationMappingIsNotModel() throws IOException, Middlewa mapData.put(Constants.RDF_VALIDATION_MAPPER_OBJECT, "{}"); middleware.execute(mapData); testForSuccessfulResult(); - } + }*/ @Test public void testHaltIfSchemaIsMissing() throws IOException, MiddlewareHaltException, URISyntaxException{ expectedEx.expect(MiddlewareHaltException.class); expectedEx.expectMessage("Schema for validation is missing"); Schema schema = null; - assertTrue(setup(schema)); + assertTrue(setup(schema,COMPLEX_UPDATE_SHEX)); mapData = new HashMap(); mapData.put(Constants.RDF_OBJECT, getValidRdf(COMPLEX_TTL)); mapData.put(Constants.METHOD_ORIGIN, ADD_REQUEST_PATH); @@ -305,13 +306,13 @@ private String readFromFile(String file) throws IOException,FileNotFoundExceptio private Model getValidRdf(String filename, String format) { setJsonld(filename); - Model model = RDFUtil.getRdfModelFromJsonld(jsonld, format); + Model model = RDFUtil.getRdfModelBasedOnFormat(jsonld, format); return model; } private Model getValidRdf(String fileName){ setJsonld(fileName); - Model model = RDFUtil.getRdfModelFromJsonld(jsonld, TTL_FORMAT); + Model model = RDFUtil.getRdfModelBasedOnFormat(jsonld, TTL_FORMAT); return model; } diff --git a/java/registry/src/main/java/io/opensaber/registry/config/GenericConfiguration.java b/java/registry/src/main/java/io/opensaber/registry/config/GenericConfiguration.java index d2040efd9..fdb4289fa 100644 --- a/java/registry/src/main/java/io/opensaber/registry/config/GenericConfiguration.java +++ b/java/registry/src/main/java/io/opensaber/registry/config/GenericConfiguration.java @@ -213,10 +213,10 @@ public void addInterceptors(InterceptorRegistry registry) { } registry.addInterceptor(rdfConversionInterceptor()) .addPathPatterns("/add", "/update").order(2); - registry.addInterceptor(rdfValidationMappingInterceptor()) - .addPathPatterns("/add", "/update").order(3); + /*registry.addInterceptor(rdfValidationMappingInterceptor()) + .addPathPatterns("/add", "/update").order(3);*/ registry.addInterceptor(rdfValidationInterceptor()) - .addPathPatterns("/add", "/update").order(4); + .addPathPatterns("/add", "/update").order(3); /* registry.addInterceptor(new JSONLDConversionInterceptor(jsonldConverter())) .addPathPatterns("/read/{id}").order(2);*/ } diff --git a/java/registry/src/test/java/io/opensaber/registry/controller/RegistryTestBase.java b/java/registry/src/test/java/io/opensaber/registry/controller/RegistryTestBase.java index 5767c7a1b..361fceea5 100644 --- a/java/registry/src/test/java/io/opensaber/registry/controller/RegistryTestBase.java +++ b/java/registry/src/test/java/io/opensaber/registry/controller/RegistryTestBase.java @@ -93,24 +93,24 @@ public String generateBaseUrl(){ public Model getNewValidRdf(String fileName, String contextConstant){ setJsonld(fileName); setJsonldWithNewRootLabel(contextConstant+generateRandomId()); - Model model = RDFUtil.getRdfModelFromJsonld(jsonld, FORMAT); + Model model = RDFUtil.getRdfModelBasedOnFormat(jsonld, FORMAT); return model; } public Model getNewValidRdf(String fileName){ setJsonld(fileName); setJsonldWithNewRootLabel(); - return RDFUtil.getRdfModelFromJsonld(jsonld, FORMAT); + return RDFUtil.getRdfModelBasedOnFormat(jsonld, FORMAT); } public Model getNewValidRdfFromJsonString(String json){ - return RDFUtil.getRdfModelFromJsonld(json, FORMAT); + return RDFUtil.getRdfModelBasedOnFormat(json, FORMAT); } public Model getNewValidRdf(String fileName, String contextConstant, String rootNodeLabel){ setJsonld(fileName); setJsonldWithNewRootLabel(rootNodeLabel); - return RDFUtil.getRdfModelFromJsonld(jsonld, FORMAT); + return RDFUtil.getRdfModelBasedOnFormat(jsonld, FORMAT); } public Model getRdfWithInvalidTpe(){ diff --git a/java/schema-configuration/src/main/java/io/opensaber/registry/schema/config/SchemaConfigurator.java b/java/schema-configuration/src/main/java/io/opensaber/registry/schema/config/SchemaConfigurator.java index 1acc8e5a8..050e895f1 100644 --- a/java/schema-configuration/src/main/java/io/opensaber/registry/schema/config/SchemaConfigurator.java +++ b/java/schema-configuration/src/main/java/io/opensaber/registry/schema/config/SchemaConfigurator.java @@ -50,7 +50,7 @@ private void loadSchemaConfigModel(String schemaFile) throws IOException { throw new IOException(Constants.SCHEMA_CONFIGURATION_MISSING); } String contents = new String(ByteStreams.toByteArray(is)); - schemaConfig = RDFUtil.getRdfModelFromJsonld(contents, FORMAT); + schemaConfig = RDFUtil.getRdfModelBasedOnFormat(contents, FORMAT); } private void loadSchemaForValidation(String validationFile, boolean isCreate) throws IOException { @@ -71,7 +71,7 @@ private void loadSchemaForValidation(String validationFile, boolean isCreate) th } private void loadValidationConfigModel(){ - validationConfig = RDFUtil.getRdfModelFromJsonld(schemaForUpdate.serialize(FORMAT).right().get(), FORMAT); + validationConfig = RDFUtil.getRdfModelBasedOnFormat(schemaForUpdate.serialize(FORMAT).right().get(), FORMAT); } public boolean isPrivate(String propertyName) { diff --git a/java/validators/shex/shaclex/src/test/java/io/opensaber/validators/shex/shaclex/ShaclexValidatorTest.java b/java/validators/shex/shaclex/src/test/java/io/opensaber/validators/shex/shaclex/ShaclexValidatorTest.java index c395c6ea3..eb455c2bf 100644 --- a/java/validators/shex/shaclex/src/test/java/io/opensaber/validators/shex/shaclex/ShaclexValidatorTest.java +++ b/java/validators/shex/shaclex/src/test/java/io/opensaber/validators/shex/shaclex/ShaclexValidatorTest.java @@ -251,7 +251,7 @@ public void test_validate_nested_enumerated_constants_property() throws Exceptio private ValidationResponse validate(String data, String dataFormat, String schemaFile, String schemaFormat, String processor) throws Exception { - Model dataModel = RDFUtil.getRdfModelFromJsonld(data, dataFormat); + Model dataModel = RDFUtil.getRdfModelBasedOnFormat(data, dataFormat); Model validationRDF = generateShapeModel(dataModel); mergeModels(dataModel, validationRDF); Schema schema = readSchema(Paths.get(schemaFile), schemaFormat, processor);