diff --git a/java/middleware/registry-middleware/authorization/src/main/java/io/opensaber/registry/authorization/AuthorizationFilter.java b/java/middleware/registry-middleware/authorization/src/main/java/io/opensaber/registry/authorization/AuthorizationFilter.java index c325c7cc1..f64c9fcc3 100644 --- a/java/middleware/registry-middleware/authorization/src/main/java/io/opensaber/registry/authorization/AuthorizationFilter.java +++ b/java/middleware/registry-middleware/authorization/src/main/java/io/opensaber/registry/authorization/AuthorizationFilter.java @@ -20,6 +20,8 @@ import io.opensaber.registry.middleware.MiddlewareHaltException; import io.opensaber.registry.middleware.util.Constants; import org.apache.commons.codec.binary.Base64; +import org.perf4j.StopWatch; +import org.perf4j.LoggingStopWatch; public class AuthorizationFilter implements BaseMiddleware { @@ -31,6 +33,7 @@ public class AuthorizationFilter implements BaseMiddleware { public KeyCloakServiceImpl keyCloakServiceImpl; + public AuthorizationFilter() {} public AuthorizationFilter(KeyCloakServiceImpl keyCloakServiceImpl) { @@ -50,7 +53,9 @@ public Map execute(Map mapObject) throws Middlew } String token = tokenObject.toString(); try { + StopWatch stopWatch = new LoggingStopWatch(); if (!keyCloakServiceImpl.verifyToken(token).trim().isEmpty()) { + stopWatch.stop("KeyCloak Authentication","Performance Monitoring !"); if (mapObject.containsKey("userName")) { logger.info("Access token for user {} verified successfully with KeyCloak server !", mapObject.get("userName")); } else { diff --git a/java/registry/src/main/java/io/opensaber/registry/app/OpenSaberApplication.java b/java/registry/src/main/java/io/opensaber/registry/app/OpenSaberApplication.java index fd9bb28b7..552ab4a65 100644 --- a/java/registry/src/main/java/io/opensaber/registry/app/OpenSaberApplication.java +++ b/java/registry/src/main/java/io/opensaber/registry/app/OpenSaberApplication.java @@ -1,5 +1,7 @@ package io.opensaber.registry.app; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.annotation.ComponentScan; @@ -8,8 +10,16 @@ @ComponentScan({"io.opensaber.registry"}) public class OpenSaberApplication { + private static Logger logger = LoggerFactory.getLogger(OpenSaberApplication.class); public static void main(String[] args) { + logger.info("****************************************************"); + logger.info(" Welcome to Opensaber Application "); + logger.info("****************************************************"); + SpringApplication.run(OpenSaberApplication.class, args); } + + + } diff --git a/java/registry/src/main/java/io/opensaber/registry/dao/impl/RegistryDaoImpl.java b/java/registry/src/main/java/io/opensaber/registry/dao/impl/RegistryDaoImpl.java index 57c23e6be..6f53dc41a 100644 --- a/java/registry/src/main/java/io/opensaber/registry/dao/impl/RegistryDaoImpl.java +++ b/java/registry/src/main/java/io/opensaber/registry/dao/impl/RegistryDaoImpl.java @@ -5,6 +5,7 @@ import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; + import com.google.common.collect.ImmutableList; import io.opensaber.registry.exception.AuditFailedException; import io.opensaber.registry.model.AuditRecord; @@ -34,103 +35,115 @@ import io.opensaber.registry.exception.RecordNotFoundException; import io.opensaber.registry.middleware.util.Constants; import org.apache.commons.validator.routines.UrlValidator; +/*import org.perf4j.StopWatch; +import org.perf4j.LoggingStopWatch;*/ +import org.springframework.util.StopWatch; @Component public class RegistryDaoImpl implements RegistryDao { - public static final String META = "meta."; - private static Logger logger = LoggerFactory.getLogger(RegistryDaoImpl.class); - - - @Autowired - private DatabaseProvider databaseProvider; - - @Autowired - private EncryptionService encryptionService; - - @Value("${registry.context.base}") - private String registryContext; - - @Autowired - SchemaConfigurator schemaConfigurator; - - @Autowired - ApplicationContext appContext; - - @Value("${feature.toggling}") - private Boolean featureToggling; - - @Override - public List getEntityList() { - // TODO Auto-generated method stub - return null; - } - - @Override - public String addEntity(Graph entity, String label) throws DuplicateRecordException, NoSuchElementException, EncryptionException, AuditFailedException, RecordNotFoundException{ - logger.debug("RegistryDaoImpl : Database Provider features: \n" + databaseProvider.getGraphStore().features()); - Graph graphFromStore = databaseProvider.getGraphStore(); - GraphTraversalSource dbGraphTraversalSource = graphFromStore.traversal(); - if (dbGraphTraversalSource.clone().V().hasLabel(label).hasNext()) { - //closeGraph(graphFromStore); - throw new DuplicateRecordException(Constants.DUPLICATE_RECORD_MESSAGE); - } - String rootNodeLabel; - TinkerGraph graph = (TinkerGraph) entity; - GraphTraversalSource traversal = graph.traversal(); - if (graphFromStore.features().graph().supportsTransactions()) { - org.apache.tinkerpop.gremlin.structure.Transaction tx = graphFromStore.tx(); - tx.onReadWrite(org.apache.tinkerpop.gremlin.structure.Transaction.READ_WRITE_BEHAVIOR.AUTO); - //rootNodeLabel = createOrUpdateEntity(dbGraphTraversalSource, graph, label, "create"); - rootNodeLabel = addOrUpdateVerticesAndEdges(dbGraphTraversalSource, traversal, label, "create"); - tx.commit(); - logger.debug("RegistryDaoImpl : Entity added for transactional DB with rootNodeLabel : {}",rootNodeLabel); - }else{ - rootNodeLabel = addOrUpdateVerticesAndEdges(dbGraphTraversalSource, traversal, label, "create"); - logger.debug("RegistryDaoImpl : Entity added for non-transactional DB with rootNodeLabel : {}",rootNodeLabel); - } - logger.info("Successfully created entity with label " + rootNodeLabel); - // closeGraph(graphFromStore); - return rootNodeLabel; - } - - - @Override - public String addEntity(Graph entity, String label, String rootNodeLabel, String property) throws DuplicateRecordException, RecordNotFoundException, NoSuchElementException, EncryptionException, AuditFailedException{ - logger.debug("RegistryDaoImpl : Database Provider features: \n" + databaseProvider.getGraphStore().features()); - Graph graphFromStore = databaseProvider.getGraphStore(); - GraphTraversalSource dbGraphTraversalSource = graphFromStore.traversal(); - if (rootNodeLabel!=null && property!=null && !dbGraphTraversalSource.clone().V().hasLabel(rootNodeLabel).hasNext()) { - //closeGraph(graphFromStore); - throw new RecordNotFoundException(Constants.ENTITY_NOT_FOUND); - } else if (dbGraphTraversalSource.clone().V().hasLabel(label).hasNext()) { - //closeGraph(graphFromStore); - throw new DuplicateRecordException(Constants.DUPLICATE_RECORD_MESSAGE); - } - - TinkerGraph graph = (TinkerGraph) entity; - GraphTraversalSource traversal = graph.traversal(); - if (graphFromStore.features().graph().supportsTransactions()) { - org.apache.tinkerpop.gremlin.structure.Transaction tx = graphFromStore.tx(); - tx.onReadWrite(org.apache.tinkerpop.gremlin.structure.Transaction.READ_WRITE_BEHAVIOR.AUTO); - //label = createOrUpdateEntity(graph, label, "create"); - label = addOrUpdateVerticesAndEdges(dbGraphTraversalSource, traversal, label, "create"); - if (rootNodeLabel!=null && property!=null){ - connectNodes(rootNodeLabel, label, property); - } - tx.commit(); - logger.debug("RegistryDaoImpl : Entity added for transactional DB with rootNodeLabel : {}, label : {}, property : {}",rootNodeLabel, label, property); - }else{ - label = addOrUpdateVerticesAndEdges(dbGraphTraversalSource, traversal, label, "create"); - logger.debug("RegistryDaoImpl : Entity added for non-transactional DB with rootNodeLabel : {}, label : {}, property : {}",rootNodeLabel, label, property); - if (rootNodeLabel!=null && property!=null){ - connectNodes(rootNodeLabel, label, property); - } - } - logger.info("Successfully created entity with label " + label); - // closeGraph(graphFromStore); - return label; - } + public static final String META = "meta."; + private static Logger logger = LoggerFactory.getLogger(RegistryDaoImpl.class); + private static Logger prefLogger = LoggerFactory.getLogger("PERFORMANCE_INSTRUMENTATION"); + //StopWatch stopWatch = new LoggingStopWatch(); + + @Autowired + private DatabaseProvider databaseProvider; + + @Autowired + private EncryptionService encryptionService; + + @Value("${registry.context.base}") + private String registryContext; + + @Autowired + SchemaConfigurator schemaConfigurator; + + @Autowired + ApplicationContext appContext; + + @Value("${feature.toggling}") + private Boolean featureToggling; + + @Override + public List getEntityList() { + // TODO Auto-generated method stub + return null; + } + + @Override + public String addEntity(Graph entity, String label) throws DuplicateRecordException, NoSuchElementException, EncryptionException, AuditFailedException, RecordNotFoundException { + logger.debug("RegistryDaoImpl : Database Provider features: \n" + databaseProvider.getGraphStore().features()); + StopWatch watch = new StopWatch(); + watch.start("addEntity"); + Graph graphFromStore = databaseProvider.getGraphStore(); + GraphTraversalSource dbGraphTraversalSource = graphFromStore.traversal(); + if (dbGraphTraversalSource.clone().V().hasLabel(label).hasNext()) { + //closeGraph(graphFromStore); + throw new DuplicateRecordException(Constants.DUPLICATE_RECORD_MESSAGE); + } + String rootNodeLabel; + TinkerGraph graph = (TinkerGraph) entity; + GraphTraversalSource traversal = graph.traversal(); + if (graphFromStore.features().graph().supportsTransactions()) { + org.apache.tinkerpop.gremlin.structure.Transaction tx = graphFromStore.tx(); + tx.onReadWrite(org.apache.tinkerpop.gremlin.structure.Transaction.READ_WRITE_BEHAVIOR.AUTO); + //rootNodeLabel = createOrUpdateEntity(dbGraphTraversalSource, graph, label, "create"); + rootNodeLabel = addOrUpdateVerticesAndEdges(dbGraphTraversalSource, traversal, label, "create"); + tx.commit(); + logger.debug("RegistryDaoImpl : Entity added for transactional DB with rootNodeLabel : {}", rootNodeLabel); + } else { + rootNodeLabel = addOrUpdateVerticesAndEdges(dbGraphTraversalSource, traversal, label, "create"); + logger.debug("RegistryDaoImpl : Entity added for non-transactional DB with rootNodeLabel : {}", rootNodeLabel); + } + logger.info("Successfully created entity with label " + rootNodeLabel); + watch.stop(); + prefLogger.info(watch.prettyPrint()); + // closeGraph(graphFromStore); + return rootNodeLabel; + } + + + @Override + public String addEntity(Graph entity, String label, String rootNodeLabel, String property) throws DuplicateRecordException, RecordNotFoundException, NoSuchElementException, EncryptionException, AuditFailedException { + logger.debug("RegistryDaoImpl : Database Provider features: \n" + databaseProvider.getGraphStore().features()); + StopWatch watch = new StopWatch(); + watch.start("addEntity with rootLabel and property"); + Graph graphFromStore = databaseProvider.getGraphStore(); + GraphTraversalSource dbGraphTraversalSource = graphFromStore.traversal(); + if (rootNodeLabel != null && property != null && !dbGraphTraversalSource.clone().V().hasLabel(rootNodeLabel).hasNext()) { + //closeGraph(graphFromStore); + throw new RecordNotFoundException(Constants.ENTITY_NOT_FOUND); + } else if (dbGraphTraversalSource.clone().V().hasLabel(label).hasNext()) { + //closeGraph(graphFromStore); + throw new DuplicateRecordException(Constants.DUPLICATE_RECORD_MESSAGE); + } + + TinkerGraph graph = (TinkerGraph) entity; + GraphTraversalSource traversal = graph.traversal(); + if (graphFromStore.features().graph().supportsTransactions()) { + org.apache.tinkerpop.gremlin.structure.Transaction tx = graphFromStore.tx(); + tx.onReadWrite(org.apache.tinkerpop.gremlin.structure.Transaction.READ_WRITE_BEHAVIOR.AUTO); + //label = createOrUpdateEntity(graph, label, "create"); + label = addOrUpdateVerticesAndEdges(dbGraphTraversalSource, traversal, label, "create"); + if (rootNodeLabel != null && property != null) { + connectNodes(rootNodeLabel, label, property); + } + tx.commit(); + logger.debug("RegistryDaoImpl : Entity added for transactional DB with rootNodeLabel : {}, label : {}, property : {}", rootNodeLabel, label, property); + } else { + label = addOrUpdateVerticesAndEdges(dbGraphTraversalSource, traversal, label, "create"); + logger.debug("RegistryDaoImpl : Entity added for non-transactional DB with rootNodeLabel : {}, label : {}, property : {}", rootNodeLabel, label, property); + if (rootNodeLabel != null && property != null) { + connectNodes(rootNodeLabel, label, property); + } + } + logger.info("Successfully created entity with label " + label); + watch.stop(); + prefLogger.info(watch.prettyPrint()); + // closeGraph(graphFromStore); + return label; + } /* private void closeGraph(Graph graph) { @@ -142,46 +155,46 @@ private void closeGraph(Graph graph) { } */ - private void connectNodes(String rootLabel, String label, String property) throws RecordNotFoundException, NoSuchElementException, EncryptionException, AuditFailedException { - Graph graphFromStore = databaseProvider.getGraphStore(); - GraphTraversalSource traversalSource = graphFromStore.traversal(); - - if (!traversalSource.clone().V().hasLabel(rootLabel).hasNext()) { - // closeGraph(graphFromStore); - throw new RecordNotFoundException(Constants.ENTITY_NOT_FOUND); - } - - if (!traversalSource.clone().V().hasLabel(label).hasNext()) { - // closeGraph(graphFromStore); - throw new RecordNotFoundException(Constants.ENTITY_NOT_FOUND); - } - connectRootToEntity(traversalSource, rootLabel, label, property); + private void connectNodes(String rootLabel, String label, String property) throws RecordNotFoundException, NoSuchElementException, EncryptionException, AuditFailedException { + Graph graphFromStore = databaseProvider.getGraphStore(); + GraphTraversalSource traversalSource = graphFromStore.traversal(); - } - - private void connectRootToEntity(GraphTraversalSource dbTraversalSource, String rootLabel, String label, String property) throws RecordNotFoundException, NoSuchElementException, EncryptionException, AuditFailedException { - GraphTraversal rootGts = dbTraversalSource.clone().V().hasLabel(rootLabel); - GraphTraversal entityGts = dbTraversalSource.clone().V().hasLabel(label); - Vertex rootVertex = rootGts.next(); - Vertex entityVertex = entityGts.next(); - rootVertex.addEdge(property, entityVertex); - AuditRecord record = appContext.getBean(AuditRecord.class); - record - .subject(rootVertex.label()) - .predicate(property) - .oldObject(null) - .newObject(entityVertex.label()) - .record(databaseProvider); - logger.debug("RegistryDaoImpl : Audit record generated of connectRootToEntity for rootLabel : {}, label : {}, property : {}",rootLabel,label,property); - } + if (!traversalSource.clone().V().hasLabel(rootLabel).hasNext()) { + // closeGraph(graphFromStore); + throw new RecordNotFoundException(Constants.ENTITY_NOT_FOUND); + } - /** - * This method is commonly used for both create and update entity - * @param entity - * @param rootLabel - * @throws EncryptionException - * @throws NoSuchElementException - */ + if (!traversalSource.clone().V().hasLabel(label).hasNext()) { + // closeGraph(graphFromStore); + throw new RecordNotFoundException(Constants.ENTITY_NOT_FOUND); + } + connectRootToEntity(traversalSource, rootLabel, label, property); + + } + + private void connectRootToEntity(GraphTraversalSource dbTraversalSource, String rootLabel, String label, String property) throws RecordNotFoundException, NoSuchElementException, EncryptionException, AuditFailedException { + GraphTraversal rootGts = dbTraversalSource.clone().V().hasLabel(rootLabel); + GraphTraversal entityGts = dbTraversalSource.clone().V().hasLabel(label); + Vertex rootVertex = rootGts.next(); + Vertex entityVertex = entityGts.next(); + rootVertex.addEdge(property, entityVertex); + AuditRecord record = appContext.getBean(AuditRecord.class); + record + .subject(rootVertex.label()) + .predicate(property) + .oldObject(null) + .newObject(entityVertex.label()) + .record(databaseProvider); + logger.debug("RegistryDaoImpl : Audit record generated of connectRootToEntity for rootLabel : {}, label : {}, property : {}", rootLabel, label, property); + } + + /** + * This method is commonly used for both create and update entity + * @param entity + * @param rootLabel + * @throws EncryptionException + * @throws NoSuchElementException + */ /*private String createOrUpdateEntity(GraphTraversalSource dbGraphTraversalSource, Graph entity, String rootLabel, String methodOrigin) throws NoSuchElementException, EncryptionException, AuditFailedException, RecordNotFoundException{ @@ -205,134 +218,136 @@ private void connectRootToEntity(GraphTraversalSource dbTraversalSource, String }*/ - /** - * This method creates the root node of the entity if it already isn't present in the graph store - * or updates the properties of the root node or adds new properties if the properties are not already - * present in the node. - * @param dbTraversalSource - * @param entitySource - * @param rootLabel - * @throws EncryptionException - * @throws NoSuchElementException - */ - private String addOrUpdateVerticesAndEdges(GraphTraversalSource dbTraversalSource, - GraphTraversalSource entitySource, String rootLabel, String methodOrigin) - throws NoSuchElementException, EncryptionException, AuditFailedException, RecordNotFoundException { - - GraphTraversal gts = entitySource.clone().V().hasLabel(rootLabel); - String label = rootLabel; - - while (gts.hasNext()) { - Vertex v = gts.next(); - GraphTraversal hasLabel = dbTraversalSource.clone().V().hasLabel(rootLabel); - - if (hasLabel.hasNext()) { - logger.info(String.format("Root node label {} already exists. Updating properties for the root node.", rootLabel)); - Vertex existingVertex = hasLabel.next(); - //if(!methodOrigin.equalsIgnoreCase("upsert")){ - copyProperties(v, existingVertex, methodOrigin); - //} - addOrUpdateVertexAndEdge(v, existingVertex, dbTraversalSource, methodOrigin); - } else { - if(methodOrigin.equalsIgnoreCase("update") && featureToggling){ - throw new RecordNotFoundException(Constants.ENTITY_NOT_FOUND); - } - label = generateBlankNodeLabel(rootLabel); - logger.info(String.format("Creating entity with label {}", rootLabel)); - Vertex newVertex = dbTraversalSource.clone().addV(label).next(); - copyProperties(v, newVertex,methodOrigin); - addOrUpdateVertexAndEdge(v, newVertex, dbTraversalSource,methodOrigin); - } - } - return label; - } - - /** - * This method takes the root node of an entity and then recursively creates or updates child vertices - * and edges. - * @param v - * @param dbVertex - * @param dbGraph - * @throws EncryptionException - * @throws NoSuchElementException - */ - private void addOrUpdateVertexAndEdge(Vertex v, Vertex dbVertex, GraphTraversalSource dbGraph, String methodOrigin) - throws NoSuchElementException, EncryptionException, AuditFailedException, RecordNotFoundException{ - Iterator edges = v.edges(Direction.OUT); - Iterator edgeList = v.edges(Direction.OUT); - Stack> parsedVertices = new Stack<>(); - List dbEdgesForVertex = ImmutableList.copyOf(dbVertex.edges(Direction.OUT)); - List edgeVertexMatchList = new ArrayList(); - while(edgeList.hasNext()) { - Edge e = edgeList.next(); - Vertex ver = e.inVertex(); - String edgeLabel = e.label(); - Optional edgeVertexAlreadyExists = - dbEdgesForVertex.stream().filter(ed -> ed.label().equalsIgnoreCase(edgeLabel) && ed.inVertex().label().equalsIgnoreCase(ver.label())).findFirst(); - if(edgeVertexAlreadyExists.isPresent()){ - edgeVertexMatchList.add(edgeVertexAlreadyExists.get()); - } - } - logger.debug("RegistryDaoImpl : Matching list size:"+edgeVertexMatchList.size()); - - while(edges.hasNext()) { - Edge e = edges.next(); - Vertex ver = e.inVertex(); - String edgeLabel = e.label(); - GraphTraversal gt = dbGraph.clone().V().hasLabel(ver.label()); - Optional edgeAlreadyExists = - dbEdgesForVertex.stream().filter(ed -> ed.label().equalsIgnoreCase(e.label())).findFirst(); - Optional edgeVertexAlreadyExists = - dbEdgesForVertex.stream().filter(ed -> ed.label().equalsIgnoreCase(edgeLabel) && ed.inVertex().label().equalsIgnoreCase(ver.label())).findFirst(); - verifyAndDelete(dbVertex, e, edgeAlreadyExists, edgeVertexMatchList, methodOrigin); - if (gt.hasNext()) { - Vertex existingV = gt.next(); - logger.info(String.format("Vertex with label {} already exists. Updating properties for the vertex", existingV.label())); - copyProperties(ver, existingV,methodOrigin); - if(!edgeVertexAlreadyExists.isPresent()){ - Edge edgeAdded = dbVertex.addEdge(edgeLabel, existingV); - edgeVertexMatchList.add(edgeAdded); - AuditRecord record = appContext.getBean(AuditRecord.class); - record - .subject(dbVertex.label()) - .predicate(e.label()) - .oldObject(null) - .newObject(existingV.label()) - .record(databaseProvider); - - logger.debug("RegistryDaoImpl : Audit record created for update/insert(upsert) with label : {} ", dbVertex.label()); - } - parsedVertices.push(new Pair<>(ver, existingV)); - } else { - if(methodOrigin.equalsIgnoreCase("update") && !isIRI(ver.label()) && featureToggling){ - throw new RecordNotFoundException(Constants.ENTITY_NOT_FOUND); - } - String label = generateBlankNodeLabel(ver.label()); - Vertex newV = dbGraph.addV(label).next(); - logger.debug(String.format("RegistryDaoImpl : Adding vertex with label {} and adding properties", newV.label())); - copyProperties(ver, newV, methodOrigin); - logger.debug(String.format("RegistryDaoImpl : Adding edge with label {} for the vertex label {}.", e.label(), newV.label())); - - Edge edgeAdded = dbVertex.addEdge(edgeLabel, newV); - edgeVertexMatchList.add(edgeAdded); - AuditRecord record = appContext.getBean(AuditRecord.class); - record - .subject(dbVertex.label()) - .predicate(e.label()) - .oldObject(null) - .newObject(newV.label()) - .record(databaseProvider); - - logger.debug("RegistryDaoImpl : Audit record created for update with label : {} ", dbVertex.label()); - parsedVertices.push(new Pair<>(ver, newV)); - } - } - for(Pair pv : parsedVertices) { - addOrUpdateVertexAndEdge(pv.getValue0(), pv.getValue1(), dbGraph,methodOrigin); - } - - - } + /** + * This method creates the root node of the entity if it already isn't present in the graph store + * or updates the properties of the root node or adds new properties if the properties are not already + * present in the node. + * + * @param dbTraversalSource + * @param entitySource + * @param rootLabel + * @throws EncryptionException + * @throws NoSuchElementException + */ + private String addOrUpdateVerticesAndEdges(GraphTraversalSource dbTraversalSource, + GraphTraversalSource entitySource, String rootLabel, String methodOrigin) + throws NoSuchElementException, EncryptionException, AuditFailedException, RecordNotFoundException { + + GraphTraversal gts = entitySource.clone().V().hasLabel(rootLabel); + String label = rootLabel; + + while (gts.hasNext()) { + Vertex v = gts.next(); + GraphTraversal hasLabel = dbTraversalSource.clone().V().hasLabel(rootLabel); + + if (hasLabel.hasNext()) { + logger.info(String.format("Root node label {} already exists. Updating properties for the root node.", rootLabel)); + Vertex existingVertex = hasLabel.next(); + //if(!methodOrigin.equalsIgnoreCase("upsert")){ + copyProperties(v, existingVertex, methodOrigin); + //} + addOrUpdateVertexAndEdge(v, existingVertex, dbTraversalSource, methodOrigin); + } else { + if (methodOrigin.equalsIgnoreCase("update") && featureToggling) { + throw new RecordNotFoundException(Constants.ENTITY_NOT_FOUND); + } + label = generateBlankNodeLabel(rootLabel); + logger.info(String.format("Creating entity with label {}", rootLabel)); + Vertex newVertex = dbTraversalSource.clone().addV(label).next(); + copyProperties(v, newVertex, methodOrigin); + addOrUpdateVertexAndEdge(v, newVertex, dbTraversalSource, methodOrigin); + } + } + return label; + } + + /** + * This method takes the root node of an entity and then recursively creates or updates child vertices + * and edges. + * + * @param v + * @param dbVertex + * @param dbGraph + * @throws EncryptionException + * @throws NoSuchElementException + */ + private void addOrUpdateVertexAndEdge(Vertex v, Vertex dbVertex, GraphTraversalSource dbGraph, String methodOrigin) + throws NoSuchElementException, EncryptionException, AuditFailedException, RecordNotFoundException { + Iterator edges = v.edges(Direction.OUT); + Iterator edgeList = v.edges(Direction.OUT); + Stack> parsedVertices = new Stack<>(); + List dbEdgesForVertex = ImmutableList.copyOf(dbVertex.edges(Direction.OUT)); + List edgeVertexMatchList = new ArrayList(); + while (edgeList.hasNext()) { + Edge e = edgeList.next(); + Vertex ver = e.inVertex(); + String edgeLabel = e.label(); + Optional edgeVertexAlreadyExists = + dbEdgesForVertex.stream().filter(ed -> ed.label().equalsIgnoreCase(edgeLabel) && ed.inVertex().label().equalsIgnoreCase(ver.label())).findFirst(); + if (edgeVertexAlreadyExists.isPresent()) { + edgeVertexMatchList.add(edgeVertexAlreadyExists.get()); + } + } + logger.debug("RegistryDaoImpl : Matching list size:" + edgeVertexMatchList.size()); + + while (edges.hasNext()) { + Edge e = edges.next(); + Vertex ver = e.inVertex(); + String edgeLabel = e.label(); + GraphTraversal gt = dbGraph.clone().V().hasLabel(ver.label()); + Optional edgeAlreadyExists = + dbEdgesForVertex.stream().filter(ed -> ed.label().equalsIgnoreCase(e.label())).findFirst(); + Optional edgeVertexAlreadyExists = + dbEdgesForVertex.stream().filter(ed -> ed.label().equalsIgnoreCase(edgeLabel) && ed.inVertex().label().equalsIgnoreCase(ver.label())).findFirst(); + verifyAndDelete(dbVertex, e, edgeAlreadyExists, edgeVertexMatchList, methodOrigin); + if (gt.hasNext()) { + Vertex existingV = gt.next(); + logger.info(String.format("Vertex with label {} already exists. Updating properties for the vertex", existingV.label())); + copyProperties(ver, existingV, methodOrigin); + if (!edgeVertexAlreadyExists.isPresent()) { + Edge edgeAdded = dbVertex.addEdge(edgeLabel, existingV); + edgeVertexMatchList.add(edgeAdded); + AuditRecord record = appContext.getBean(AuditRecord.class); + record + .subject(dbVertex.label()) + .predicate(e.label()) + .oldObject(null) + .newObject(existingV.label()) + .record(databaseProvider); + + logger.debug("RegistryDaoImpl : Audit record created for update/insert(upsert) with label : {} ", dbVertex.label()); + } + parsedVertices.push(new Pair<>(ver, existingV)); + } else { + if (methodOrigin.equalsIgnoreCase("update") && !isIRI(ver.label()) && featureToggling) { + throw new RecordNotFoundException(Constants.ENTITY_NOT_FOUND); + } + String label = generateBlankNodeLabel(ver.label()); + Vertex newV = dbGraph.addV(label).next(); + logger.debug(String.format("RegistryDaoImpl : Adding vertex with label {} and adding properties", newV.label())); + copyProperties(ver, newV, methodOrigin); + logger.debug(String.format("RegistryDaoImpl : Adding edge with label {} for the vertex label {}.", e.label(), newV.label())); + + Edge edgeAdded = dbVertex.addEdge(edgeLabel, newV); + edgeVertexMatchList.add(edgeAdded); + AuditRecord record = appContext.getBean(AuditRecord.class); + record + .subject(dbVertex.label()) + .predicate(e.label()) + .oldObject(null) + .newObject(newV.label()) + .record(databaseProvider); + + logger.debug("RegistryDaoImpl : Audit record created for update with label : {} ", dbVertex.label()); + parsedVertices.push(new Pair<>(ver, newV)); + } + } + for (Pair pv : parsedVertices) { + addOrUpdateVertexAndEdge(pv.getValue0(), pv.getValue1(), dbGraph, methodOrigin); + } + + + } /*private void deleteEdgeAndNode(Vertex dbVertex, Edge e, Optional edgeAlreadyExists,List edgeVertexMatchList, String methodOrigin) throws AuditFailedException, RecordNotFoundException{ @@ -357,224 +372,250 @@ private void addOrUpdateVertexAndEdge(Vertex v, Vertex dbVertex, GraphTraversalS } }*/ - /** - * This method checks if deletion of edge and node is required based on criteria and invokes deleteEdgeAndNode method - * @param dbSourceVertex - * @param e - * @param edgeAlreadyExists - * @param edgeVertexMatchList - * @param methodOrigin - * @throws AuditFailedException - */ - private void verifyAndDelete(Vertex dbSourceVertex, Edge e, Optional edgeAlreadyExists,List edgeVertexMatchList, String methodOrigin) - throws AuditFailedException{ - boolean isSingleValued = schemaConfigurator.isSingleValued(e.label()); - if((edgeAlreadyExists.isPresent() && methodOrigin.equalsIgnoreCase("update")) || isSingleValued){ - Iterator edgeIter = dbSourceVertex.edges(Direction.OUT, e.label()); - while(edgeIter.hasNext()){ - Edge edge = edgeIter.next(); - Optional existingEdgeVertex = - edgeVertexMatchList.stream().filter(ed -> ed.label().equalsIgnoreCase(edge.label()) && ed.inVertex().label().equalsIgnoreCase(edge.inVertex().label())).findFirst(); - if(!existingEdgeVertex.isPresent()){ - deleteEdgeAndNode(dbSourceVertex, edge, null); - } - } - } - } - - /** - * This method deletes the edge and node if the node is an orphan node and if not, deletes only the edge - * @param v - * @param dbEdgeToBeRemoved - * @param dbVertexToBeDeleted - * @throws AuditFailedException - */ - private void deleteEdgeAndNode(Vertex v, Edge dbEdgeToBeRemoved, Vertex dbVertexToBeDeleted) throws AuditFailedException{ - if(dbVertexToBeDeleted !=null) { - logger.info("Deleting edge and node for edge-label : {} , vertex-label : {}", dbEdgeToBeRemoved.label(), dbVertexToBeDeleted.label()); - }else{ - logger.info("Deleting edge and node for edge-label : {} ", dbEdgeToBeRemoved.label()); - } - - if(dbVertexToBeDeleted == null){ - dbVertexToBeDeleted = dbEdgeToBeRemoved.inVertex(); - } - Iterator inEdgeIter = dbVertexToBeDeleted.edges(Direction.IN); - Iterator outEdgeIter = dbVertexToBeDeleted.edges(Direction.OUT); - if((inEdgeIter.hasNext() && IteratorUtils.count(inEdgeIter) > 1) || outEdgeIter.hasNext()){ - logger.debug("RegistryDaoImpl : Deleting edge only for edge-label: {}",dbEdgeToBeRemoved.label()); - dbEdgeToBeRemoved.remove(); - }else{ - logger.debug("RegistryDaoImpl : Deleting edge and node for edge-label: {} and vertex-label : {}", dbEdgeToBeRemoved.label(), dbVertexToBeDeleted.label()); - dbVertexToBeDeleted.remove(); - dbEdgeToBeRemoved.remove(); - } - AuditRecord record = appContext.getBean(AuditRecord.class); - String tailOfdbVertex=v.label().substring(v.label().lastIndexOf("/") + 1).trim(); - String auditVertexlabel= registryContext+tailOfdbVertex; - record - .subject(auditVertexlabel) - .predicate(dbEdgeToBeRemoved.label()) - .oldObject(dbVertexToBeDeleted.label()) - .newObject(null) - .record(databaseProvider); - logger.debug("RegistryDaoImpl : Audit record created for deletion of vertex : {}", dbVertexToBeDeleted); - - } - - - /** - * Blank nodes are no longer supported. If the input data has a blank node, which is identified - * by the node's label which starts with :_, then a random UUID is used as the label for the blank node. - * @param label - * @return - */ - private String generateBlankNodeLabel(String label) { - if(!isIRI(label)){ - label = String.format("%s%s", registryContext, generateRandomUUID()); - } - return label; - } - - /*private boolean isBlankNode(String label){ - if(label.startsWith("_:")) { - return true; - } - return false; - } + /** + * This method checks if deletion of edge and node is required based on criteria and invokes deleteEdgeAndNode method + * + * @param dbSourceVertex + * @param e + * @param edgeAlreadyExists + * @param edgeVertexMatchList + * @param methodOrigin + * @throws AuditFailedException + */ + private void verifyAndDelete(Vertex dbSourceVertex, Edge e, Optional edgeAlreadyExists, List edgeVertexMatchList, String methodOrigin) + throws AuditFailedException { + boolean isSingleValued = schemaConfigurator.isSingleValued(e.label()); + if ((edgeAlreadyExists.isPresent() && methodOrigin.equalsIgnoreCase("update")) || isSingleValued) { + Iterator edgeIter = dbSourceVertex.edges(Direction.OUT, e.label()); + while (edgeIter.hasNext()) { + Edge edge = edgeIter.next(); + Optional existingEdgeVertex = + edgeVertexMatchList.stream().filter(ed -> ed.label().equalsIgnoreCase(edge.label()) && ed.inVertex().label().equalsIgnoreCase(edge.inVertex().label())).findFirst(); + if (!existingEdgeVertex.isPresent()) { + deleteEdgeAndNode(dbSourceVertex, edge, null); + } + } + } + } + + /** + * This method deletes the edge and node if the node is an orphan node and if not, deletes only the edge + * + * @param v + * @param dbEdgeToBeRemoved + * @param dbVertexToBeDeleted + * @throws AuditFailedException + */ + private void deleteEdgeAndNode(Vertex v, Edge dbEdgeToBeRemoved, Vertex dbVertexToBeDeleted) throws AuditFailedException { + if (dbVertexToBeDeleted != null) { + logger.info("Deleting edge and node for edge-label : {} , vertex-label : {}", dbEdgeToBeRemoved.label(), dbVertexToBeDeleted.label()); + } else { + logger.info("Deleting edge and node for edge-label : {} ", dbEdgeToBeRemoved.label()); + } + + if (dbVertexToBeDeleted == null) { + dbVertexToBeDeleted = dbEdgeToBeRemoved.inVertex(); + } + Iterator inEdgeIter = dbVertexToBeDeleted.edges(Direction.IN); + Iterator outEdgeIter = dbVertexToBeDeleted.edges(Direction.OUT); + if ((inEdgeIter.hasNext() && IteratorUtils.count(inEdgeIter) > 1) || outEdgeIter.hasNext()) { + logger.debug("RegistryDaoImpl : Deleting edge only for edge-label: {}", dbEdgeToBeRemoved.label()); + dbEdgeToBeRemoved.remove(); + } else { + logger.debug("RegistryDaoImpl : Deleting edge and node for edge-label: {} and vertex-label : {}", dbEdgeToBeRemoved.label(), dbVertexToBeDeleted.label()); + dbVertexToBeDeleted.remove(); + dbEdgeToBeRemoved.remove(); + } + AuditRecord record = appContext.getBean(AuditRecord.class); + String tailOfdbVertex = v.label().substring(v.label().lastIndexOf("/") + 1).trim(); + String auditVertexlabel = registryContext + tailOfdbVertex; + record + .subject(auditVertexlabel) + .predicate(dbEdgeToBeRemoved.label()) + .oldObject(dbVertexToBeDeleted.label()) + .newObject(null) + .record(databaseProvider); + logger.debug("RegistryDaoImpl : Audit record created for deletion of vertex : {}", dbVertexToBeDeleted); + + } + + + /** + * Blank nodes are no longer supported. If the input data has a blank node, which is identified + * by the node's label which starts with :_, then a random UUID is used as the label for the blank node. + * + * @param label + * @return + */ + private String generateBlankNodeLabel(String label) { + if (!isIRI(label)) { + label = String.format("%s%s", registryContext, generateRandomUUID()); + } + return label; + } + + /*private boolean isBlankNode(String label){ + if(label.startsWith("_:")) { + return true; + } + return false; + } */ - private boolean isIRI(String label){ - UrlValidator urlValidator = new UrlValidator(UrlValidator.ALLOW_LOCAL_URLS); - if(urlValidator.isValid(label)) { - return true; - } - return false; - } - - public static String generateRandomUUID() { - return UUID.randomUUID().toString(); - } - - @Override - public boolean updateEntity(Graph entityForUpdate, String rootNodeLabel, String methodOrigin) - throws RecordNotFoundException, NoSuchElementException, EncryptionException, AuditFailedException { - Graph graphFromStore = databaseProvider.getGraphStore(); - GraphTraversalSource dbGraphTraversalSource = graphFromStore.traversal(); - TinkerGraph graphForUpdate = (TinkerGraph) entityForUpdate; - GraphTraversalSource traversal = graphForUpdate.traversal(); - // Check if the root node being updated exists in the database - GraphTraversal hasRootLabel = dbGraphTraversalSource.clone().V().hasLabel(rootNodeLabel); - if (!hasRootLabel.hasNext()) { - // closeGraph(graphFromStore); - throw new RecordNotFoundException(Constants.ENTITY_NOT_FOUND); - } else { - if (graphFromStore.features().graph().supportsTransactions()) { - org.apache.tinkerpop.gremlin.structure.Transaction tx = graphFromStore.tx(); - tx.onReadWrite(org.apache.tinkerpop.gremlin.structure.Transaction.READ_WRITE_BEHAVIOR.AUTO); - //createOrUpdateEntity(graphForUpdate, rootNodeLabel, methodOrigin); - addOrUpdateVerticesAndEdges(dbGraphTraversalSource, traversal, rootNodeLabel, methodOrigin); - tx.commit(); - logger.debug("RegistryDaoImpl : Entity Updated for transactional DB with rootNodeLabel : {}",rootNodeLabel); - }else{ - addOrUpdateVerticesAndEdges(dbGraphTraversalSource, traversal, rootNodeLabel, methodOrigin); - logger.debug("RegistryDaoImpl : Entity Updated for non-transactional DB with rootNodeLabel : {}",rootNodeLabel); - //createOrUpdateEntity(graphForUpdate, rootNodeLabel, methodOrigin); - } - //closeGraph(graphFromStore); - } - return false; - } - - - @Override - public Graph getEntityById(String label) throws RecordNotFoundException, NoSuchElementException, EncryptionException, AuditFailedException { - Graph graphFromStore = databaseProvider.getGraphStore(); - GraphTraversalSource traversalSource = graphFromStore.traversal(); - GraphTraversal hasLabel = traversalSource.clone().V().hasLabel(label); - Graph parsedGraph = TinkerGraph.open(); - if (!hasLabel.hasNext()) { + private boolean isIRI(String label) { + UrlValidator urlValidator = new UrlValidator(UrlValidator.ALLOW_LOCAL_URLS); + if (urlValidator.isValid(label)) { + return true; + } + return false; + } + + public static String generateRandomUUID() { + return UUID.randomUUID().toString(); + } + + @Override + public boolean updateEntity(Graph entityForUpdate, String rootNodeLabel, String methodOrigin) + throws RecordNotFoundException, NoSuchElementException, EncryptionException, AuditFailedException { + StopWatch watch = new StopWatch(); + watch.start("Update"); + Graph graphFromStore = databaseProvider.getGraphStore(); + GraphTraversalSource dbGraphTraversalSource = graphFromStore.traversal(); + TinkerGraph graphForUpdate = (TinkerGraph) entityForUpdate; + GraphTraversalSource traversal = graphForUpdate.traversal(); + // Check if the root node being updated exists in the database + GraphTraversal hasRootLabel = dbGraphTraversalSource.clone().V().hasLabel(rootNodeLabel); + if (!hasRootLabel.hasNext()) { // closeGraph(graphFromStore); - logger.info("Record not found for label : {}",label); - throw new RecordNotFoundException(Constants.ENTITY_NOT_FOUND); - } else { - logger.info("Record exists for label : {}",label); - Vertex subject = hasLabel.next(); - Vertex newSubject = parsedGraph.addVertex(subject.label()); - copyProperties(subject, newSubject,"read"); - extractGraphFromVertex(parsedGraph,newSubject,subject); + throw new RecordNotFoundException(Constants.ENTITY_NOT_FOUND); + } else { + if (graphFromStore.features().graph().supportsTransactions()) { + org.apache.tinkerpop.gremlin.structure.Transaction tx = graphFromStore.tx(); + tx.onReadWrite(org.apache.tinkerpop.gremlin.structure.Transaction.READ_WRITE_BEHAVIOR.AUTO); + //createOrUpdateEntity(graphForUpdate, rootNodeLabel, methodOrigin); + addOrUpdateVerticesAndEdges(dbGraphTraversalSource, traversal, rootNodeLabel, methodOrigin); + tx.commit(); + logger.debug("RegistryDaoImpl : Entity Updated for transactional DB with rootNodeLabel : {}", rootNodeLabel); + } else { + addOrUpdateVerticesAndEdges(dbGraphTraversalSource, traversal, rootNodeLabel, methodOrigin); + logger.debug("RegistryDaoImpl : Entity Updated for non-transactional DB with rootNodeLabel : {}", rootNodeLabel); + //createOrUpdateEntity(graphForUpdate, rootNodeLabel, methodOrigin); + } + //closeGraph(graphFromStore); + } + watch.stop(); + logger.info(watch.prettyPrint()); + return false; + } + + + @Override + public Graph getEntityById(String label) throws RecordNotFoundException, NoSuchElementException, EncryptionException, AuditFailedException { + StopWatch watch = new StopWatch(); + watch.start("getEntityById"); + Graph graphFromStore = databaseProvider.getGraphStore(); + GraphTraversalSource traversalSource = graphFromStore.traversal(); + GraphTraversal hasLabel = traversalSource.clone().V().hasLabel(label); + Graph parsedGraph = TinkerGraph.open(); + if (!hasLabel.hasNext()) { + // closeGraph(graphFromStore); + logger.info("Record not found for label : {}", label); + throw new RecordNotFoundException(Constants.ENTITY_NOT_FOUND); + } else { + logger.info("Record exists for label : {}", label); + Vertex subject = hasLabel.next(); + Vertex newSubject = parsedGraph.addVertex(subject.label()); + copyProperties(subject, newSubject, "read"); + extractGraphFromVertex(parsedGraph, newSubject, subject); // closeGraph(graphFromStore); - } - return parsedGraph; - } - - - private void copyProperties(Vertex subject, Vertex newSubject, String methodOrigin) - throws NoSuchElementException, EncryptionException, AuditFailedException { - HashMap> propertyMetaPropertyMap = new HashMap>(); - HashMap metaPropertyMap; - Object propertyValue = null; - Iterator> iter = subject.properties(); - int privatePropertyCount =0; - - while (iter.hasNext()) { - VertexProperty property = iter.next(); - String tailOfPropertyKey = property.key().substring(property.key().lastIndexOf("/") + 1).trim(); - boolean existingEncyptedPropertyKey = tailOfPropertyKey - .substring(0, Math.min(tailOfPropertyKey.length(), 9)).equalsIgnoreCase("encrypted"); - if ((methodOrigin.equalsIgnoreCase("create") || methodOrigin.equalsIgnoreCase("update")) && existingEncyptedPropertyKey) { - property.remove(); - } - if ((methodOrigin.equalsIgnoreCase("create") || methodOrigin.equalsIgnoreCase("update")) && schemaConfigurator.isPrivate(property.key())) { - propertyValue = encryptionService.encrypt(property.value()); - if (!existingEncyptedPropertyKey) { - String encryptedKey = "encrypted" + tailOfPropertyKey; - setProperty(newSubject, property.key().replace(tailOfPropertyKey, encryptedKey), propertyValue, methodOrigin); - logger.debug("RegistryDaoImpl : private property {} encrypted successfully!"+privatePropertyCount++); - } - } else if (methodOrigin.equalsIgnoreCase("read") && schemaConfigurator.isEncrypted(tailOfPropertyKey)) { - propertyValue = encryptionService.decrypt(property.value()); - String decryptedKey = property.key().replace(tailOfPropertyKey, tailOfPropertyKey.substring(9)); - setProperty(newSubject, decryptedKey, propertyValue, methodOrigin); - logger.debug("RegistryDaoImpl : Private property {} decrypted successfully !"+privatePropertyCount++); - } else if (isaMetaProperty(property.key())) { - buildPropertyMetaMap(propertyMetaPropertyMap, property); - } else { - if (!(methodOrigin.equalsIgnoreCase("read") - && property.key().contains("@audit"))) { - setProperty(newSubject, property.key(), property.value(), methodOrigin); - - } - } - setMetaProperty(subject, newSubject, property, methodOrigin); - } - setMetaPropertyFromMap(newSubject, propertyMetaPropertyMap); - } - - private boolean isaMetaProperty(String key) { - return key.startsWith(META); - } - - private void setProperty(Vertex v, String key, Object newValue, String methodOrigin) throws AuditFailedException { - VertexProperty vp = v.property(key); - Object oldValue = vp.isPresent() ? vp.value() : null; - if(oldValue!=null && !methodOrigin.equalsIgnoreCase("update") && !schemaConfigurator.isSingleValued(key)){ - List valueList = new ArrayList(); - if(oldValue instanceof List){ - valueList = (List)oldValue; - } else{ - String valueStr = (String)oldValue; - valueList.add(valueStr); - } - - if(newValue instanceof List){ - valueList.addAll((List)newValue); - } else{ - valueList.add(newValue); - } - newValue = valueList; - } - v.property(key, newValue); + } + watch.stop(); + logger.info(watch.prettyPrint()); + return parsedGraph; + } + + + private void copyProperties(Vertex subject, Vertex newSubject, String methodOrigin) + throws NoSuchElementException, EncryptionException, AuditFailedException { + HashMap> propertyMetaPropertyMap = new HashMap>(); + HashMap metaPropertyMap; + Object propertyValue = null; + Iterator> iter = subject.properties(); + int privatePropertyCount = 0; + StopWatch watch = new StopWatch(); + + while (iter.hasNext()) { + VertexProperty property = iter.next(); + String tailOfPropertyKey = property.key().substring(property.key().lastIndexOf("/") + 1).trim(); + boolean existingEncyptedPropertyKey = tailOfPropertyKey + .substring(0, Math.min(tailOfPropertyKey.length(), 9)).equalsIgnoreCase("encrypted"); + if ((methodOrigin.equalsIgnoreCase("create") || methodOrigin.equalsIgnoreCase("update")) && existingEncyptedPropertyKey) { + property.remove(); + } + if ((methodOrigin.equalsIgnoreCase("create") || methodOrigin.equalsIgnoreCase("update")) && schemaConfigurator.isPrivate(property.key())) { + watch.start("Encryption Service"); + //logger.info("Encryption service start time :" +stopWatch.getStartTime()); + propertyValue = encryptionService.encrypt(property.value()); + //logger.info("Elapsed Time : "+ stopWatch.getElapsedTime()); + watch.stop(); + logger.info("RegistryDaoImpl : Encryption done for property - {}", property.key()); + if (!existingEncyptedPropertyKey) { + String encryptedKey = "encrypted" + tailOfPropertyKey; + setProperty(newSubject, property.key().replace(tailOfPropertyKey, encryptedKey), propertyValue, methodOrigin); + logger.info("RegistryDaoImpl : private property count while Encryption - {} " + privatePropertyCount++); + } + prefLogger.info("Encryption Service took total: " + watch.getTotalTimeSeconds() + "seconds"); + prefLogger.info("Decryption Performance" + watch.prettyPrint()); + } else if (methodOrigin.equalsIgnoreCase("read") && schemaConfigurator.isEncrypted(tailOfPropertyKey)) { + watch.start("Decryption Service"); + //logger.info("Decryption service start time :" +stopWatch.getStartTime()); + propertyValue = encryptionService.decrypt(property.value()); + //logger.info("Elapsed Time : "+ stopWatch.getElapsedTime()); + watch.stop(); + logger.info("RegistryDaoImpl : Decryption done for property - {}", property.key()); + String decryptedKey = property.key().replace(tailOfPropertyKey, tailOfPropertyKey.substring(9)); + setProperty(newSubject, decryptedKey, propertyValue, methodOrigin); + logger.info("RegistryDaoImpl : Private property count while Decryption - {} " + privatePropertyCount++); + logger.info("Decryption Service took total: " + watch.getTotalTimeSeconds() + "seconds"); + prefLogger.info("Decryption Performance" + watch.prettyPrint()); + } else if (isaMetaProperty(property.key())) { + buildPropertyMetaMap(propertyMetaPropertyMap, property); + } else { + if (!(methodOrigin.equalsIgnoreCase("read") + && property.key().contains("@audit"))) { + setProperty(newSubject, property.key(), property.value(), methodOrigin); + + } + } + setMetaProperty(subject, newSubject, property, methodOrigin); + } + setMetaPropertyFromMap(newSubject, propertyMetaPropertyMap); + } + + private boolean isaMetaProperty(String key) { + return key.startsWith(META); + } + + private void setProperty(Vertex v, String key, Object newValue, String methodOrigin) throws AuditFailedException { + VertexProperty vp = v.property(key); + Object oldValue = vp.isPresent() ? vp.value() : null; + if (oldValue != null && !methodOrigin.equalsIgnoreCase("update") && !schemaConfigurator.isSingleValued(key)) { + List valueList = new ArrayList(); + if (oldValue instanceof List) { + valueList = (List) oldValue; + } else { + String valueStr = (String) oldValue; + valueList.add(valueStr); + } + + if (newValue instanceof List) { + valueList.addAll((List) newValue); + } else { + valueList.add(newValue); + } + newValue = valueList; + } + v.property(key, newValue); /*if(schemaConfigurator.isSingleValued(key)){ System.out.println("Printing value for single-valued:"+newValue); v.property(key, newValue); @@ -594,82 +635,82 @@ private void setProperty(Vertex v, String key, Object newValue, String methodOri System.out.println("Printing value for else case:"+newValue); v.property(key, newValue); }*/ - if (!isaMetaProperty(key) && !Objects.equals(oldValue, newValue)) { - GraphTraversal configTraversal = - v.graph().traversal().clone().V().has(T.label, Constants.GRAPH_GLOBAL_CONFIG); - if (configTraversal.hasNext() - && configTraversal.next().property(Constants.PERSISTENT_GRAPH).value().equals(true)) { - - AuditRecord record = appContext.getBean(AuditRecord.class); - record - .subject(v.label()) - .predicate(key) - .oldObject(oldValue) - .newObject(newValue) - .record(databaseProvider); - logger.debug("Audit record created for {} !", v.label()); - } else { - // System.out.println("NOT AUDITING"); - } - } else { - logger.debug("No change found for auditing !"); - } - } - - private void setMetaPropertyFromMap(Vertex newSubject, HashMap> propertyMetaPropertyMap) { - Iterator propertyIter = propertyMetaPropertyMap.entrySet().iterator(); - while(propertyIter.hasNext()){ - Map.Entry pair = (Map.Entry)propertyIter.next(); - logger.info("PROPERTY <- " + pair.getKey()); - HashMap _mpmap = (HashMap) pair.getValue(); - Iterator _mpmapIter = _mpmap.entrySet().iterator(); - while(_mpmapIter.hasNext()) { - Map.Entry _pair = (Map.Entry)_mpmapIter.next(); - logger.info("META PROPERTY <- " + _pair.getKey() + "|" + _pair.getValue()); - newSubject.property(pair.getKey().toString()).property(_pair.getKey().toString(),_pair.getValue().toString()); - } - } - } - - private void setMetaProperty(Vertex subject, Vertex newSubject, VertexProperty property, String methodOrigin) throws AuditFailedException { - if(subject.graph().features().vertex().supportsMetaProperties()) { - Iterator> metaPropertyIter = property.properties(); - while (metaPropertyIter.hasNext()) { - Property metaProperty = metaPropertyIter.next(); - if (newSubject.graph().features().vertex().supportsMetaProperties()) { - newSubject.property(property.key()).property(metaProperty.key(), metaProperty.value()); - } else { - String metaKey = getMetaKey(property, metaProperty); - setProperty(newSubject,metaKey, metaProperty.value(), methodOrigin); - } - } - } - } - - private String getMetaKey(VertexProperty property, Property metaProperty) { - return META + property.key() + "." + metaProperty.key(); - } - - private void buildPropertyMetaMap(HashMap> propertyMetaPropertyMap, VertexProperty property) { - HashMap metaPropertyMap; - logger.debug("RegistryDaoImpl : Meta Property: "+property); - Pattern pattern = Pattern.compile("meta\\.(.*)\\.(.*)"); - Matcher match = pattern.matcher(property.key().toString()); - if(match.find()){ - String _property = match.group(1); - String _meta_property = match.group(2); - logger.debug("RegistryDaoImpl : Matched meta property "+match.group(1)+ " "+match.group(2)); - if(propertyMetaPropertyMap.containsKey(property.key())){ - logger.debug("RegistryDaoImpl : Found in propertyMetaPropertyMap"); - metaPropertyMap = propertyMetaPropertyMap.get(property.key()); - } else { - logger.debug("RegistryDaoImpl : Creating metaPropertyMap in propertyMetaPropertyMap"); - metaPropertyMap = new HashMap<>(); - propertyMetaPropertyMap.put(_property,metaPropertyMap); - } - metaPropertyMap.put(_meta_property,property.value().toString()); - } - } + if (!isaMetaProperty(key) && !Objects.equals(oldValue, newValue)) { + GraphTraversal configTraversal = + v.graph().traversal().clone().V().has(T.label, Constants.GRAPH_GLOBAL_CONFIG); + if (configTraversal.hasNext() + && configTraversal.next().property(Constants.PERSISTENT_GRAPH).value().equals(true)) { + + AuditRecord record = appContext.getBean(AuditRecord.class); + record + .subject(v.label()) + .predicate(key) + .oldObject(oldValue) + .newObject(newValue) + .record(databaseProvider); + logger.debug("Audit record created for {} !", v.label()); + } else { + // System.out.println("NOT AUDITING"); + } + } else { + logger.debug("No change found for auditing !"); + } + } + + private void setMetaPropertyFromMap(Vertex newSubject, HashMap> propertyMetaPropertyMap) { + Iterator propertyIter = propertyMetaPropertyMap.entrySet().iterator(); + while (propertyIter.hasNext()) { + Map.Entry pair = (Map.Entry) propertyIter.next(); + logger.info("PROPERTY <- " + pair.getKey()); + HashMap _mpmap = (HashMap) pair.getValue(); + Iterator _mpmapIter = _mpmap.entrySet().iterator(); + while (_mpmapIter.hasNext()) { + Map.Entry _pair = (Map.Entry) _mpmapIter.next(); + logger.info("META PROPERTY <- " + _pair.getKey() + "|" + _pair.getValue()); + newSubject.property(pair.getKey().toString()).property(_pair.getKey().toString(), _pair.getValue().toString()); + } + } + } + + private void setMetaProperty(Vertex subject, Vertex newSubject, VertexProperty property, String methodOrigin) throws AuditFailedException { + if (subject.graph().features().vertex().supportsMetaProperties()) { + Iterator> metaPropertyIter = property.properties(); + while (metaPropertyIter.hasNext()) { + Property metaProperty = metaPropertyIter.next(); + if (newSubject.graph().features().vertex().supportsMetaProperties()) { + newSubject.property(property.key()).property(metaProperty.key(), metaProperty.value()); + } else { + String metaKey = getMetaKey(property, metaProperty); + setProperty(newSubject, metaKey, metaProperty.value(), methodOrigin); + } + } + } + } + + private String getMetaKey(VertexProperty property, Property metaProperty) { + return META + property.key() + "." + metaProperty.key(); + } + + private void buildPropertyMetaMap(HashMap> propertyMetaPropertyMap, VertexProperty property) { + HashMap metaPropertyMap; + logger.debug("RegistryDaoImpl : Meta Property: " + property); + Pattern pattern = Pattern.compile("meta\\.(.*)\\.(.*)"); + Matcher match = pattern.matcher(property.key().toString()); + if (match.find()) { + String _property = match.group(1); + String _meta_property = match.group(2); + logger.debug("RegistryDaoImpl : Matched meta property " + match.group(1) + " " + match.group(2)); + if (propertyMetaPropertyMap.containsKey(property.key())) { + logger.debug("RegistryDaoImpl : Found in propertyMetaPropertyMap"); + metaPropertyMap = propertyMetaPropertyMap.get(property.key()); + } else { + logger.debug("RegistryDaoImpl : Creating metaPropertyMap in propertyMetaPropertyMap"); + metaPropertyMap = new HashMap<>(); + propertyMetaPropertyMap.put(_property, metaPropertyMap); + } + metaPropertyMap.put(_meta_property, property.value().toString()); + } + } /*@Override public boolean deleteEntity (Graph entity, String rootLabel) throws RecordNotFoundException,AuditFailedException { @@ -747,29 +788,29 @@ public boolean deleteEntity (Graph entity, String rootLabel) throws RecordNotFou }*/ - private void extractGraphFromVertex(Graph parsedGraph,Vertex parsedGraphSubject,Vertex s) - throws NoSuchElementException, EncryptionException, AuditFailedException { - Iterator edgeIter = s.edges(Direction.OUT); - Edge edge; - Stack vStack = new Stack(); - Stack parsedVStack = new Stack(); - while(edgeIter.hasNext()){ - edge = edgeIter.next(); - Vertex o = edge.inVertex(); - Vertex newo = parsedGraph.addVertex(o.label()); - copyProperties(o, newo,"read"); - parsedGraphSubject.addEdge(edge.label(), newo); - vStack.push(o); - parsedVStack.push(newo); - } - Iterator vIterator = vStack.iterator(); - Iterator parsedVIterator = parsedVStack.iterator(); - while(vIterator.hasNext()){ - s = vIterator.next(); - parsedGraphSubject = parsedVIterator.next(); - extractGraphFromVertex(parsedGraph,parsedGraphSubject,s); - } - } + private void extractGraphFromVertex(Graph parsedGraph, Vertex parsedGraphSubject, Vertex s) + throws NoSuchElementException, EncryptionException, AuditFailedException { + Iterator edgeIter = s.edges(Direction.OUT); + Edge edge; + Stack vStack = new Stack(); + Stack parsedVStack = new Stack(); + while (edgeIter.hasNext()) { + edge = edgeIter.next(); + Vertex o = edge.inVertex(); + Vertex newo = parsedGraph.addVertex(o.label()); + copyProperties(o, newo, "read"); + parsedGraphSubject.addEdge(edge.label(), newo); + vStack.push(o); + parsedVStack.push(newo); + } + Iterator vIterator = vStack.iterator(); + Iterator parsedVIterator = parsedVStack.iterator(); + while (vIterator.hasNext()) { + s = vIterator.next(); + parsedGraphSubject = parsedVIterator.next(); + extractGraphFromVertex(parsedGraph, parsedGraphSubject, s); + } + } /*private void dump_graph(Graph parsedGraph, String filename) { try { diff --git a/java/registry/src/main/java/io/opensaber/registry/service/impl/EncryptionServiceImpl.java b/java/registry/src/main/java/io/opensaber/registry/service/impl/EncryptionServiceImpl.java index 61d5b605b..407458d77 100644 --- a/java/registry/src/main/java/io/opensaber/registry/service/impl/EncryptionServiceImpl.java +++ b/java/registry/src/main/java/io/opensaber/registry/service/impl/EncryptionServiceImpl.java @@ -44,7 +44,6 @@ public String encrypt(Object propertyValue) throws EncryptionException { HttpEntity> request = new HttpEntity>(map); try { ResponseEntity response = new RestTemplate().postForEntity(encryptionUri, request, String.class); - logger.info("Property encrypted successfully !"); return response.getBody(); }catch(ResourceAccessException e) { logger.error("ResourceAccessException while connecting enryption service : ", e); @@ -59,14 +58,13 @@ public String encrypt(Object propertyValue) throws EncryptionException { } @Override - public String decrypt(Object propertyValue) throws EncryptionException { - + public String decrypt(Object propertyValue) throws EncryptionException { + MultiValueMap map= new LinkedMultiValueMap(); map.add("value", propertyValue); HttpEntity> request = new HttpEntity>(map); try { ResponseEntity response = new RestTemplate().postForEntity(decryptionUri, request, String.class); - logger.info("Property decrypted successfully !"); return response.getBody(); }catch(ResourceAccessException e) { logger.error("ResourceAccessException while connecting dcryption service : ", e); diff --git a/java/registry/src/main/java/io/opensaber/registry/service/impl/RegistryServiceImpl.java b/java/registry/src/main/java/io/opensaber/registry/service/impl/RegistryServiceImpl.java index 9610ca874..b833330ee 100644 --- a/java/registry/src/main/java/io/opensaber/registry/service/impl/RegistryServiceImpl.java +++ b/java/registry/src/main/java/io/opensaber/registry/service/impl/RegistryServiceImpl.java @@ -89,21 +89,13 @@ public String addEntity(Model rdfModel) throws DuplicateRecordException, EntityC @Override public String addEntity(Model rdfModel, String subject, String property) throws DuplicateRecordException, EntityCreationException, EncryptionException, AuditFailedException, MultipleEntityException, RecordNotFoundException { - try { - String label = getRootLabel(rdfModel); - Graph graph = generateGraphFromRDF(rdfModel); - - // Append _: to the root node label to create the entity as Apache Jena removes the _: for the root node label - // if it is a blank node - logger.debug("RegistryServiceImpl : adding subject : {} with property : {}",subject,property); - return registryDao.addEntity(graph, label, subject, property); + String label = getRootLabel(rdfModel); + Graph graph = generateGraphFromRDF(rdfModel); - } catch (EntityCreationException | EncryptionException | AuditFailedException ex) { - throw ex; - } catch (Exception ex) { - logger.error("RegistryServiceImpl : Exception when creating entity: ", ex); - throw ex; - } + // Append _: to the root node label to create the entity as Apache Jena removes the _: for the root node label + // if it is a blank node + logger.debug("RegistryServiceImpl : adding subject : {} with property : {}", subject, property); + return registryDao.addEntity(graph, label, subject, property); } @Override diff --git a/java/registry/src/main/resources/logback.xml b/java/registry/src/main/resources/logback.xml index 707dfe7d4..50d469929 100644 --- a/java/registry/src/main/resources/logback.xml +++ b/java/registry/src/main/resources/logback.xml @@ -1,8 +1,6 @@ - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n @@ -19,7 +17,18 @@ - %d{HH:mm:ss.SSS} %-4relative [%thread] %-5level %logger{35} - %msg%n + %d{HH:mm:ss.SSS} %-4relative [%thread] %-5level %logger{35} - %msg%n + + + + + logs/performance_instrumentation.log + true + true + + # Pattern of log message for file appender + + %-4relative [%thread] %-5level %logger{35} - %msg%n @@ -29,6 +38,11 @@ + + + + + diff --git a/java/registry/src/test/java/io/opensaber/registry/dao/impl/EncryptionDaoImplTest.java b/java/registry/src/test/java/io/opensaber/registry/dao/impl/EncryptionDaoImplTest.java index de18520a5..04e508de4 100644 --- a/java/registry/src/test/java/io/opensaber/registry/dao/impl/EncryptionDaoImplTest.java +++ b/java/registry/src/test/java/io/opensaber/registry/dao/impl/EncryptionDaoImplTest.java @@ -26,6 +26,8 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.context.SpringBootTest; @@ -75,7 +77,7 @@ @FixMethodOrder(MethodSorters.NAME_ASCENDING) @ActiveProfiles(Constants.TEST_ENVIRONMENT) public class EncryptionDaoImplTest extends RegistryTestBase { - + private static Logger logger = LoggerFactory.getLogger(EncryptionDaoImplTest.class); @Rule public ExpectedException expectedEx = ExpectedException.none(); @@ -108,17 +110,17 @@ public class EncryptionDaoImplTest extends RegistryTestBase { public TestRule watcher = new TestWatcher() { @Override protected void starting(Description description) { - System.out.println("Executing test: " + description.getMethodName()); + logger.debug("Executing test: " + description.getMethodName()); } @Override protected void succeeded(Description description) { - System.out.println("Successfully executed test: " + description.getMethodName()); + logger.debug("Successfully executed test: " + description.getMethodName()); } @Override protected void failed(Throwable e, Description description) { - System.out.println(String.format("Test %s failed. Error message: %s", description.getMethodName(), e.getMessage())); + logger.debug(String.format("Test %s failed. Error message: %s", description.getMethodName(), e.getMessage())); } }; diff --git a/java/registry/src/test/java/io/opensaber/registry/dao/impl/RegistryDaoImplTest.java b/java/registry/src/test/java/io/opensaber/registry/dao/impl/RegistryDaoImplTest.java index 32e67e61f..6a580eb8f 100644 --- a/java/registry/src/test/java/io/opensaber/registry/dao/impl/RegistryDaoImplTest.java +++ b/java/registry/src/test/java/io/opensaber/registry/dao/impl/RegistryDaoImplTest.java @@ -26,6 +26,8 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.context.SpringBootTest; @@ -75,7 +77,7 @@ @FixMethodOrder(MethodSorters.NAME_ASCENDING) @ActiveProfiles(Constants.TEST_ENVIRONMENT) public class RegistryDaoImplTest extends RegistryTestBase { - + private static Logger logger = LoggerFactory.getLogger(RegistryDaoImplTest.class); @Rule public ExpectedException expectedEx = ExpectedException.none(); @@ -111,17 +113,17 @@ public class RegistryDaoImplTest extends RegistryTestBase { public TestRule watcher = new TestWatcher() { @Override protected void starting(Description description) { - System.out.println("Executing test: " + description.getMethodName()); + logger.debug("Executing test: " + description.getMethodName()); } @Override protected void succeeded(Description description) { - System.out.println("Successfully executed test: " + description.getMethodName()); + logger.debug("Successfully executed test: " + description.getMethodName()); } @Override protected void failed(Throwable e, Description description) { - System.out.println(String.format("Test %s failed. Error message: %s", description.getMethodName(), e.getMessage())); + logger.debug(String.format("Test %s failed. Error message: %s", description.getMethodName(), e.getMessage())); } }; @@ -299,14 +301,14 @@ public void test_adding_shared_nodes_with_new_properties() throws DuplicateRecor Model rdfModel = getNewValidRdf(); TinkerGraph graph = TinkerGraph.open(); String rootLabel = updateGraphFromRdf(rdfModel, graph); - System.out.println("PRINTING MODEL TO ADD"); + logger.debug("-------- MODEL TO ADD----------"); printModel(rdfModel); String newEntityResponse; newEntityResponse = registryDao.addEntity(graph, String.format("_:%s", rootLabel)); Graph entity = registryDao.getEntityById(newEntityResponse); - System.out.println("CHECKING AUDIT RECORDS"); + logger.debug("-------- CHECKING AUDIT RECORDS-------"); int count1 = checkIfAuditRecordsAreRight(entity, null); - System.out.println("AUDIT RECORDS "+count1); + logger.debug("--------- AUDIT RECORDS -------"+count1); // Create a new TinkerGraph with the existing jsonld Graph newEntityGraph = TinkerGraph.open(); Model newRdfModel = getNewValidRdf(); @@ -326,7 +328,7 @@ public void test_adding_shared_nodes_with_new_properties() throws DuplicateRecor newEntityResponse = registryDao.addEntity(newEntityGraph, String.format("_:%s", newRootLabel)); entity = registryDao.getEntityById(newEntityResponse); int count2 = checkIfAuditRecordsAreRight(entity, null); - System.out.println("AUDIT RECORDS "+count2); + logger.debug("------- AUDIT RECORDS ------"+count2); String propertyValue = (String) entity.traversal().clone().V() .properties("http://example.com/voc/teacher/1.0.0/districtAlias") .next() @@ -339,7 +341,7 @@ public void test_adding_shared_nodes_with_new_properties() throws DuplicateRecor private void printModel(Model rdfModel) { Iterator iter = rdfModel.listStatements(); while(iter.hasNext()){ - System.out.println(iter.next()); + logger.debug("-------next iterator in printModel() : {} ",iter.next()); } } @@ -519,7 +521,7 @@ public void test_update_single_literal_node() private Model getModelwithOnlyUpdateFacts(Model rdfModel, Model updateRdfModel, List predicatedToExclude) { Model updatedFacts = updateRdfModel.difference(rdfModel); - System.out.println("UPDATED FACTS are:"+updatedFacts); + logger.debug("-------- UPDATED FACTS are: ----------"+updatedFacts); /* Property propertyToRemove; propertyToRemove=ResourceFactory.createProperty("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"); Model updateRdfModelWithoutType = restrictModel(updateRdfModel,propertyToRemove); @@ -556,12 +558,12 @@ private Model getModelwithDeletedFacts(Model rdfModel, Model updateRdfModel, Mod } } } - System.out.println("DELETED FACTS are:"+deletedFacts); + logger.debug("------- DELETED FACTS are: --------"+deletedFacts); return deletedFacts; } private Model restrictModel(Model updateRdfModel,Property property) { - System.out.println("Removing "+property); + logger.debug("--------Removing property : "+property); return updateRdfModel.difference( updateRdfModel.listStatements( null, @@ -590,7 +592,7 @@ private Map> generateUpdateMapFromRDF(Model updateR map.put(subject,innerMap); } } - System.out.println("generateUpdateMapFromRDF "+map); + logger.debug("generateUpdateMapFromRDF "+map); return map; } @@ -608,7 +610,7 @@ private Map> generateUpdateMapFromRDF(Model updateR encounteredSubPred.put(subject.toString(), predicate.toString()); StmtIterator diffIter = deletedFacts.listStatements(subject, predicate, (RDFNode)null); deletedCount = (int)IteratorUtils.count(diffIter); - System.out.println(String.format("Number of facts deleted for %s and %s : %d",subject.toString(), predicate.toString(),deletedCount)); + logger.debug(String.format("Number of facts deleted for %s and %s : %d",subject.toString(), predicate.toString(),deletedCount)); } if(map.containsKey(subject.toString())){ Map innerMap = map.get(subject.toString()); @@ -623,7 +625,7 @@ private Map> generateUpdateMapFromRDF(Model updateR map.put(subject.toString(),innerMap); } } - System.out.println("generateUpdateMapFromRDF "+map); + logger.debug("generateUpdateMapFromRDF "+map); return map; } @@ -846,7 +848,7 @@ private String getJsonldFromGraph(Graph entity, String rootLabel) { try { jsonldOutput = RDFUtil.frameEntity(model); } catch (IOException ex) { - System.out.println("IO Exception = " + ex); + logger.debug("IO Exception = " + ex); } return jsonldOutput; } diff --git a/java/registry/src/test/java/io/opensaber/registry/service/impl/RegistryEncryptionServiceImplTest.java b/java/registry/src/test/java/io/opensaber/registry/service/impl/RegistryEncryptionServiceImplTest.java index 56db6548a..acfed8d8e 100644 --- a/java/registry/src/test/java/io/opensaber/registry/service/impl/RegistryEncryptionServiceImplTest.java +++ b/java/registry/src/test/java/io/opensaber/registry/service/impl/RegistryEncryptionServiceImplTest.java @@ -10,7 +10,6 @@ import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; -import org.mockito.InjectMocks; import org.mockito.Mock; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; @@ -21,7 +20,6 @@ import io.opensaber.registry.config.GenericConfiguration; import io.opensaber.registry.controller.RegistryTestBase; import io.opensaber.registry.middleware.util.Constants; -import io.opensaber.registry.model.AuditRecord; import io.opensaber.registry.schema.config.SchemaConfigurator; import io.opensaber.registry.service.EncryptionService;