Merge pull request #277 from Sreejit-K/reference_entity_add_vertex

Reference entity add vertex

Author: srprasanna

Makefile

@@ -80,4 +80,4 @@ release: test
     	  docker push $$image:latest;\
     	  docker push $$image:$(RELEASE_VERSION);\
       	done
-	@cd tools/cli/ && npm publish
+	@cd tools/cli/ && npm publish

docker-compose.yml

@@ -101,6 +101,7 @@ services:
       - redis_host=redis
       - redis_port=6379
       - manager_type=${MANAGER_TYPE-DefinitionsManager}
+      - expand_reference=${EXPAND_REFERENCE-false}
       - notification_async_enabled=${NOTIFICATION_ASYNC_ENABLED-false}
       - notification_enabled=${NOTIFICATION_ENABLED-false}
       - notification_url=${NOTIFICATION_URL-http://notification-ms:8765/notification-service/v1/notification}

java/apitest/src/test/java/e2e/registry/inviteFlow.json

@@ -11,13 +11,14 @@
   },
   "instituteSchema": {
     "name": "Institute",
-    "schema": "{\n    \"$schema\": \"http://json-schema.org/draft-07/schema\",\n    \"type\": \"object\",\n    \"properties\": {\n      \"Institute\": {\n        \"$ref\": \"#/definitions/Institute\"\n      }\n    },\n    \"required\": [\n      \"Institute\"\n    ],\n    \"title\":\"Institute\",\n    \"definitions\": {\n      \"Institute\": {\n        \"$id\": \"#/properties/Institute\",\n        \"type\": \"object\",\n        \"title\": \"The Institute Schema\",\n        \"required\": [\n  \n        ],\n        \"properties\": {\n          \"name\": {\n            \"type\": \"string\"\n          },\n          \"email\": {\n            \"type\": \"string\"\n          },\n          \"mobile\": {\n            \"type\": \"string\"\n          },\n          \"address\": {\n            \"type\": \"array\", \n            \"items\": {\n              \"type\": \"object\",\n              \"title\": \"Address\",\n              \"required\": [],\n              \"properties\": {\n                \"addressLine1\": {\n                    \"type\": \"string\",\n                    \"title\": \"Address Line1\"\n                },\n                \"postalCode\": {\n                    \"type\": \"string\",\n                    \"title\": \"Postal Code\"\n                },\n                \"phoneNo\": {\n                   \"type\": \"array\",\n                   \"items\": {\n                      \"type\": \"string\"\n                   }\n                }\n             }\n            }\n          }\n        }\n      }\n    },\n    \"_osConfig\": {\n      \"osComment\": [\"This section contains the OpenSABER specific configuration information\",\n        \"privateFields: Optional; list of field names to be encrypted and stored in database\",\n        \"signedFields: Optional; list of field names that must be pre-signed\",\n        \"indexFields: Optional; list of field names used for creating index. Enclose within braces to indicate it is a composite index. In this definition, (serialNum, studentCode) is a composite index and studentName is a single column index.\",\n        \"uniqueIndexFields: Optional; list of field names used for creating unique index. Field names must be different from index field name\",\n        \"systemFields: Optional; list of fields names used for system standard information like created, updated timestamps and userid\"],\n  \n      \"privateFields\": [\n  \n      ],\n      \"signedFields\": [],\n      \"indexFields\": [],\n      \"uniqueIndexFields\": [],\n      \"systemFields\": [\n        \"_osCreatedAt\",\n        \"_osUpdatedAt\",\n        \"_osCreatedBy\",\n        \"_osUpdatedBy\",\n        \"_osAttestedData\",\n        \"_osClaimId\",\n        \"_osState\"\n      ],\n      \"inviteRoles\":[\"Board\"],\n      \"enableLogin\": true,\n      \"ownershipAttributes\": [\n        {\n          \"email\": \"/email\",\n          \"mobile\": \"/mobile\",\n          \"userId\": \"/mobile\"\n        }\n      ]\n    }\n  }",
+    "schema": "{\n    \"_osConfig\": {\n        \"enableLogin\": true,\n        \"indexFields\": [\n        ],\n        \"inviteRoles\": [\n            \"Board\"\n        ],\n        \"osComment\": [\n            \"This section contains the OpenSABER specific configuration information\",\n            \"privateFields: Optional; list of field names to be encrypted and stored in database\",\n            \"signedFields: Optional; list of field names that must be pre-signed\",\n            \"indexFields: Optional; list of field names used for creating index. Enclose within braces to indicate it is a composite index. In this definition, (serialNum, studentCode) is a composite index and studentName is a single column index.\",\n            \"uniqueIndexFields: Optional; list of field names used for creating unique index. Field names must be different from index field name\",\n            \"systemFields: Optional; list of fields names used for system standard information like created, updated timestamps and userid\"\n        ],\n        \"ownershipAttributes\": [\n            {\n                \"email\": \"/email\",\n                \"mobile\": \"/mobile\",\n                \"userId\": \"/mobile\"\n            }\n        ],\n        \"privateFields\": [\n        ],\n        \"signedFields\": [\n        ],\n        \"systemFields\": [\n            \"_osCreatedAt\",\n            \"_osUpdatedAt\",\n            \"_osCreatedBy\",\n            \"_osUpdatedBy\",\n            \"_osAttestedData\",\n            \"_osClaimId\",\n            \"_osState\"\n        ],\n        \"uniqueIndexFields\": [\n        ]\n    },\n    \"$schema\": \"http://json-schema.org/draft-07/schema\",\n    \"definitions\": {\n        \"Institute\": {\n            \"$id\": \"#/properties/Institute\",\n            \"properties\": {\n                \"address\": {\n                    \"items\": {\n                        \"properties\": {\n                            \"addressLine1\": {\n                                \"title\": \"Address Line1\",\n                                \"type\": \"string\"\n                            },\n                            \"phoneNo\": {\n                                \"items\": {\n                                    \"type\": \"string\"\n                                },\n                                \"type\": \"array\"\n                            },\n                            \"postalCode\": {\n                                \"title\": \"Postal Code\",\n                                \"type\": \"string\"\n                            }\n                        },\n                        \"required\": [\n                        ],\n                        \"title\": \"Address\",\n                        \"type\": \"object\"\n                    },\n                    \"type\": \"array\"\n                },\n                \"email\": {\n                    \"type\": \"string\"\n                },\n                \"mobile\": {\n                    \"type\": \"string\"\n                },\n                \"name\": {\n                    \"type\": \"string\"\n                },\n                \"references\": {\n                    \"type\": \"string\"\n                }\n            },\n            \"required\": [\n            ],\n            \"title\": \"The Institute Schema\",\n            \"type\": \"object\"\n        }\n    },\n    \"properties\": {\n        \"Institute\": {\n            \"$ref\": \"#/definitions/Institute\"\n        }\n    },\n    \"required\": [\n        \"Institute\"\n    ],\n    \"title\": \"Institute\",\n    \"type\": \"object\"\n}",
     "status": "PUBLISHED"
   },
   "instituteRequest": {
     "name": "test_institute",
     "email": "[email protected]",
     "mobile": "456",
+    "references": "did:Board:123",
     "address": [{
       "addressLine": "test_address",
       "postalCode": "111",

java/apitest/src/test/java/e2e/registry/registry.feature

@@ -295,6 +295,7 @@ Feature: Registry api tests
     And request sample.boardInviteRequest
     When method post
     Then status 200
+    * def boardOsid = response.result.Board.osid
   #  get board token
     * url authUrl
     * path 'auth/realms/sunbird-rc/protocol/openid-connect/token'
@@ -320,7 +321,9 @@ Feature: Registry api tests
   # invite institute with token
     Given url registryUrl
     And path 'api/v1/Institute/invite'
-    And request sample.instituteRequest
+    * def requestBody = sample.instituteRequest
+    * requestBody.references = 'did:Board:' + boardOsid
+    And request requestBody
     And header Authorization = board_token
     When method post
     Then status 200

java/middleware-commons/src/main/java/dev/sunbirdrc/registry/middleware/util/Constants.java

@@ -127,6 +127,7 @@ public class Constants {
 	public static final String MOBILE = "mobile";
 	public static final String SVG_MEDIA_TYPE = "image/svg+xml";
 	public static final String CONNECTION_FAILURE = "CONNECTION_FAILURE";
+	public static final String DID_TYPE = "did";
 
     public enum GraphDatabaseProvider {
 		NEO4J("NEO4J"), ORIENTDB("ORIENTDB"), SQLG("SQLG"), CASSANDRA("CASSANDRA"), TINKERGRAPH("TINKERGRAPH");

java/registry/src/main/java/dev/sunbirdrc/registry/dao/RegistryDaoImpl.java

@@ -17,6 +17,7 @@
 
 public class RegistryDaoImpl implements IRegistryDao {
     public String uuidPropertyName;
+    private final boolean expandReferenceObj;
     private IDefinitionsManager definitionsManager;
     private DatabaseProvider databaseProvider;
     private List<String> privatePropertyList;
@@ -33,10 +34,11 @@ public void setPrivatePropertyList(List<String> privatePropertyList) {
         this.privatePropertyList = privatePropertyList;
     }
 
-    public RegistryDaoImpl(DatabaseProvider dbProvider, IDefinitionsManager defnManager, String uuidPropName) {
+    public RegistryDaoImpl(DatabaseProvider dbProvider, IDefinitionsManager defnManager, String uuidPropName, boolean expandReferenceObj) {
         databaseProvider = dbProvider;
         definitionsManager = defnManager;
         uuidPropertyName = uuidPropName;
+        this.expandReferenceObj = expandReferenceObj;
     }
 
     public DatabaseProvider getDatabaseProvider() {
@@ -64,7 +66,7 @@ public String addEntity(Graph graph, JsonNode rootNode) {
      */
     public JsonNode getEntity(Graph graph, String entityType, String uuid, ReadConfigurator readConfigurator) throws Exception {
 
-        VertexReader vr = new VertexReader(getDatabaseProvider(), graph, readConfigurator, uuidPropertyName, definitionsManager);
+        VertexReader vr = new VertexReader(getDatabaseProvider(), graph, readConfigurator, uuidPropertyName, definitionsManager, expandReferenceObj);
         JsonNode result = vr.read(entityType, uuid);
 
         return result;
@@ -73,7 +75,7 @@ public JsonNode getEntity(Graph graph, String entityType, String uuid, ReadConfi
 
     public JsonNode getEntity(Graph graph, Vertex vertex, ReadConfigurator readConfigurator, boolean expandInternal) throws Exception {
 
-        VertexReader vr = new VertexReader(getDatabaseProvider(), graph, readConfigurator, uuidPropertyName, definitionsManager);
+        VertexReader vr = new VertexReader(getDatabaseProvider(), graph, readConfigurator, uuidPropertyName, definitionsManager, expandReferenceObj);
         ObjectNode constructObject = vr.constructObject(vertex);
         if (expandInternal) {
             String entityType = (String) ValueType.getValue(constructObject.get(TypePropertyHelper.getTypeName()));

java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexReader.java

@@ -8,12 +8,7 @@
 import dev.sunbirdrc.registry.middleware.util.Constants;
 import dev.sunbirdrc.registry.middleware.util.JSONUtil;
 import dev.sunbirdrc.registry.sink.DatabaseProvider;
-import dev.sunbirdrc.registry.util.ArrayHelper;
-import dev.sunbirdrc.registry.util.Definition;
-import dev.sunbirdrc.registry.util.IDefinitionsManager;
-import dev.sunbirdrc.registry.util.ReadConfigurator;
-import dev.sunbirdrc.registry.util.RefLabelHelper;
-import dev.sunbirdrc.registry.util.TypePropertyHelper;
+import dev.sunbirdrc.registry.util.*;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
 import org.apache.tinkerpop.gremlin.structure.Direction;
 import org.apache.tinkerpop.gremlin.structure.Graph;
@@ -32,6 +27,8 @@
 import java.util.NoSuchElementException;
 import java.util.Set;
 
+import static dev.sunbirdrc.registry.middleware.util.Constants.DID_TYPE;
+
 /**
  * Given a vertex from the graph, constructs a json out it
  */
@@ -46,15 +43,18 @@ public class VertexReader {
     private Vertex rootVertex;
     private LinkedHashMap<String, Vertex> uuidVertexMap = new LinkedHashMap<>();
 
+    private boolean expandReferenceObj;
     private Logger logger = LoggerFactory.getLogger(VertexReader.class);
 
+
     public VertexReader(DatabaseProvider databaseProvider, Graph graph, ReadConfigurator configurator, String uuidPropertyName,
-                        IDefinitionsManager definitionsManager) {
+                        IDefinitionsManager definitionsManager, boolean expandReferenceObj) {
         this.databaseProvider = databaseProvider;
         this.graph = graph;
         this.configurator = configurator;
         this.uuidPropertyName = uuidPropertyName;
         this.definitionsManager = definitionsManager;
+        this.expandReferenceObj = expandReferenceObj;
     }
 
     /**
@@ -460,7 +460,7 @@ public JsonNode readInternal(Vertex rootVertex) throws Exception {
             throw new RecordNotFoundException("entity status is inactive");
         }
         ObjectNode rootNode = constructObject(rootVertex);
-        entityType = (String) ValueType.getValue(rootNode.get(TypePropertyHelper.getTypeName()));
+        String entityType = (String) ValueType.getValue(rootNode.get(TypePropertyHelper.getTypeName()));
 
         // Set the type for the root node, so as to wrap.
         populateMaps(rootNode, rootVertex);
@@ -485,14 +485,42 @@ public JsonNode readInternal(Vertex rootVertex) throws Exception {
         // objects.
         // The properties could exist anywhere. Refer to the local arrMap.
         expandChildObject(rootNode, 0);
-
+        if(expandReferenceObj)
+            expandReferenceNodes(rootNode);
         entityNode.set(entityType, rootNode);
 
+
         // After reading the entire type, now trim the @type property
         trimAttributes(entityNode);
         return entityNode;
     }
 
+    private void expandReferenceNodes(ObjectNode rootNode) {
+        rootNode.fields().forEachRemaining(entry -> {
+            // Regex pattern to check for a DID
+            String pattern = "^"+ DID_TYPE+":[^:]+:[^:]+";
+            if(entry.getValue().isValueNode() && entry.getValue().asText().matches(pattern)) {
+                String[] dids = entry.getValue().asText().split(":");
+                String osid = RecordIdentifier.parse(dids[2]).getUuid();
+                Iterator<Vertex> vertexIterator = graph.traversal().clone().V().hasLabel(dids[1]).has(uuidPropertyName, osid);
+                while (vertexIterator.hasNext()) {
+                    Vertex dependent = vertexIterator.next();
+                    ObjectNode references;
+                    try {
+                        references = (ObjectNode) readInternal(dependent);
+                        references = (ObjectNode) references.get(references.fieldNames().next());
+                        references = (ObjectNode) JSONUtil.removeNodesByPath(references, definitionsManager.getExcludingFieldsForEntity(dids[1]));
+                    } catch (Exception e) {
+                        throw new RuntimeException(e);
+                    }
+                    if(references != null) {
+                        entry.setValue(references);
+                    }
+                }
+            }
+        });
+    }
+
     /**
      * Trims out local helper attributes like the type, uuid depending on the
      * ReadConfigurator

java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexWriter.java

@@ -2,10 +2,12 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import dev.sunbirdrc.registry.middleware.util.Constants;
 import dev.sunbirdrc.registry.sink.DatabaseProvider;
 import dev.sunbirdrc.registry.util.ArrayHelper;
+import dev.sunbirdrc.registry.util.RecordIdentifier;
 import dev.sunbirdrc.registry.util.RefLabelHelper;
 import dev.sunbirdrc.registry.util.TypePropertyHelper;
 import org.apache.tinkerpop.gremlin.process.traversal.P;
@@ -16,12 +18,15 @@
 import org.apache.tinkerpop.gremlin.structure.VertexProperty;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
 
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
+import static dev.sunbirdrc.registry.middleware.util.Constants.DID_TYPE;
+
 /**
  * Helps in writing a vertex, edge into the database
  */
@@ -31,7 +36,6 @@ public class VertexWriter {
     private DatabaseProvider databaseProvider;
     private String parentOSid;
     private static final String EMPTY_STR = "";
-
     private Logger logger = LoggerFactory.getLogger(VertexWriter.class);
 
     public VertexWriter(Graph graph, DatabaseProvider databaseProvider, String uuidPropertyName) {
@@ -215,7 +219,6 @@ private Vertex processNode(String label, JsonNode jsonObject) {
         jsonObject.fields().forEachRemaining(entry -> {
             JsonNode entryValue = entry.getValue();
             logger.debug("Processing {} -> {}", entry.getKey(), entry.getValue());
-
             if (entryValue.isValueNode()) {
                 // Directly add under the vertex as a property
                 vertex.property(entry.getKey(), ValueType.getValue(entryValue));

java/registry/src/main/java/dev/sunbirdrc/registry/helper/RegistryHelper.java

@@ -1057,7 +1057,7 @@ public Vertex deleteEntity(String entityName, String entityId, String userId) th
         Shard shard = shardManager.activateShard(shardId);
         ReadConfigurator configurator = ReadConfiguratorFactory.getOne(false);
         Vertex vertex = registryService.deleteEntityById(shard, entityName, userId, recordId.getUuid());
-        VertexReader vertexReader = new VertexReader(shard.getDatabaseProvider(), vertex.graph(), configurator, uuidPropertyName, definitionsManager);
+        VertexReader vertexReader = new VertexReader(shard.getDatabaseProvider(), vertex.graph(), configurator, uuidPropertyName, definitionsManager, true);
         JsonNode deletedNode = JsonNodeFactory.instance.objectNode().set(entityName, vertexReader.constructObject(vertex));
         if(notificationEnabled) notificationHelper.sendNotification(deletedNode, DELETE);
         return vertex;