[Issue-371]: added support for hard delete entities from registry

java/apitest/src/test/java/e2e/registry/registry.feature

@@ -688,7 +688,7 @@ Feature: Registry api tests
     And path '/v1/metrics'
     When method get
     Then status 200
-    And assert response.birthcertificate.READ == "5"
+    And assert response.birthcertificate.READ == "6"
     And assert response.birthcertificate.UPDATE == "1"
     And assert response.birthcertificate.ADD == "1"
     And assert response.birthcertificate.DELETE == "1"

java/elastic-search/src/main/java/dev/sunbirdrc/elastic/ElasticServiceImpl.java

@@ -4,8 +4,6 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
-import com.jayway.jsonpath.DocumentContext;
-import com.jayway.jsonpath.JsonPath;
 import dev.sunbirdrc.pojos.ComponentHealthInfo;
 import dev.sunbirdrc.pojos.Filter;
 import dev.sunbirdrc.pojos.FilterOperators;
@@ -25,11 +23,13 @@
 import org.apache.http.auth.UsernamePasswordCredentials;
 import org.apache.http.client.CredentialsProvider;
 import org.apache.http.impl.client.BasicCredentialsProvider;
-import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
+import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.index.IndexRequest;
@@ -62,6 +62,7 @@ public class ElasticServiceImpl implements IElasticService {
     private static String userName;
     private static String password;
     private static String defaultScheme;
+    private static boolean isHardDeleteEnabled;
 
     public void setConnectionInfo(String connection) {
         connectionInfo = connection;
@@ -257,13 +258,16 @@ public RestStatus updateEntity(String index, String osid, JsonNode inputEntity)
      */
     @Override
     public RestStatus deleteEntity(String index, String osid) {
-        UpdateResponse response = null;
+        DocWriteResponse response = null;
         try {
             String indexL = index.toLowerCase();
             Map<String, Object> readMap = readEntity(indexL, osid);
-           // Map<String, Object> entityMap = (Map<String, Object>) readMap.get(index);
-            readMap.put(Constants.STATUS_KEYWORD, Constants.STATUS_INACTIVE);
-            response = getClient(indexL).update(new UpdateRequest(indexL, searchType, osid).doc(readMap), RequestOptions.DEFAULT);
+            if (isHardDeleteEnabled) {
+                response = getClient(indexL).delete(new DeleteRequest(indexL, searchType, osid), RequestOptions.DEFAULT);
+            } else {
+                readMap.put(Constants.STATUS_KEYWORD, Constants.STATUS_INACTIVE);
+                response = getClient(indexL).update(new UpdateRequest(indexL, searchType, osid).doc(readMap), RequestOptions.DEFAULT);
+            }
         } catch (NullPointerException | IOException e) {
             logger.error("exception in deleteEntity {}", ExceptionUtils.getStackTrace(e));
             return RestStatus.NOT_FOUND;
@@ -403,4 +407,7 @@ public void setUserName(String userName) {
     public void setPassword(String password) {
         this.password = password;
     }
+    public void setIsHardDeleteEnabled(boolean isHardDeleteEnabled) {
+        ElasticServiceImpl.isHardDeleteEnabled = isHardDeleteEnabled;
+    }
 }

java/registry/src/main/java/dev/sunbirdrc/registry/config/GenericConfiguration.java

@@ -155,6 +155,10 @@ public class GenericConfiguration implements WebMvcConfigurer {
 	private int httpMaxConnections;
 	@Value("${elastic.search.scheme}")
 	private String scheme;
+
+	@Value("${registry.hard_delete_enabled}")
+	private boolean isHardDeleteEnabled;
+
 	@Autowired
 	private DBConnectionInfoMgr dbConnectionInfoMgr;
 	@Autowired
@@ -445,6 +449,7 @@ public IElasticService elasticService() throws IOException {
 			Set<String> indices = new HashSet<>(iDefinitionsManager.getAllKnownDefinitions());
 			indices.add(ATTESTATION_POLICY);
 			elasticService.init(indices);
+			elasticService.setIsHardDeleteEnabled(isHardDeleteEnabled);
 		}
 		return elasticService;
 	}

java/registry/src/main/java/dev/sunbirdrc/registry/controller/RegistryEntityController.java

@@ -166,15 +166,20 @@ public ResponseEntity<Object> deleteEntity(
             checkEntityNameInDefinitionManager(entityName);
             String tag = "RegistryController.delete " + entityName;
             watch.start(tag);
+            JsonNode jsonNode = registryHelper.readEntity(userId, entityName, entityId, false, null, false);
+            if (jsonNode != null && jsonNode.has(entityName) && jsonNode.get(entityName).has(OSSystemFields._osSignedData.name())) {
+                String signedData = jsonNode.get(entityName).get(OSSystemFields._osSignedData.name()).asText();
+                registryHelper.revokeExistingCredentials(entityName, entityId, userId, signedData);
+            }
             Vertex deletedEntity = registryHelper.deleteEntity(entityName, entityId, userId);
             if (deletedEntity != null && deletedEntity.keys().contains(OSSystemFields._osSignedData.name())) {
-                registryHelper.revokeExistingCredentials(entityName, entityId, userId, deletedEntity.value(OSSystemFields._osSignedData.name()));
+                deletedEntity.property(OSSystemFields._osSignedData.name()).toString();
+
             }
             responseParams.setErrmsg("");
             responseParams.setStatus(Response.Status.SUCCESSFUL);
             watch.stop(tag);
             return new ResponseEntity<>(response, HttpStatus.OK);
-
         } catch (RecordNotFoundException e) {
             createSchemaNotFoundResponse(e.getMessage(), responseParams);
             response = new Response(Response.API_ID.DELETE, "ERROR", responseParams);
@@ -436,7 +441,7 @@ private JsonNode getAttestationSignedData(String attestationId, JsonNode node) t
     }
 
     @Nullable
-    private JsonNode getAttestationNode(String attestationId, JsonNode node) {
+    private JsonNode getAttestationNode(String attestationId, JsonNode node) throws AttestationNotFoundException, JsonProcessingException {
         Iterator<JsonNode> iterator = node.iterator();
         JsonNode attestationNode = null;
         while (iterator.hasNext()) {
@@ -445,6 +450,10 @@ private JsonNode getAttestationNode(String attestationId, JsonNode node) {
                 break;
             }
         }
+        assert attestationNode != null;
+        if (attestationNode.get(OSSystemFields._osAttestedData.name()) == null)
+            throw new AttestationNotFoundException();
+        attestationNode = objectMapper.readTree(attestationNode.get(OSSystemFields._osAttestedData.name()).asText());
         return attestationNode;
     }
 
@@ -925,4 +934,4 @@ public ResponseEntity<Object> revokeACredential (
             return new ResponseEntity<>(response,HttpStatus.INTERNAL_SERVER_ERROR);
         }
     }
-}
+}

java/registry/src/main/java/dev/sunbirdrc/registry/dao/IRegistryDao.java

@@ -4,7 +4,6 @@
 import dev.sunbirdrc.registry.util.ReadConfigurator;
 import org.apache.tinkerpop.gremlin.structure.Graph;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
-
 public interface IRegistryDao {
 
 	String addEntity(Graph graph, JsonNode rootNode);
@@ -13,4 +12,5 @@ public interface IRegistryDao {
 	void updateVertex(Graph graph, Vertex rootVertex, JsonNode inputJsonNode, String parentName) throws Exception;
     void deleteEntity(Vertex uuid);
 
+	void hardDeleteEntity(Vertex vertex);
 }

java/registry/src/main/java/dev/sunbirdrc/registry/dao/RegistryDaoImpl.java

@@ -8,6 +8,7 @@
 import dev.sunbirdrc.registry.util.IDefinitionsManager;
 import dev.sunbirdrc.registry.util.ReadConfigurator;
 import dev.sunbirdrc.registry.util.TypePropertyHelper;
+import org.apache.tinkerpop.gremlin.structure.Direction;
 import org.apache.tinkerpop.gremlin.structure.Graph;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 import org.slf4j.Logger;
@@ -145,4 +146,15 @@ public void deleteEntity(Vertex vertex) {
             logger.error("Can't mark delete - Null vertex passed");
         }
     }
+    @Override
+    public void hardDeleteEntity(Vertex vertex) {
+        if (vertex != null) {
+            vertex.edges(Direction.OUT).forEachRemaining(d -> {
+                this.hardDeleteEntity(d.inVertex());
+            });
+            vertex.remove();
+        } else {
+            logger.error("Can't delete - Null vertex passed");
+        }
+    }
 }

java/registry/src/main/java/dev/sunbirdrc/registry/helper/RegistryHelper.java

@@ -174,6 +174,9 @@ public class RegistryHelper {
     @Value("${view_template.decrypt_private_fields:false}")
     private boolean viewTemplateDecryptPrivateFields;
 
+    @Value("${registry.hard_delete_enabled}")
+    private boolean isHardDeleteEnabled;
+
     @Autowired
     private EntityTypeHandler entityTypeHandler;
 
@@ -1056,9 +1059,15 @@ public Vertex deleteEntity(String entityName, String entityId, String userId) th
         String shardId = dbConnectionInfoMgr.getShardId(recordId.getShardLabel());
         Shard shard = shardManager.activateShard(shardId);
         ReadConfigurator configurator = ReadConfiguratorFactory.getOne(false);
+        JsonNode deletedNode = null;
+        if(isHardDeleteEnabled) {
+            deletedNode = readEntity(userId, entityName, entityId, false, null, false);
+        }
         Vertex vertex = registryService.deleteEntityById(shard, entityName, userId, recordId.getUuid());
-        VertexReader vertexReader = new VertexReader(shard.getDatabaseProvider(), vertex.graph(), configurator, uuidPropertyName, definitionsManager, true);
-        JsonNode deletedNode = JsonNodeFactory.instance.objectNode().set(entityName, vertexReader.constructObject(vertex));
+        if (!isHardDeleteEnabled) {
+            VertexReader vertexReader = new VertexReader(shard.getDatabaseProvider(), vertex.graph(), configurator, uuidPropertyName, definitionsManager, true);
+            deletedNode = JsonNodeFactory.instance.objectNode().set(entityName, vertexReader.constructObject(vertex));
+        }
         if(notificationEnabled) notificationHelper.sendNotification(deletedNode, DELETE);
         return vertex;
     }

java/registry/src/main/java/dev/sunbirdrc/registry/service/NotificationHelper.java

@@ -16,6 +16,7 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.jmx.export.notification.UnableToSendNotificationException;
 import org.springframework.stereotype.Service;
 
 import java.io.IOException;
@@ -49,6 +50,9 @@ public NotificationHelper() {
 
     public void sendNotification(JsonNode inputJson, String operationType) throws Exception {
         if (!notificationEnabled) return;
+        if(inputJson == null) {
+            throw new UnableToSendNotificationException("Notification input is null for action " + operationType);
+        }
         String entityType = inputJson.fields().next().getKey();
         List<NotificationTemplate> templates = getNotificationTemplate(entityType, operationType);
         Map<String, Object> objectNodeMap = (Map<String, Object>) JSONUtil.convertJsonNodeToMap(inputJson).get(entityType);

java/registry/src/main/java/dev/sunbirdrc/registry/service/impl/RegistryServiceImpl.java

@@ -9,7 +9,6 @@
 import com.jayway.jsonpath.DocumentContext;
 import com.jayway.jsonpath.JsonPath;
 import dev.sunbirdrc.actors.factory.MessageFactory;
-import dev.sunbirdrc.elastic.IElasticService;
 import dev.sunbirdrc.pojos.ComponentHealthInfo;
 import dev.sunbirdrc.pojos.HealthCheckResponse;
 import dev.sunbirdrc.pojos.HealthIndicator;
@@ -22,8 +21,8 @@
 import dev.sunbirdrc.registry.middleware.util.Constants;
 import dev.sunbirdrc.registry.middleware.util.JSONUtil;
 import dev.sunbirdrc.registry.middleware.util.OSSystemFields;
-import dev.sunbirdrc.registry.model.event.Event;
 import dev.sunbirdrc.registry.model.EventType;
+import dev.sunbirdrc.registry.model.event.Event;
 import dev.sunbirdrc.registry.service.*;
 import dev.sunbirdrc.registry.sink.DatabaseProvider;
 import dev.sunbirdrc.registry.sink.OSGraph;
@@ -38,7 +37,6 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.beans.factory.annotation.Value;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.kafka.core.KafkaTemplate;
 import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Service;
@@ -78,6 +76,9 @@ public class RegistryServiceImpl implements RegistryService {
     @Value("${encryption.enabled}")
     private boolean encryptionEnabled;
 
+    @Value("${registry.hard_delete_enabled}")
+    private boolean isHardDeleteEnabled;
+
     @Value("${event.enabled}")
     private boolean isEventsEnabled;
 
@@ -183,9 +184,13 @@ public Vertex deleteEntityById(Shard shard, String entityName, String userId, St
                 if (!StringUtils.isEmpty(index) && index.equals(Schema)) {
                     schemaService.deleteSchemaIfExists(vertex);
                 }
-                if (!(vertex.property(Constants.STATUS_KEYWORD).isPresent()
+                if (isHardDeleteEnabled || !(vertex.property(Constants.STATUS_KEYWORD).isPresent()
                         && vertex.property(Constants.STATUS_KEYWORD).value().equals(Constants.STATUS_INACTIVE))) {
-                    registryDao.deleteEntity(vertex);
+                    if (isHardDeleteEnabled) {
+                        registryDao.hardDeleteEntity(vertex);
+                    } else {
+                        registryDao.deleteEntity(vertex);
+                    }
                     databaseProvider.commitTransaction(graph, tx);
                     auditService.auditDelete(
                             auditService.createAuditRecord(userId, uuid, tx, index),

java/registry/src/main/resources/application.yml

@@ -79,6 +79,7 @@ registry:
   redis:
     host: ${redis_host:localhost}
     port: ${redis_port:6379}
+  hard_delete_enabled: ${hard_delete_enabled:false}
   expandReference: ${expand_reference:false}
 workflow:
   enabled: ${workflow.enable:true}

java/sunbirdrc-actors/src/main/java/dev/sunbirdrc/actors/ElasticSearchActor.java

@@ -5,7 +5,6 @@
 import dev.sunbirdrc.elastic.ESMessage;
 import dev.sunbirdrc.elastic.ElasticServiceImpl;
 import dev.sunbirdrc.elastic.IElasticService;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.sunbird.akka.core.BaseActor;
 import org.sunbird.akka.core.MessageProtos;