diff --git a/java/apitest/src/test/java/e2e/registry/registry.feature b/java/apitest/src/test/java/e2e/registry/registry.feature index 110866f94..79e3678a1 100644 --- a/java/apitest/src/test/java/e2e/registry/registry.feature +++ b/java/apitest/src/test/java/e2e/registry/registry.feature @@ -688,7 +688,7 @@ Feature: Registry api tests And path '/v1/metrics' When method get Then status 200 - And assert response.birthcertificate.READ == "5" + And assert response.birthcertificate.READ == "6" And assert response.birthcertificate.UPDATE == "1" And assert response.birthcertificate.ADD == "1" And assert response.birthcertificate.DELETE == "1" diff --git a/java/elastic-search/src/main/java/dev/sunbirdrc/elastic/ElasticServiceImpl.java b/java/elastic-search/src/main/java/dev/sunbirdrc/elastic/ElasticServiceImpl.java index d866103ce..7eda5e32c 100644 --- a/java/elastic-search/src/main/java/dev/sunbirdrc/elastic/ElasticServiceImpl.java +++ b/java/elastic-search/src/main/java/dev/sunbirdrc/elastic/ElasticServiceImpl.java @@ -4,8 +4,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; -import com.jayway.jsonpath.DocumentContext; -import com.jayway.jsonpath.JsonPath; import dev.sunbirdrc.pojos.ComponentHealthInfo; import dev.sunbirdrc.pojos.Filter; import dev.sunbirdrc.pojos.FilterOperators; @@ -25,11 +23,13 @@ import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; import org.apache.http.impl.client.BasicCredentialsProvider; -import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; +import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; @@ -62,6 +62,7 @@ public class ElasticServiceImpl implements IElasticService { private static String userName; private static String password; private static String defaultScheme; + private static boolean isHardDeleteEnabled; public void setConnectionInfo(String connection) { connectionInfo = connection; @@ -257,13 +258,16 @@ public RestStatus updateEntity(String index, String osid, JsonNode inputEntity) */ @Override public RestStatus deleteEntity(String index, String osid) { - UpdateResponse response = null; + DocWriteResponse response = null; try { String indexL = index.toLowerCase(); Map readMap = readEntity(indexL, osid); - // Map entityMap = (Map) readMap.get(index); - readMap.put(Constants.STATUS_KEYWORD, Constants.STATUS_INACTIVE); - response = getClient(indexL).update(new UpdateRequest(indexL, searchType, osid).doc(readMap), RequestOptions.DEFAULT); + if (isHardDeleteEnabled) { + response = getClient(indexL).delete(new DeleteRequest(indexL, searchType, osid), RequestOptions.DEFAULT); + } else { + readMap.put(Constants.STATUS_KEYWORD, Constants.STATUS_INACTIVE); + response = getClient(indexL).update(new UpdateRequest(indexL, searchType, osid).doc(readMap), RequestOptions.DEFAULT); + } } catch (NullPointerException | IOException e) { logger.error("exception in deleteEntity {}", ExceptionUtils.getStackTrace(e)); return RestStatus.NOT_FOUND; @@ -403,4 +407,7 @@ public void setUserName(String userName) { public void setPassword(String password) { this.password = password; } + public void setIsHardDeleteEnabled(boolean isHardDeleteEnabled) { + ElasticServiceImpl.isHardDeleteEnabled = isHardDeleteEnabled; + } } diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/config/GenericConfiguration.java b/java/registry/src/main/java/dev/sunbirdrc/registry/config/GenericConfiguration.java index 1af013954..5e50ca582 100644 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/config/GenericConfiguration.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/config/GenericConfiguration.java @@ -155,6 +155,10 @@ public class GenericConfiguration implements WebMvcConfigurer { private int httpMaxConnections; @Value("${elastic.search.scheme}") private String scheme; + + @Value("${registry.hard_delete_enabled}") + private boolean isHardDeleteEnabled; + @Autowired private DBConnectionInfoMgr dbConnectionInfoMgr; @Autowired @@ -445,6 +449,7 @@ public IElasticService elasticService() throws IOException { Set indices = new HashSet<>(iDefinitionsManager.getAllKnownDefinitions()); indices.add(ATTESTATION_POLICY); elasticService.init(indices); + elasticService.setIsHardDeleteEnabled(isHardDeleteEnabled); } return elasticService; } diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/controller/RegistryEntityController.java b/java/registry/src/main/java/dev/sunbirdrc/registry/controller/RegistryEntityController.java index c0bebdf18..68eaa2b7e 100644 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/controller/RegistryEntityController.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/controller/RegistryEntityController.java @@ -166,15 +166,20 @@ public ResponseEntity deleteEntity( checkEntityNameInDefinitionManager(entityName); String tag = "RegistryController.delete " + entityName; watch.start(tag); + JsonNode jsonNode = registryHelper.readEntity(userId, entityName, entityId, false, null, false); + if (jsonNode != null && jsonNode.has(entityName) && jsonNode.get(entityName).has(OSSystemFields._osSignedData.name())) { + String signedData = jsonNode.get(entityName).get(OSSystemFields._osSignedData.name()).asText(); + registryHelper.revokeExistingCredentials(entityName, entityId, userId, signedData); + } Vertex deletedEntity = registryHelper.deleteEntity(entityName, entityId, userId); if (deletedEntity != null && deletedEntity.keys().contains(OSSystemFields._osSignedData.name())) { - registryHelper.revokeExistingCredentials(entityName, entityId, userId, deletedEntity.value(OSSystemFields._osSignedData.name())); + deletedEntity.property(OSSystemFields._osSignedData.name()).toString(); + } responseParams.setErrmsg(""); responseParams.setStatus(Response.Status.SUCCESSFUL); watch.stop(tag); return new ResponseEntity<>(response, HttpStatus.OK); - } catch (RecordNotFoundException e) { createSchemaNotFoundResponse(e.getMessage(), responseParams); response = new Response(Response.API_ID.DELETE, "ERROR", responseParams); @@ -436,7 +441,7 @@ private JsonNode getAttestationSignedData(String attestationId, JsonNode node) t } @Nullable - private JsonNode getAttestationNode(String attestationId, JsonNode node) { + private JsonNode getAttestationNode(String attestationId, JsonNode node) throws AttestationNotFoundException, JsonProcessingException { Iterator iterator = node.iterator(); JsonNode attestationNode = null; while (iterator.hasNext()) { @@ -445,6 +450,10 @@ private JsonNode getAttestationNode(String attestationId, JsonNode node) { break; } } + assert attestationNode != null; + if (attestationNode.get(OSSystemFields._osAttestedData.name()) == null) + throw new AttestationNotFoundException(); + attestationNode = objectMapper.readTree(attestationNode.get(OSSystemFields._osAttestedData.name()).asText()); return attestationNode; } @@ -925,4 +934,4 @@ public ResponseEntity revokeACredential ( return new ResponseEntity<>(response,HttpStatus.INTERNAL_SERVER_ERROR); } } -} +} \ No newline at end of file diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/dao/IRegistryDao.java b/java/registry/src/main/java/dev/sunbirdrc/registry/dao/IRegistryDao.java index 33209b56a..5584b773b 100644 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/dao/IRegistryDao.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/dao/IRegistryDao.java @@ -4,7 +4,6 @@ import dev.sunbirdrc.registry.util.ReadConfigurator; import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.structure.Vertex; - public interface IRegistryDao { String addEntity(Graph graph, JsonNode rootNode); @@ -13,4 +12,5 @@ public interface IRegistryDao { void updateVertex(Graph graph, Vertex rootVertex, JsonNode inputJsonNode, String parentName) throws Exception; void deleteEntity(Vertex uuid); + void hardDeleteEntity(Vertex vertex); } diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/dao/RegistryDaoImpl.java b/java/registry/src/main/java/dev/sunbirdrc/registry/dao/RegistryDaoImpl.java index 93ca049ea..e8b0fb955 100644 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/dao/RegistryDaoImpl.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/dao/RegistryDaoImpl.java @@ -8,6 +8,7 @@ import dev.sunbirdrc.registry.util.IDefinitionsManager; import dev.sunbirdrc.registry.util.ReadConfigurator; import dev.sunbirdrc.registry.util.TypePropertyHelper; +import org.apache.tinkerpop.gremlin.structure.Direction; import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.slf4j.Logger; @@ -145,4 +146,15 @@ public void deleteEntity(Vertex vertex) { logger.error("Can't mark delete - Null vertex passed"); } } + @Override + public void hardDeleteEntity(Vertex vertex) { + if (vertex != null) { + vertex.edges(Direction.OUT).forEachRemaining(d -> { + this.hardDeleteEntity(d.inVertex()); + }); + vertex.remove(); + } else { + logger.error("Can't delete - Null vertex passed"); + } + } } diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/helper/RegistryHelper.java b/java/registry/src/main/java/dev/sunbirdrc/registry/helper/RegistryHelper.java index bbb924232..4bb4b7e09 100644 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/helper/RegistryHelper.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/helper/RegistryHelper.java @@ -174,6 +174,9 @@ public class RegistryHelper { @Value("${view_template.decrypt_private_fields:false}") private boolean viewTemplateDecryptPrivateFields; + @Value("${registry.hard_delete_enabled}") + private boolean isHardDeleteEnabled; + @Autowired private EntityTypeHandler entityTypeHandler; @@ -1056,9 +1059,15 @@ public Vertex deleteEntity(String entityName, String entityId, String userId) th String shardId = dbConnectionInfoMgr.getShardId(recordId.getShardLabel()); Shard shard = shardManager.activateShard(shardId); ReadConfigurator configurator = ReadConfiguratorFactory.getOne(false); + JsonNode deletedNode = null; + if(isHardDeleteEnabled) { + deletedNode = readEntity(userId, entityName, entityId, false, null, false); + } Vertex vertex = registryService.deleteEntityById(shard, entityName, userId, recordId.getUuid()); - VertexReader vertexReader = new VertexReader(shard.getDatabaseProvider(), vertex.graph(), configurator, uuidPropertyName, definitionsManager, true); - JsonNode deletedNode = JsonNodeFactory.instance.objectNode().set(entityName, vertexReader.constructObject(vertex)); + if (!isHardDeleteEnabled) { + VertexReader vertexReader = new VertexReader(shard.getDatabaseProvider(), vertex.graph(), configurator, uuidPropertyName, definitionsManager, true); + deletedNode = JsonNodeFactory.instance.objectNode().set(entityName, vertexReader.constructObject(vertex)); + } if(notificationEnabled) notificationHelper.sendNotification(deletedNode, DELETE); return vertex; } diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/service/NotificationHelper.java b/java/registry/src/main/java/dev/sunbirdrc/registry/service/NotificationHelper.java index d50973c26..6c4dfd574 100644 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/service/NotificationHelper.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/service/NotificationHelper.java @@ -16,6 +16,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.jmx.export.notification.UnableToSendNotificationException; import org.springframework.stereotype.Service; import java.io.IOException; @@ -49,6 +50,9 @@ public NotificationHelper() { public void sendNotification(JsonNode inputJson, String operationType) throws Exception { if (!notificationEnabled) return; + if(inputJson == null) { + throw new UnableToSendNotificationException("Notification input is null for action " + operationType); + } String entityType = inputJson.fields().next().getKey(); List templates = getNotificationTemplate(entityType, operationType); Map objectNodeMap = (Map) JSONUtil.convertJsonNodeToMap(inputJson).get(entityType); diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/service/impl/RegistryServiceImpl.java b/java/registry/src/main/java/dev/sunbirdrc/registry/service/impl/RegistryServiceImpl.java index ae23e2501..1d07c1846 100755 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/service/impl/RegistryServiceImpl.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/service/impl/RegistryServiceImpl.java @@ -9,7 +9,6 @@ import com.jayway.jsonpath.DocumentContext; import com.jayway.jsonpath.JsonPath; import dev.sunbirdrc.actors.factory.MessageFactory; -import dev.sunbirdrc.elastic.IElasticService; import dev.sunbirdrc.pojos.ComponentHealthInfo; import dev.sunbirdrc.pojos.HealthCheckResponse; import dev.sunbirdrc.pojos.HealthIndicator; @@ -22,8 +21,8 @@ import dev.sunbirdrc.registry.middleware.util.Constants; import dev.sunbirdrc.registry.middleware.util.JSONUtil; import dev.sunbirdrc.registry.middleware.util.OSSystemFields; -import dev.sunbirdrc.registry.model.event.Event; import dev.sunbirdrc.registry.model.EventType; +import dev.sunbirdrc.registry.model.event.Event; import dev.sunbirdrc.registry.service.*; import dev.sunbirdrc.registry.sink.DatabaseProvider; import dev.sunbirdrc.registry.sink.OSGraph; @@ -38,7 +37,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.kafka.core.KafkaTemplate; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; @@ -78,6 +76,9 @@ public class RegistryServiceImpl implements RegistryService { @Value("${encryption.enabled}") private boolean encryptionEnabled; + @Value("${registry.hard_delete_enabled}") + private boolean isHardDeleteEnabled; + @Value("${event.enabled}") private boolean isEventsEnabled; @@ -183,9 +184,13 @@ public Vertex deleteEntityById(Shard shard, String entityName, String userId, St if (!StringUtils.isEmpty(index) && index.equals(Schema)) { schemaService.deleteSchemaIfExists(vertex); } - if (!(vertex.property(Constants.STATUS_KEYWORD).isPresent() + if (isHardDeleteEnabled || !(vertex.property(Constants.STATUS_KEYWORD).isPresent() && vertex.property(Constants.STATUS_KEYWORD).value().equals(Constants.STATUS_INACTIVE))) { - registryDao.deleteEntity(vertex); + if (isHardDeleteEnabled) { + registryDao.hardDeleteEntity(vertex); + } else { + registryDao.deleteEntity(vertex); + } databaseProvider.commitTransaction(graph, tx); auditService.auditDelete( auditService.createAuditRecord(userId, uuid, tx, index), diff --git a/java/registry/src/main/resources/application.yml b/java/registry/src/main/resources/application.yml index 9f60de5b6..aed11d2a3 100644 --- a/java/registry/src/main/resources/application.yml +++ b/java/registry/src/main/resources/application.yml @@ -79,6 +79,7 @@ registry: redis: host: ${redis_host:localhost} port: ${redis_port:6379} + hard_delete_enabled: ${hard_delete_enabled:false} expandReference: ${expand_reference:false} workflow: enabled: ${workflow.enable:true} diff --git a/java/sunbirdrc-actors/src/main/java/dev/sunbirdrc/actors/ElasticSearchActor.java b/java/sunbirdrc-actors/src/main/java/dev/sunbirdrc/actors/ElasticSearchActor.java index 272cde556..1ab3ca97a 100644 --- a/java/sunbirdrc-actors/src/main/java/dev/sunbirdrc/actors/ElasticSearchActor.java +++ b/java/sunbirdrc-actors/src/main/java/dev/sunbirdrc/actors/ElasticSearchActor.java @@ -5,7 +5,6 @@ import dev.sunbirdrc.elastic.ESMessage; import dev.sunbirdrc.elastic.ElasticServiceImpl; import dev.sunbirdrc.elastic.IElasticService; -import org.springframework.beans.factory.annotation.Autowired; import org.sunbird.akka.core.BaseActor; import org.sunbird.akka.core.MessageProtos;