From bdc7e4e47726431b8b165a9026cdbe9ab70566d7 Mon Sep 17 00:00:00 2001 From: Kumar Gauraw Date: Tue, 5 Mar 2024 19:51:18 +0530 Subject: [PATCH] Issue IQ-682 fix: fix for Quml 1.1 objects --- .../functions/AutoCreatorFunction.scala | 8 +++++-- .../autocreatorv2/helpers/AutoCreator.scala | 21 ++++++++++++------- .../helpers/HierarchyEnricher.scala | 2 +- .../autocreatorv2/helpers/ObjectUpdater.scala | 4 ++-- .../spec/helper/AutoCreatorSpec.scala | 2 +- 5 files changed, 24 insertions(+), 13 deletions(-) diff --git a/auto-creator-v2/src/main/scala/org/sunbird/job/autocreatorv2/functions/AutoCreatorFunction.scala b/auto-creator-v2/src/main/scala/org/sunbird/job/autocreatorv2/functions/AutoCreatorFunction.scala index d7f3d4530..307fe035b 100644 --- a/auto-creator-v2/src/main/scala/org/sunbird/job/autocreatorv2/functions/AutoCreatorFunction.scala +++ b/auto-creator-v2/src/main/scala/org/sunbird/job/autocreatorv2/functions/AutoCreatorFunction.scala @@ -1,5 +1,6 @@ package org.sunbird.job.autocreatorv2.functions +import org.apache.commons.lang3.StringUtils import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.configuration.Configuration import org.apache.flink.streaming.api.functions.ProcessFunction @@ -48,8 +49,11 @@ class AutoCreatorFunction(config: AutoCreatorV2Config, httpUtil: HttpUtil, if (event.isValid) { logger.info("Processing event for bulk approval operation having identifier : " + event.objectId) logger.debug("event edata : " + event.eData) - val definition: ObjectDefinition = defCache.getDefinition(event.objectType, config.schemaSupportVersionMap.getOrElse(event.objectType.toLowerCase(), "1.0").asInstanceOf[String], config.definitionBasePath) - val obj: ObjectData = getObject(event.objectId, event.objectType, event.downloadUrl, event.repository)(config, httpUtil, definition) + val obj: ObjectData = getObject(event.objectId, event.objectType, event.downloadUrl, event.repository)(config, httpUtil, defCache) + val schemaVersion = obj.metadata.getOrElse("schemaVersion", "1.0").asInstanceOf[String] + val defVer: String = if (StringUtils.equalsIgnoreCase("1.1", schemaVersion)) schemaVersion else config.schemaSupportVersionMap.getOrElse(event.objectType.toLowerCase(), "1.0").asInstanceOf[String] + logger.info("Fetching Schema With Version " + defVer) + val definition: ObjectDefinition = defCache.getDefinition(event.objectType, defVer, config.definitionBasePath) logger.debug("Constructed the ObjectData for " + obj.identifier) val enObj = enrichMetadata(obj, event.metadata)(config) logger.info("Enriched metadata for " + enObj.identifier) diff --git a/auto-creator-v2/src/main/scala/org/sunbird/job/autocreatorv2/helpers/AutoCreator.scala b/auto-creator-v2/src/main/scala/org/sunbird/job/autocreatorv2/helpers/AutoCreator.scala index 1d10ab487..47a825be7 100644 --- a/auto-creator-v2/src/main/scala/org/sunbird/job/autocreatorv2/helpers/AutoCreator.scala +++ b/auto-creator-v2/src/main/scala/org/sunbird/job/autocreatorv2/helpers/AutoCreator.scala @@ -14,11 +14,14 @@ trait AutoCreator extends ObjectUpdater with CollectionUpdater with HierarchyEnr private[this] val logger = LoggerFactory.getLogger(classOf[AutoCreator]) - def getObject(identifier: String, objType: String, downloadUrl: String, metaUrl: Option[String] = None)(implicit config: AutoCreatorV2Config, httpUtil: HttpUtil, objDef: ObjectDefinition): ObjectData = { + def getObject(identifier: String, objType: String, downloadUrl: String, metaUrl: Option[String] = None)(implicit config: AutoCreatorV2Config, httpUtil: HttpUtil, defCache: DefinitionCache): ObjectData = { val extractPath = extractDataZip(identifier, downloadUrl) val manifestData = getObjectDetails(identifier, extractPath, objType, metaUrl) - val metadata = manifestData.filterKeys(k => !(objDef.getRelationLabels().contains(k) || objDef.externalProperties.contains(k))) - val extData = manifestData.filterKeys(k => objDef.externalProperties.contains(k)) + val schemaVersion = manifestData.getOrElse("schemaVersion", "").asInstanceOf[String] + val defVer = if (StringUtils.equalsIgnoreCase("1.1", schemaVersion)) schemaVersion else config.schemaSupportVersionMap.getOrElse(objType.toLowerCase(), "1.0").asInstanceOf[String] + val definition: ObjectDefinition = defCache.getDefinition(objType, defVer, config.definitionBasePath) + val metadata = manifestData.filterKeys(k => !(definition.getRelationLabels().contains(k) || definition.externalProperties.contains(k))) + val extData = manifestData.filterKeys(k => definition.externalProperties.contains(k)) val hierarchy = getHierarchy(extractPath, objType)(config) val externalData = if (hierarchy.nonEmpty) extData ++ Map("hierarchy" -> hierarchy) else extData new ObjectData(identifier, objType, metadata, Some(externalData), Some(hierarchy)) @@ -41,7 +44,8 @@ trait AutoCreator extends ObjectUpdater with CollectionUpdater with HierarchyEnr .find(p => StringUtils.equalsIgnoreCase(identifier, p.getOrElse("identifier", "").asInstanceOf[String])).getOrElse(Map()) if (metaUrl.nonEmpty) { // TODO: deprecate setting "origin" after single sourcing refactoring. - val originData = s"""{\"identifier\": \"$identifier\",\"repository\":\"${metaUrl.head}\"}""" + val repoUrl = if(StringUtils.isNotBlank(metaUrl.head) && StringUtils.contains(metaUrl.head, "fields")) metaUrl.head.split("\\?")(0) else metaUrl.head + val originData = s"""{\"identifier\": \"$identifier\",\"repository\":\"${repoUrl}\"}""" val originDetails = Map[String, AnyRef]("origin" -> identifier, "originData" -> originData) val metadata = getMetaUrlData(metaUrl.head, objectType)(httpUtil) ++ originDetails manifestMetadata.++(metadata) @@ -97,11 +101,14 @@ trait AutoCreator extends ObjectUpdater with CollectionUpdater with HierarchyEnr children.flatMap(ch => { logger.info("Processing Children Having Identifier : " + ch._1) val objType = ch._2.asInstanceOf[Map[String, AnyRef]].getOrElse("objectType", "").asInstanceOf[String] - val definition: ObjectDefinition = defCache.getDefinition(objType, config.schemaSupportVersionMap.getOrElse(objType.toLowerCase(), "1.0").asInstanceOf[String], config.definitionBasePath) + val schemaVersion = ch._2.asInstanceOf[Map[String, AnyRef]].getOrElse("schemaVersion", "").asInstanceOf[String] + val defVer: String = if(StringUtils.equalsIgnoreCase("1.1", schemaVersion)) schemaVersion else config.schemaSupportVersionMap.getOrElse(objType.toLowerCase(), "1.0").asInstanceOf[String] + val definition: ObjectDefinition = defCache.getDefinition(objType, defVer, config.definitionBasePath) val downloadUrl = ch._2.asInstanceOf[Map[String, AnyRef]].getOrElse("downloadUrl", "").asInstanceOf[String] val props = definition.getSchemaProps() ++ definition.getExternalProps().keySet.toList - val repository = s"""${config.sourceBaseUrl}/${objType.toLowerCase}/v1/read/${ch._1}?fields=${props.mkString(",")}""" - val obj: ObjectData = getObject(ch._1, objType, downloadUrl, Some(repository))(config, httpUtil, definition) + val apiVer = if (StringUtils.equalsIgnoreCase("1.1", schemaVersion)) "v2" else "v1" + val repository = s"""${config.sourceBaseUrl}/${objType.toLowerCase}/${apiVer}/read/${ch._1}?fields=${props.mkString(",")}""" + val obj: ObjectData = getObject(ch._1, objType, downloadUrl, Some(repository))(config, httpUtil, defCache) logger.debug("Graph metadata for " + obj.identifier + " : " + obj.metadata) val enObj = enrichMetadata(obj, ch._2.asInstanceOf[Map[String, AnyRef]], overrideCloudProps = true)(config) logger.debug("Enriched metadata for " + enObj.identifier + " : " + enObj.metadata) diff --git a/auto-creator-v2/src/main/scala/org/sunbird/job/autocreatorv2/helpers/HierarchyEnricher.scala b/auto-creator-v2/src/main/scala/org/sunbird/job/autocreatorv2/helpers/HierarchyEnricher.scala index ef37baa34..19d706659 100644 --- a/auto-creator-v2/src/main/scala/org/sunbird/job/autocreatorv2/helpers/HierarchyEnricher.scala +++ b/auto-creator-v2/src/main/scala/org/sunbird/job/autocreatorv2/helpers/HierarchyEnricher.scala @@ -57,7 +57,7 @@ trait HierarchyEnricher { } def getChildData(data: Map[String, AnyRef])(implicit config: AutoCreatorV2Config): Map[String, AnyRef] = { - val props = config.cloudProps ++ List("objectType") + val props = config.cloudProps ++ List("objectType","schemaVersion") props.filter(x => data.get(x).nonEmpty).map(prop => (prop, data.getOrElse(prop, ""))).toMap } diff --git a/auto-creator-v2/src/main/scala/org/sunbird/job/autocreatorv2/helpers/ObjectUpdater.scala b/auto-creator-v2/src/main/scala/org/sunbird/job/autocreatorv2/helpers/ObjectUpdater.scala index 77484047f..fb9030c30 100644 --- a/auto-creator-v2/src/main/scala/org/sunbird/job/autocreatorv2/helpers/ObjectUpdater.scala +++ b/auto-creator-v2/src/main/scala/org/sunbird/job/autocreatorv2/helpers/ObjectUpdater.scala @@ -17,7 +17,7 @@ trait ObjectUpdater { val metadata = data - ("identifier", "objectType") val metaQuery = metaDataQuery(metadata, objectDef) val query = s"""MERGE (n:domain{IL_UNIQUE_ID:"$identifier"}) ON CREATE SET $metaQuery ON MATCH SET $metaQuery;""" - logger.debug("Graph Query: " + query) + logger.info("Graph Query: " + query) val result: StatementResult = neo4JUtil.executeQuery(query) if (null != result) { logger.info("Object graph data stored successfully for " + identifier) @@ -41,7 +41,7 @@ trait ObjectUpdater { case _ => query.value(d._1, d._2) } }) - logger.debug(s"Saving object external data for $identifier | Query : ${query.toString}") + logger.info(s"Saving object external data for $identifier | Query : ${query.toString}") val result = cassandraUtil.upsert(query.toString) if (result) { logger.info(s"Object external data saved successfully for ${identifier}") diff --git a/auto-creator-v2/src/test/scala/org/sunbird/job/autocreatorv2/spec/helper/AutoCreatorSpec.scala b/auto-creator-v2/src/test/scala/org/sunbird/job/autocreatorv2/spec/helper/AutoCreatorSpec.scala index 219969dd7..ec0d4207c 100644 --- a/auto-creator-v2/src/test/scala/org/sunbird/job/autocreatorv2/spec/helper/AutoCreatorSpec.scala +++ b/auto-creator-v2/src/test/scala/org/sunbird/job/autocreatorv2/spec/helper/AutoCreatorSpec.scala @@ -58,7 +58,7 @@ class AutoCreatorSpec extends FlatSpec with BeforeAndAfterAll with Matchers with "getObject" should "return a valid object" in { val downloadUrl = "https://dockstorage.blob.core.windows.net/sunbird-content-dock/questionset/do_113244425048121344131/added1_1616751462043_do_113244425048121344131_1_SPINE.ecar" - val result = new TestAutoCreator().getObject("do_113244425048121344131", "QuestionSet", downloadUrl)(jobConfig, httpUtil, qsDefinition) + val result = new TestAutoCreator().getObject("do_113244425048121344131", "QuestionSet", downloadUrl)(jobConfig, httpUtil, defCache) result.identifier shouldEqual "do_113244425048121344131" result.metadata.nonEmpty shouldBe(true) }