diff --git a/.circleci/config.yml b/.circleci/config.yml index 378d1a698..91028a4ec 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,8 +2,8 @@ version: 2.0 jobs: unit-tests: - docker: - - image: circleci/openjdk:14-jdk-buster-node-browsers-legacy + machine: + image: ubuntu-2004:202201-02 resource_class: medium working_directory: ~/kp steps: diff --git a/jobs-core/pom.xml b/jobs-core/pom.xml index e2f3d7c3b..d2c5c3366 100644 --- a/jobs-core/pom.xml +++ b/jobs-core/pom.xml @@ -157,7 +157,7 @@ org.elasticsearch.client elasticsearch-rest-high-level-client - 6.8.22 + 7.17.13 com.twitter diff --git a/jobs-core/src/main/scala/org/sunbird/job/util/ElasticSearchUtil.scala b/jobs-core/src/main/scala/org/sunbird/job/util/ElasticSearchUtil.scala index 16127e8eb..0bd328619 100644 --- a/jobs-core/src/main/scala/org/sunbird/job/util/ElasticSearchUtil.scala +++ b/jobs-core/src/main/scala/org/sunbird/job/util/ElasticSearchUtil.scala @@ -8,20 +8,19 @@ import org.apache.commons.lang3.StringUtils import org.apache.http.HttpHost import org.apache.http.client.config.RequestConfig import org.elasticsearch.action.admin.indices.alias.Alias -import org.elasticsearch.action.admin.indices.create.CreateIndexRequest import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.delete.DeleteRequest import org.elasticsearch.action.get.GetRequest import org.elasticsearch.action.index.IndexRequest import org.elasticsearch.action.update.UpdateRequest -import org.elasticsearch.client.{Response, RestClient, RestClientBuilder, RestHighLevelClient} +import org.elasticsearch.client.indices.CreateIndexRequest +import org.elasticsearch.client.{Request, RequestOptions, Response, RestClient, RestClientBuilder, RestHighLevelClient} import org.elasticsearch.common.settings.Settings -import org.elasticsearch.common.xcontent.XContentType +import org.elasticsearch.xcontent.XContentType import org.slf4j.LoggerFactory - import scala.collection.convert.ImplicitConversions.`collection AsScalaIterable` -class ElasticSearchUtil(connectionInfo: String, indexName: String, indexType: String, batchSize: Int = 1000) extends Serializable { +class ElasticSearchUtil(connectionInfo: String, indexName: String, batchSize: Int = 1000) extends Serializable { private val resultLimit = 100 private val esClient: RestHighLevelClient = createClient(connectionInfo) @@ -49,7 +48,7 @@ class ElasticSearchUtil(connectionInfo: String, indexName: String, indexType: St def isIndexExists(): Boolean = { try { - val response = esClient.getLowLevelClient.performRequest("HEAD", "/" + indexName) + val response = esClient.getLowLevelClient.performRequest(new Request("HEAD", "/" + indexName)) response.getStatusLine.getStatusCode == 200 } catch { case e: IOException => { @@ -59,19 +58,19 @@ class ElasticSearchUtil(connectionInfo: String, indexName: String, indexType: St } } - def addIndex(settings: String, mappings: String, alias: String = ""): Boolean = { - var response = false - val client = esClient - if (!isIndexExists()) { - val createRequest = new CreateIndexRequest(indexName) - if (StringUtils.isNotBlank(alias)) createRequest.alias(new Alias(alias)) - if (StringUtils.isNotBlank(settings)) createRequest.settings(Settings.builder.loadFromSource(settings, XContentType.JSON)) - if (StringUtils.isNotBlank(indexType) && StringUtils.isNotBlank(mappings)) createRequest.mapping(indexType, mappings, XContentType.JSON) - val createIndexResponse = client.indices.create(createRequest) - response = createIndexResponse.isAcknowledged + def addIndex(settings: String, mappings: String, alias: String = ""): Boolean = { + var response = false + val client = esClient + if (!isIndexExists()) { + val createRequest = new CreateIndexRequest(indexName) + if (StringUtils.isNotBlank(alias)) createRequest.alias(new Alias(alias)) + if (StringUtils.isNotBlank(settings)) createRequest.settings(Settings.builder.loadFromSource(settings, XContentType.JSON)) + if (StringUtils.isNotBlank(mappings)) createRequest.mapping(mappings, XContentType.JSON) + val createIndexResponse = client.indices.create(createRequest, RequestOptions.DEFAULT) + response = createIndexResponse.isAcknowledged + } + response } - response - } def addDocument(identifier: String, document: String): Unit = { try { @@ -79,7 +78,7 @@ class ElasticSearchUtil(connectionInfo: String, indexName: String, indexType: St // Replace mapper with JSONUtil once the JSONUtil is fixed val doc = mapper.readValue(document, new TypeReference[util.Map[String, AnyRef]]() {}) val updatedDoc = checkDocStringLength(doc) - val response = esClient.index(new IndexRequest(indexName, indexType, identifier).source(updatedDoc)) + val response = esClient.index(new IndexRequest(indexName).id(identifier).source(updatedDoc), RequestOptions.DEFAULT) logger.info(s"Added ${response.getId} to index ${response.getIndex}") } catch { case e: IOException => @@ -94,14 +93,12 @@ class ElasticSearchUtil(connectionInfo: String, indexName: String, indexType: St // Replace mapper with JSONUtil once the JSONUtil is fixed val doc = mapper.readValue(document, new TypeReference[util.Map[String, AnyRef]]() {}) val updatedDoc = checkDocStringLength(doc) - val indexRequest = if(identifier == null) new IndexRequest(indexName, indexType) else new IndexRequest(indexName, indexType, identifier) - val response = esClient.index(indexRequest.source(updatedDoc)) + val indexRequest = if(identifier == null) new IndexRequest(indexName) else new IndexRequest(indexName).id(identifier) + val response = esClient.index(indexRequest.source(updatedDoc), RequestOptions.DEFAULT) logger.info(s"Added ${response.getId} to index ${response.getIndex}") } catch { case e: IOException => logger.error(s"ElasticSearchUtil:: Error while adding document to index : $indexName : " + e.getMessage) - e.printStackTrace() - throw e } } @@ -111,9 +108,9 @@ class ElasticSearchUtil(connectionInfo: String, indexName: String, indexType: St // Replace mapper with JSONUtil once the JSONUtil is fixed val doc = mapper.readValue(document, new TypeReference[util.Map[String, AnyRef]]() {}) val updatedDoc = checkDocStringLength(doc) - val indexRequest = new IndexRequest(indexName, indexType, identifier).source(updatedDoc) - val request = new UpdateRequest().index(indexName).`type`(indexType).id(identifier).doc(updatedDoc).upsert(indexRequest) - val response = esClient.update(request) + val indexRequest = new IndexRequest(indexName).id(identifier).source(updatedDoc) + val request = new UpdateRequest().index(indexName).id(identifier).doc(updatedDoc).upsert(indexRequest) + val response = esClient.update(request, RequestOptions.DEFAULT) logger.info(s"Updated ${response.getId} to index ${response.getIndex}") } catch { case e: IOException => @@ -122,12 +119,12 @@ class ElasticSearchUtil(connectionInfo: String, indexName: String, indexType: St } def deleteDocument(identifier: String): Unit = { - val response = esClient.delete(new DeleteRequest(indexName, indexType, identifier)) + val response = esClient.delete(new DeleteRequest(indexName).id(identifier), RequestOptions.DEFAULT) logger.info(s"Deleted ${response.getId} to index ${response.getIndex}") } def getDocumentAsString(identifier: String): String = { - val response = esClient.get(new GetRequest(indexName, indexType, identifier)) + val response = esClient.get(new GetRequest(indexName).id(identifier), RequestOptions.DEFAULT) response.getSourceAsString } @@ -152,9 +149,9 @@ class ElasticSearchUtil(connectionInfo: String, indexName: String, indexType: St val doc: util.Map[String, AnyRef] = mapper.readValue(document, new TypeReference[util.Map[String, AnyRef]]() {}) val updatedDoc = checkDocStringLength(doc) logger.debug("ElasticSearchUtil:: bulkIndexWithIndexId:: doc: " + updatedDoc) - request.add(new IndexRequest(indexName, documentType, key).source(updatedDoc)) + request.add(new IndexRequest(indexName).id(key).source(updatedDoc)) if (count % batchSize == 0 || (count % batchSize < batchSize && count == jsonObjects.size)) { - val bulkResponse = esClient.bulk(request) + val bulkResponse = esClient.bulk(request, RequestOptions.DEFAULT) if (bulkResponse.hasFailures) logger.info("ElasticSearchUtil:: bulkIndexWithIndexId:: Failures in Elasticsearch bulkIndex : " + bulkResponse.buildFailureMessage) } } @@ -165,7 +162,7 @@ class ElasticSearchUtil(connectionInfo: String, indexName: String, indexType: St def isIndexExists(indexName: String): Boolean = { try { - val response: Response = esClient.getLowLevelClient.performRequest("HEAD", "/" + indexName) + val response: Response = esClient.getLowLevelClient.performRequest(new Request("HEAD", "/" + indexName)) response.getStatusLine.getStatusCode == 200 } catch { case e: IOException => false diff --git a/jobs-core/src/test/scala/org/sunbird/spec/ElasticSearchUtilSpec.scala b/jobs-core/src/test/scala/org/sunbird/spec/ElasticSearchUtilSpec.scala index 838e33ac2..cf5330502 100644 --- a/jobs-core/src/test/scala/org/sunbird/spec/ElasticSearchUtilSpec.scala +++ b/jobs-core/src/test/scala/org/sunbird/spec/ElasticSearchUtilSpec.scala @@ -9,7 +9,7 @@ import org.sunbird.job.util.ElasticSearchUtil class ElasticSearchUtilSpec extends FlatSpec with Matchers { val config: Config = ConfigFactory.load("base-test.conf") - val esUtil = new ElasticSearchUtil(config.getString("es.basePath"), "compositesearch", "cs") + val esUtil = new ElasticSearchUtil(config.getString("es.basePath"), "compositesearch") // "isIndexExists" should "return true if index exists" in { // val indexExists = esUtil.isIndexExists("compositesearch") diff --git a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/function/CollectionPublishFunction.scala b/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/function/CollectionPublishFunction.scala index 5dd7e0f63..48deff45c 100644 --- a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/function/CollectionPublishFunction.scala +++ b/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/function/CollectionPublishFunction.scala @@ -47,7 +47,7 @@ class CollectionPublishFunction(config: ContentPublishConfig, httpUtil: HttpUtil super.open(parameters) cassandraUtil = new CassandraUtil(config.cassandraHost, config.cassandraPort, config) neo4JUtil = new Neo4JUtil(config.graphRoutePath, config.graphName, config) - esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.compositeSearchIndexName, config.compositeSearchIndexType) + esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.compositeSearchIndexName) cloudStorageUtil = new CloudStorageUtil(config) ec = ExecutionContexts.global definitionCache = new DefinitionCache() diff --git a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/function/LiveCollectionPublishFunction.scala b/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/function/LiveCollectionPublishFunction.scala index b776e4206..1ad6e255f 100644 --- a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/function/LiveCollectionPublishFunction.scala +++ b/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/function/LiveCollectionPublishFunction.scala @@ -47,7 +47,7 @@ class LiveCollectionPublishFunction(config: LiveNodePublisherConfig, httpUtil: H super.open(parameters) cassandraUtil = new CassandraUtil(config.cassandraHost, config.cassandraPort, config) neo4JUtil = new Neo4JUtil(config.graphRoutePath, config.graphName, config) - esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.compositeSearchIndexName, config.compositeSearchIndexType) + esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.compositeSearchIndexName) cloudStorageUtil = new CloudStorageUtil(config) ec = ExecutionContexts.global definitionCache = new DefinitionCache() diff --git a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/EcarGeneratorSpec.scala b/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/EcarGeneratorSpec.scala index e62da72e5..c71006152 100644 --- a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/EcarGeneratorSpec.scala +++ b/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/EcarGeneratorSpec.scala @@ -35,7 +35,7 @@ class EcarGeneratorSpec extends FlatSpec with BeforeAndAfterAll with Matchers { "Object Ecar Generator generateEcar" should "return a Map containing Packaging Type and its url after uploading it to cloud" in { val hierarchy = Map("identifier" -> "do_123", "children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1", "objectType" -> "Question"), Map("identifier" -> "do_345", "name" -> "Children-2", "objectType" -> "Question"))) - val metadata = Map("identifier" -> "do_123", "appIcon" -> "https://dev.knowlg.sunbird.org/content/preview/assets/icons/avatar_anonymous.png", "identifier" -> "do_123", "objectType" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live") + val metadata = Map("identifier" -> "do_123", "appIcon" -> "https://dev.sunbirded.org/content/preview/assets/icons/avatar_anonymous.png", "identifier" -> "do_123", "objectType" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live") val objData = new ObjectData("do_123", metadata, None, Some(hierarchy)) val obj = new TestEcarGenerator() val result = obj.generateEcar(objData,List("SPINE")) @@ -50,6 +50,6 @@ class TestEcarGenerator extends EcarGenerator { "src" -> "somepath/sunbird_1551961194254.jpeg", "baseUrl" -> "some_base_url" ) - val testObj = List(Map("children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1", "objectType" -> "Question"), Map("identifier" -> "do_345", "name" -> "Children-2", "objectType" -> "Question")), "name" -> "Test QuestionSet", "appIcon" -> "https://dev.knowlg.sunbird.org/content/preview/assets/icons/avatar_anonymous.png", "objectType" -> "QuestionSet", "identifier" -> "do_123", "status" -> "Live", "identifier" -> "do_123"), Map("identifier" -> "do_234", "name" -> "Children-1", "objectType" -> "Question", "media" -> ScalaJsonUtil.serialize(List(media))), Map("identifier" -> "do_345", "name" -> "Children-2", "objectType" -> "Question")) + val testObj = List(Map("children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1", "objectType" -> "Question"), Map("identifier" -> "do_345", "name" -> "Children-2", "objectType" -> "Question")), "name" -> "Test QuestionSet", "appIcon" -> "https://dev.sunbirded.org/content/preview/assets/icons/avatar_anonymous.png", "objectType" -> "QuestionSet", "identifier" -> "do_123", "status" -> "Live", "identifier" -> "do_123"), Map("identifier" -> "do_234", "name" -> "Children-1", "objectType" -> "Question", "media" -> ScalaJsonUtil.serialize(List(media))), Map("identifier" -> "do_345", "name" -> "Children-2", "objectType" -> "Question")) override def getDataForEcar(obj: ObjectData): Option[List[Map[String, AnyRef]]] = Some(testObj) } diff --git a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectEnrichmentSpec.scala b/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectEnrichmentSpec.scala index d66422709..f43a3e159 100644 --- a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectEnrichmentSpec.scala +++ b/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectEnrichmentSpec.scala @@ -37,7 +37,7 @@ class ObjectEnrichmentSpec extends FlatSpec with BeforeAndAfterAll with Matchers when(mockNeo4JUtil.getNodesName(List("NCERT"))).thenReturn(Map("NCERT"-> "NCERT")) val hierarchy = Map("identifier" -> "do_123", "children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1"), Map("identifier" -> "do_345", "name" -> "Children-2"))) - val metadata = Map("identifier" -> "do_123", "targetFWIds" -> List("NCERT"), "boardIds" -> List("NCERT"), "appIcon" -> "https://dev.knowlg.sunbird.org/content/preview/assets/icons/avatar_anonymous.png", "IL_UNIQUE_ID" -> "do_123", "IL_FUNC_OBJECT_TYPE" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live") + val metadata = Map("identifier" -> "do_123", "targetFWIds" -> List("NCERT"), "boardIds" -> List("NCERT"), "appIcon" -> "https://dev.sunbirded.org/content/preview/assets/icons/avatar_anonymous.png", "IL_UNIQUE_ID" -> "do_123", "IL_FUNC_OBJECT_TYPE" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live") val objData = new ObjectData("do_123", metadata, None, Some(hierarchy)) val objectEnrichment = new TestObjectEnrichment() diff --git a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ThumbnailGeneratorSpec.scala b/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ThumbnailGeneratorSpec.scala index a5767806f..35a049da8 100644 --- a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ThumbnailGeneratorSpec.scala +++ b/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ThumbnailGeneratorSpec.scala @@ -25,7 +25,7 @@ class ThumbnailGeneratorSpec extends FlatSpec with BeforeAndAfterAll with Matche "Object Thumbnail Generator generateThumbnail" should "add the thumbnail to ObjectData" in { val hierarchy = Map("identifier" -> "do_123", "children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1"), Map("identifier" -> "do_345", "name" -> "Children-2"))) - val metadata = Map("identifier" -> "do_123", "appIcon" -> "https://dev.knowlg.sunbird.org/content/preview/assets/icons/avatar_anonymous.png", "IL_UNIQUE_ID" -> "do_123", "objectType" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live") + val metadata = Map("identifier" -> "do_123", "appIcon" -> "https://dev.sunbirded.org/content/preview/assets/icons/avatar_anonymous.png", "IL_UNIQUE_ID" -> "do_123", "objectType" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live") val objData = new ObjectData("do_123", metadata, None, Some(hierarchy)) val thumbnailGenerator = new TestThumbnailGenerator() @@ -35,7 +35,7 @@ class ThumbnailGeneratorSpec extends FlatSpec with BeforeAndAfterAll with Matche resultMetadata.isEmpty should be(false) resultMetadata.getOrElse("posterImage", "").asInstanceOf[String].isEmpty should be(false) resultMetadata.getOrElse("appIcon", "").asInstanceOf[String].isEmpty should be(false) - resultMetadata.getOrElse("posterImage", "").asInstanceOf[String] shouldBe "https://dev.knowlg.sunbird.org/content/preview/assets/icons/avatar_anonymous.png" + resultMetadata.getOrElse("posterImage", "").asInstanceOf[String] shouldBe "https://dev.sunbirded.org/content/preview/assets/icons/avatar_anonymous.png" resultMetadata.getOrElse("appIcon", "").asInstanceOf[String] shouldBe "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/questionset/do_123/artifact/avatar_anonymous.thumb.png" } diff --git a/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/functions/QRCodeImageGeneratorFunction.scala b/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/functions/QRCodeImageGeneratorFunction.scala index 348ab95f2..b781fa068 100644 --- a/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/functions/QRCodeImageGeneratorFunction.scala +++ b/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/functions/QRCodeImageGeneratorFunction.scala @@ -27,7 +27,7 @@ class QRCodeImageGeneratorFunction(config: QRCodeImageGeneratorConfig, override def open(parameters: Configuration): Unit = { cassandraUtil = new CassandraUtil(config.cassandraHost, config.cassandraPort, config) cloudStorageUtil = new CloudStorageUtil(config) - esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.dialcodeExternalIndex, config.dialcodeExternalIndexType) + esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.dialcodeExternalIndex) qRCodeImageGeneratorUtil = new QRCodeImageGeneratorUtil(config, cassandraUtil, cloudStorageUtil, esUtil) super.open(parameters) } diff --git a/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/functions/QRCodeIndexImageUrlFunction.scala b/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/functions/QRCodeIndexImageUrlFunction.scala index c5d2a4001..776461c06 100644 --- a/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/functions/QRCodeIndexImageUrlFunction.scala +++ b/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/functions/QRCodeIndexImageUrlFunction.scala @@ -24,7 +24,7 @@ class QRCodeIndexImageUrlFunction(config: QRCodeImageGeneratorConfig, override def open(parameters: Configuration): Unit = { cassandraUtil = new CassandraUtil(config.cassandraHost, config.cassandraPort, config) - esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.dialcodeExternalIndex, config.dialcodeExternalIndexType) + esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.dialcodeExternalIndex) qRCodeImageGeneratorUtil = new QRCodeImageGeneratorUtil(config, cassandraUtil, cloudStorageUtil, esUtil) super.open(parameters) } diff --git a/search-indexer/pom.xml b/search-indexer/pom.xml index a865f9b0b..32e6a3dda 100644 --- a/search-indexer/pom.xml +++ b/search-indexer/pom.xml @@ -73,9 +73,9 @@ test - pl.allegro.tech - embedded-elasticsearch - 2.7.0 + org.testcontainers + elasticsearch + 1.19.0 test diff --git a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/CompositeSearchIndexerHelper.scala b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/CompositeSearchIndexerHelper.scala index 0915868e3..14fa62a1d 100644 --- a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/CompositeSearchIndexerHelper.scala +++ b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/CompositeSearchIndexerHelper.scala @@ -13,8 +13,8 @@ trait CompositeSearchIndexerHelper { private[this] val logger = LoggerFactory.getLogger(classOf[CompositeSearchIndexerHelper]) def createCompositeSearchIndex()(esUtil: ElasticSearchUtil): Boolean = { - val settings = """{"max_ngram_diff":"29","mapping":{"total_fields":{"limit":"1500"}},"analysis":{"filter":{"mynGram":{"token_chars":["letter","digit","whitespace","punctuation","symbol"],"min_gram":"1","type":"nGram","max_gram":"30"}},"analyzer":{"cs_index_analyzer":{"filter":["lowercase","mynGram"],"type":"custom","tokenizer":"standard"},"keylower":{"filter":"lowercase","tokenizer":"keyword"},"cs_search_analyzer":{"filter":["standard","lowercase"],"type":"custom","tokenizer":"standard"}}}}""" - val mappings = """{"dynamic_templates":[{"nested":{"match_mapping_type":"object","mapping":{"type":"nested","fields":{"type":"nested"}}}},{"longs":{"match_mapping_type":"long","mapping":{"type":"long","fields":{"raw":{"type":"long"}}}}},{"booleans":{"match_mapping_type":"boolean","mapping":{"type":"boolean","fields":{"raw":{"type":"boolean"}}}}},{"doubles":{"match_mapping_type":"double","mapping":{"type":"double","fields":{"raw":{"type":"double"}}}}},{"dates":{"match_mapping_type":"date","mapping":{"type":"date","fields":{"raw":{"type":"date"}}}}},{"strings":{"match_mapping_type":"string","mapping":{"type":"text","copy_to":"all_fields","analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer","fields":{"raw":{"type":"text","fielddata":true,"analyzer":"keylower"}}}}}],"properties":{"screenshots":{"type":"text","index":false},"body":{"type":"text","index":false},"appIcon":{"type":"text","index":false},"all_fields":{"type":"text","analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer","fields":{"raw":{"type":"text","fielddata":true,"analyzer":"keylower"}}}}}""" + val settings = """{"max_ngram_diff":"29","mapping":{"total_fields":{"limit":"1500"}},"analysis":{"filter":{"mynGram":{"token_chars":["letter","digit","whitespace","punctuation","symbol"],"min_gram":"1","type":"nGram","max_gram":"30"}},"analyzer":{"cs_index_analyzer":{"filter":["lowercase","mynGram"],"type":"custom","tokenizer":"standard"},"keylower":{"filter":"lowercase","tokenizer":"keyword"},"cs_search_analyzer":{"filter":["lowercase"],"type":"custom","tokenizer":"standard"}}}}""" + val mappings = """{"dynamic_templates":[{"nested":{"match_mapping_type":"object","mapping":{"fields":{"type":"nested"},"type":"nested"}}},{"longs":{"match_mapping_type":"long","mapping":{"fields":{"raw":{"type":"long"}},"type":"long"}}},{"booleans":{"match_mapping_type":"boolean","mapping":{"fields":{"raw":{"type":"boolean"}},"type":"boolean"}}},{"doubles":{"match_mapping_type":"double","mapping":{"fields":{"raw":{"type":"double"}},"type":"double"}}},{"dates":{"match_mapping_type":"date","mapping":{"fields":{"raw":{"type":"date"}},"type":"date"}}},{"strings":{"match_mapping_type":"string","mapping":{"analyzer":"cs_index_analyzer","copy_to":"all_fields","fields":{"raw":{"fielddata":true,"analyzer":"keylower","type":"text"}},"search_analyzer":"cs_search_analyzer","type":"text"}}}],"properties":{"IL_FUNC_OBJECT_TYPE":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"IL_SYS_NODE_TYPE":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"IL_UNIQUE_ID":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"SYS_INTERNAL_LAST_UPDATED_ON":{"type":"date","fields":{"raw":{"type":"date"}}},"additionalCategories":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"all_fields":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"appIcon":{"type":"text","index":false},"appId":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"artifactUrl":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"assets":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"associations":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"associationswith":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"audience":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"author":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"autoCreateBatch":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"batches":{"type":"nested","properties":{"batchId":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"createdFor":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"enrollmentType":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"name":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"startDate":{"type":"date","fields":{"raw":{"type":"date"}}},"status":{"type":"long","fields":{"raw":{"type":"long"}}}}},"board":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"body":{"type":"text","index":false},"categories":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"category":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"categoryId":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"channel":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"channels":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"childNodes":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"cloudStorageKey":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"code":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"compatibilityLevel":{"type":"long","fields":{"raw":{"type":"long"}}},"consumerId":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"contentDisposition":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"contentEncoding":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"contentType":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"contentTypesCount":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"copyright":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"copyrightYear":{"type":"long","fields":{"raw":{"type":"long"}}},"createdBy":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"createdFor":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"createdOn":{"type":"date","fields":{"raw":{"type":"date"}}},"creator":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"credentials":{"type":"nested","properties":{"enabled":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"}}},"defaultCourseFramework":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"defaultFramework":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"defaultLicense":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"depth":{"type":"long","fields":{"raw":{"type":"long"}}},"description":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"dialcodeRequired":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"dialcodes":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"discussionForum":{"type":"nested","properties":{"enabled":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"}}},"downloadUrl":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"editorState":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"framework":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"frameworks":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"generateDIALCodes":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"gradeLevel":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"graph_id":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"idealScreenDensity":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"idealScreenSize":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"identifier":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"index":{"type":"long","fields":{"raw":{"type":"long"}}},"interceptionPoints":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"language":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"languageCode":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"lastPublishedBy":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"lastPublishedOn":{"type":"date","fields":{"raw":{"type":"date"}}},"lastStatusChangedOn":{"type":"date","fields":{"raw":{"type":"date"}}},"lastSubmittedOn":{"type":"date","fields":{"raw":{"type":"date"}}},"lastUpdatedBy":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"lastUpdatedOn":{"type":"date","fields":{"raw":{"type":"date"}}},"leafNodes":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"leafNodesCount":{"type":"long","fields":{"raw":{"type":"long"}}},"license":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"lockKey":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"mediaType":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"medium":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"mimeType":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"mimeTypesCount":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"name":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"nodeType":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"node_id":{"type":"long","fields":{"raw":{"type":"long"}}},"objectType":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"orgIdFieldName":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"organisation":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"os":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"osId":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"ownershipType":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"parent":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"pkgVersion":{"type":"long","fields":{"raw":{"type":"long"}}},"plugins":{"type":"nested","properties":{"identifier":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"semanticVersion":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"}}},"pragma":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"prevState":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"prevStatus":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"previewUrl":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"primaryCategory":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"publishError":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"publish_type":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"reservedDialcodes":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"resourceType":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"s3Key":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"sYS_INTERNAL_LAST_UPDATED_ON":{"type":"date","fields":{"raw":{"type":"date"}}},"screenshots":{"type":"text","index":false},"se_FWIds":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"se_boardIds":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"se_boards":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"se_gradeLevelIds":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"se_gradeLevels":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"se_mediumIds":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"se_mediums":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"se_subjectIds":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"se_subjects":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"searchIdFieldName":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"searchLabelFieldName":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"size":{"type":"double","fields":{"raw":{"type":"double"}}},"status":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"streamingUrl":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"subject":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"subjectIds":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"systemDefault":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"targetBoardIds":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"targetFWIds":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"targetGradeLevelIds":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"targetIdFieldName":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"targetMediumIds":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"targetObjectType":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"targetSubjectIds":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"terms":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"toc_url":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"totalCompressedSize":{"type":"long","fields":{"raw":{"type":"long"}}},"totalQuestions":{"type":"long","fields":{"raw":{"type":"long"}}},"totalScore":{"type":"long","fields":{"raw":{"type":"long"}}},"trackable":{"type":"nested","properties":{"autoBatch":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"enabled":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"}}},"type":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"url":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"userConsent":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"variants":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"version":{"type":"long","fields":{"raw":{"type":"long"}}},"versionKey":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"visibility":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"}}}""" esUtil.addIndex(settings, mappings) } diff --git a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/DIALCodeIndexerHelper.scala b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/DIALCodeIndexerHelper.scala index 5b5050ede..4343b1eaf 100644 --- a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/DIALCodeIndexerHelper.scala +++ b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/DIALCodeIndexerHelper.scala @@ -10,7 +10,7 @@ trait DIALCodeIndexerHelper { private[this] val logger = LoggerFactory.getLogger(classOf[DIALCodeIndexerHelper]) def createDialCodeIndex()(esUtil: ElasticSearchUtil): Boolean = { - val settings: String = """{"max_ngram_diff":"29","mapping":{"total_fields":{"limit":"1050"}},"analysis":{"analyzer":{"dc_index_analyzer":{"type":"custom","tokenizer":"standard","filter":["lowercase","mynGram"]},"dc_search_analyzer":{"type":"custom","tokenizer":"standard","filter":["standard","lowercase"]},"keylower":{"tokenizer":"keyword","filter":"lowercase"}},"filter":{"mynGram":{"type":"nGram","min_gram":1,"max_gram":30,"token_chars":["letter","digit","whitespace","punctuation","symbol"]}}}}""" + val settings: String = """{"max_ngram_diff":"29","mapping":{"total_fields":{"limit":"1050"}},"analysis":{"analyzer":{"dc_index_analyzer":{"type":"custom","tokenizer":"standard","filter":["lowercase","mynGram"]},"dc_search_analyzer":{"type":"custom","tokenizer":"standard","filter":["lowercase"]},"keylower":{"tokenizer":"keyword","filter":"lowercase"}},"filter":{"mynGram":{"type":"nGram","min_gram":1,"max_gram":30,"token_chars":["letter","digit","whitespace","punctuation","symbol"]}}}}""" val mappings: String = """{"dynamic_templates":[{"longs":{"match_mapping_type":"long","mapping":{"type":"long","fields":{"raw":{"type":"long"}}}}},{"booleans":{"match_mapping_type":"boolean","mapping":{"type":"boolean","fields":{"raw":{"type":"boolean"}}}}},{"doubles":{"match_mapping_type":"double","mapping":{"type":"double","fields":{"raw":{"type":"double"}}}}},{"dates":{"match_mapping_type":"date","mapping":{"type":"date","fields":{"raw":{"type":"date"}}}}},{"strings":{"match_mapping_type":"string","mapping":{"type":"text","copy_to":"all_fields","analyzer":"dc_index_analyzer","search_analyzer":"dc_search_analyzer","fields":{"raw":{"type":"text","fielddata":true,"analyzer":"keylower"}}}}}],"properties":{"all_fields":{"type":"text","analyzer":"dc_index_analyzer","search_analyzer":"dc_search_analyzer","fields":{"raw":{"type":"text","fielddata":true,"analyzer":"keylower"}}}}}""" esUtil.addIndex(settings, mappings) } diff --git a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/DIALCodeMetricsIndexerHelper.scala b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/DIALCodeMetricsIndexerHelper.scala index 3c0a403af..bae69abcd 100644 --- a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/DIALCodeMetricsIndexerHelper.scala +++ b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/DIALCodeMetricsIndexerHelper.scala @@ -11,7 +11,7 @@ trait DIALCodeMetricsIndexerHelper { def createDialCodeIndex()(esUtil: ElasticSearchUtil): Boolean = { val settings: String = """{"number_of_shards":5}""" - val mappings: String = """{"dcm":{"dynamic":false,"properties":{"dial_code":{"type":"keyword"},"total_dial_scans_local":{"type":"double"},"total_dial_scans_global":{"type":"double"},"average_scans_per_day":{"type":"double"},"last_scan":{"type":"date","format":"strict_date_optional_time||epoch_millis"},"first_scan":{"type":"date","format":"strict_date_optional_time||epoch_millis"}}}}""" + val mappings: String = """{"dynamic":false,"properties":{"dial_code":{"type":"keyword"},"total_dial_scans_local":{"type":"double"},"total_dial_scans_global":{"type":"double"},"average_scans_per_day":{"type":"double"},"last_scan":{"type":"date","format":"strict_date_optional_time||epoch_millis"},"first_scan":{"type":"date","format":"strict_date_optional_time||epoch_millis"}}}""" esUtil.addIndex(settings, mappings) } diff --git a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/CompositeSearchIndexerFunction.scala b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/CompositeSearchIndexerFunction.scala index 998dce0f3..a1845db62 100644 --- a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/CompositeSearchIndexerFunction.scala +++ b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/CompositeSearchIndexerFunction.scala @@ -23,7 +23,7 @@ class CompositeSearchIndexerFunction(config: SearchIndexerConfig, override def open(parameters: Configuration): Unit = { super.open(parameters) - elasticUtil = new ElasticSearchUtil(config.esConnectionInfo, config.compositeSearchIndex, config.compositeSearchIndexType) + elasticUtil = new ElasticSearchUtil(config.esConnectionInfo, config.compositeSearchIndex) createCompositeSearchIndex()(elasticUtil) } diff --git a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/DIALCodeIndexerFunction.scala b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/DIALCodeIndexerFunction.scala index 41f4e1cfd..fa7160157 100644 --- a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/DIALCodeIndexerFunction.scala +++ b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/DIALCodeIndexerFunction.scala @@ -21,7 +21,7 @@ class DIALCodeIndexerFunction(config: SearchIndexerConfig, override def open(parameters: Configuration): Unit = { super.open(parameters) - elasticUtil = new ElasticSearchUtil(config.esConnectionInfo, config.dialcodeExternalIndex, config.dialcodeExternalIndexType) + elasticUtil = new ElasticSearchUtil(config.esConnectionInfo, config.dialcodeExternalIndex) createDialCodeIndex()(elasticUtil) } diff --git a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/DIALCodeMetricsIndexerFunction.scala b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/DIALCodeMetricsIndexerFunction.scala index eeb6684d2..6db5ff9fe 100644 --- a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/DIALCodeMetricsIndexerFunction.scala +++ b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/DIALCodeMetricsIndexerFunction.scala @@ -21,7 +21,7 @@ class DIALCodeMetricsIndexerFunction(config: SearchIndexerConfig, override def open(parameters: Configuration): Unit = { super.open(parameters) - elasticUtil = new ElasticSearchUtil(config.esConnectionInfo, config.dialcodeMetricIndex, config.dialcodeMetricIndexType) + elasticUtil = new ElasticSearchUtil(config.esConnectionInfo, config.dialcodeMetricIndex) createDialCodeIndex()(elasticUtil) } diff --git a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/task/SearchIndexerConfig.scala b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/task/SearchIndexerConfig.scala index 9adad0e00..77bb323eb 100644 --- a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/task/SearchIndexerConfig.scala +++ b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/task/SearchIndexerConfig.scala @@ -72,4 +72,11 @@ class SearchIndexerConfig(override val config: Config) extends BaseJobConfig(con val ignoredFields: List[String] = if (config.hasPath("ignored.fields")) config.getStringList("ignored.fields").asScala.toList else List("responseDeclaration", "body") val isrRelativePathEnabled: Boolean = if (config.hasPath("cloudstorage.metadata.replace_absolute_path")) config.getBoolean("cloudstorage.metadata.replace_absolute_path") else false -} + val esImage: String = if (config.hasPath("es.image")) config.getString("es.image") else "docker.elastic.co/elasticsearch/elasticsearch" + val esImageTag: String = if (config.hasPath("es.imageTag")) config.getString("es.imageTag") else "7.17.13" + val esPorts : util.List[String] = if (config.hasPath("es.ports")) config.getStringList("es.ports") else List("9200:9200").asJava + val esJavaOpts: String = if (config.hasPath("es.javaOpts")) config.getString("es.javaOpts") else "-Xms128m -Xmx512m" + val esJavaOptsKey: String = if (config.hasPath("es.javaOptsKey")) config.getString("es.javaOptsKey") else "ES_JAVA_OPTS" + val xpackSecurityEnabled: String = if (config.hasPath("es.xpackSecurityEnabled")) config.getString("es.xpackSecurityEnabled") else "false" + val xpackSecurityKey: String = if (config.hasPath("es.xpackSecurityKey")) config.getString("es.xpackSecurityKey") else "xpack.security.enabled" +} \ No newline at end of file diff --git a/search-indexer/src/test/resources/test.conf b/search-indexer/src/test/resources/test.conf index 1faacbda4..edae4e796 100644 --- a/search-indexer/src/test/resources/test.conf +++ b/search-indexer/src/test/resources/test.conf @@ -32,4 +32,16 @@ schema { itemset = "2.0" } } -schema.definition_cache.expiry = 14400 \ No newline at end of file +schema.definition_cache.expiry = 14400 + +es { + image = "docker.elastic.co/elasticsearch/elasticsearch" + imageTag = "7.17.13" + ports = ["9200:9200"] + javaOptsKey = "ES_JAVA_OPTS" + javaOpts = "-Xms128m -Xmx512m" + xpackSecurityKey = "xpack.security.enabled" + xpackSecurityEnabled = "false" + +} + diff --git a/search-indexer/src/test/scala/org/sunbird/job/spec/SearchIndexerTaskTestSpec.scala b/search-indexer/src/test/scala/org/sunbird/job/spec/SearchIndexerTaskTestSpec.scala index d8bc1ee6b..4e93135e7 100644 --- a/search-indexer/src/test/scala/org/sunbird/job/spec/SearchIndexerTaskTestSpec.scala +++ b/search-indexer/src/test/scala/org/sunbird/job/spec/SearchIndexerTaskTestSpec.scala @@ -8,6 +8,8 @@ import org.apache.flink.streaming.api.functions.sink.SinkFunction import org.apache.flink.streaming.api.functions.source.SourceFunction import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext import org.apache.flink.test.util.MiniClusterWithClientResource +import org.apache.http.HttpHost +import org.elasticsearch.client.{Request, RestClient, RestClientBuilder} import org.mockito.ArgumentMatchers.anyString import org.mockito.Mockito import org.mockito.Mockito.{doNothing, times, verify, when} @@ -19,8 +21,11 @@ import org.sunbird.job.searchindexer.functions.{CompositeSearchIndexerFunction, import org.sunbird.job.searchindexer.task.{SearchIndexerConfig, SearchIndexerStreamTask} import org.sunbird.job.util.{ElasticSearchUtil, ScalaJsonUtil} import org.sunbird.spec.{BaseMetricsReporter, BaseTestSpec} -import pl.allegro.tech.embeddedelasticsearch.EmbeddedElastic - +import org.apache.http.client.config.RequestConfig +import org.testcontainers.containers.wait.strategy.Wait +import org.testcontainers.elasticsearch.ElasticsearchContainer +import org.testcontainers.utility.DockerImageName +import java.time.Duration import java.util import scala.collection.JavaConverters._ @@ -39,23 +44,40 @@ class SearchIndexerTaskTestSpec extends BaseTestSpec { val config: Config = ConfigFactory.load("test.conf") val jobConfig = new SearchIndexerConfig(config) val mockElasticUtil = mock[ElasticSearchUtil](Mockito.withSettings().serializable()) - var embeddedElastic: EmbeddedElastic = _ + var elasticContainer = new ElasticsearchContainer(DockerImageName.parse(jobConfig.esImage).withTag(jobConfig.esImageTag)) + var restClient: RestClient = _ val defCache = new DefinitionCache() + def setupElasticContainer(): Unit = { + elasticContainer.setHostAccessible(true) + elasticContainer.setPortBindings(jobConfig.esPorts) + elasticContainer.withEnv(jobConfig.esJavaOptsKey, jobConfig.esJavaOpts) + elasticContainer.withEnv(jobConfig.xpackSecurityKey, jobConfig.xpackSecurityEnabled) + elasticContainer.withStartupTimeout(Duration.ofSeconds(60)) + elasticContainer.waitingFor(Wait.forListeningPort()) + elasticContainer.start() + Thread.sleep(20000) + + restClient = RestClient + .builder(HttpHost.create(elasticContainer.getHttpHostAddress)) + .setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() { + override def customizeRequestConfig(requestConfigBuilder: RequestConfig.Builder): RequestConfig.Builder = { + requestConfigBuilder.setConnectionRequestTimeout(-1) + } + }).build() + } + override protected def beforeAll(): Unit = { super.beforeAll() - - embeddedElastic = EmbeddedElastic.builder() - .withElasticVersion("6.8.22") - .withEsJavaOpts("-Xms128m -Xmx512m") - .build() - .start() + setupElasticContainer() flinkCluster.before() } override protected def afterAll(): Unit = { super.afterAll() - embeddedElastic.stop() + if(elasticContainer != null){ + elasticContainer.stop() + } flinkCluster.after() } @@ -341,7 +363,7 @@ class SearchIndexerTaskTestSpec extends BaseTestSpec { when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.compositeSearchIndex, jobConfig.compositeSearchIndexType) + val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.compositeSearchIndex) val data = elasticUtil.getDocumentAsString("do_1132247274257203201191") data.isEmpty should be(false) data.contains("do_1132247274257203201191") should be(true) @@ -356,7 +378,7 @@ class SearchIndexerTaskTestSpec extends BaseTestSpec { when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.compositeSearchIndex, jobConfig.compositeSearchIndexType) + val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.compositeSearchIndex) val data = elasticUtil.getDocumentAsString("do_1132247274257203201191") data.isEmpty should be(false) data.contains("do_1132247274257203201191") should be(true) @@ -368,12 +390,12 @@ class SearchIndexerTaskTestSpec extends BaseTestSpec { } "Composite Search Indexer" should " create and delete the Data Node " in { - embeddedElastic.deleteIndices() + restClient.performRequest(new Request("DELETE", "/_all")) when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CompositeSearchEventSource(List[String](EventFixture.DATA_NODE_CREATE, EventFixture.DATA_NODE_DELETE))) when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.compositeSearchIndex, jobConfig.compositeSearchIndexType) + val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.compositeSearchIndex) val data = elasticUtil.getDocumentAsString("do_1132247274257203201191") data should be(null) BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(2) @@ -383,12 +405,12 @@ class SearchIndexerTaskTestSpec extends BaseTestSpec { } "Composite Search Indexer" should " do nothing for the Data Node due to UNKNOWN Operation " in { - embeddedElastic.deleteIndices() + restClient.performRequest(new Request("DELETE", "/_all")) when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CompositeSearchEventSource(List[String](EventFixture.DATA_NODE_UNKNOWN))) when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.compositeSearchIndex, jobConfig.compositeSearchIndexType) + val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.compositeSearchIndex) val data = elasticUtil.getDocumentAsString("do_1132247274257203201191") data should be(null) } @@ -398,7 +420,7 @@ class SearchIndexerTaskTestSpec extends BaseTestSpec { when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeExternalIndex, jobConfig.dialcodeExternalIndexType) + val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeExternalIndex) val data = elasticUtil.getDocumentAsString("X8R3W4") data.isEmpty should be(false) data.contains("X8R3W4") should be(true) @@ -413,7 +435,7 @@ class SearchIndexerTaskTestSpec extends BaseTestSpec { when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeExternalIndex, jobConfig.dialcodeExternalIndexType) + val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeExternalIndex) val data = elasticUtil.getDocumentAsString("X8R3W4") data.isEmpty should be(false) data.contains("X8R3W4") should be(true) @@ -425,12 +447,12 @@ class SearchIndexerTaskTestSpec extends BaseTestSpec { } "Composite Search Indexer" should " create and delete the External Dialcode Data " in { - embeddedElastic.deleteIndices() + restClient.performRequest(new Request("DELETE", "/_all")) when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CompositeSearchEventSource(List[String](EventFixture.DIALCODE_EXTERNAL_CREATE, EventFixture.DIALCODE_EXTERNAL_DELETE))) when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeExternalIndex, jobConfig.dialcodeExternalIndexType) + val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeExternalIndex) val data = elasticUtil.getDocumentAsString("X8R3W4") data should be(null) BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(2) @@ -440,12 +462,12 @@ class SearchIndexerTaskTestSpec extends BaseTestSpec { } "Composite Search Indexer" should " do nothing for the External Dialcode Data due to UNKNOWN Operation " in { - embeddedElastic.deleteIndices() + restClient.performRequest(new Request("DELETE", "/_all")) when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CompositeSearchEventSource(List[String](EventFixture.DIALCODE_EXTERNAL_UNKNOWN))) when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeExternalIndex, jobConfig.dialcodeExternalIndexType) + val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeExternalIndex) val data = elasticUtil.getDocumentAsString("X8R3W4") data should be(null) } @@ -455,7 +477,7 @@ class SearchIndexerTaskTestSpec extends BaseTestSpec { when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeMetricIndex, jobConfig.dialcodeMetricIndexType) + val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeMetricIndex) val data = elasticUtil.getDocumentAsString("QR1234") data.isEmpty should be(false) data.contains("QR1234") should be(true) @@ -470,7 +492,7 @@ class SearchIndexerTaskTestSpec extends BaseTestSpec { when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeMetricIndex, jobConfig.dialcodeMetricIndexType) + val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeMetricIndex) val data = elasticUtil.getDocumentAsString("QR1234") data.isEmpty should be(false) data.contains("QR1234") should be(true) @@ -482,12 +504,12 @@ class SearchIndexerTaskTestSpec extends BaseTestSpec { } "Composite Search Indexer" should " create and delete the Dialcode Metrics Data " in { - embeddedElastic.deleteIndices() + restClient.performRequest(new Request("DELETE", "/_all")) when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CompositeSearchEventSource(List[String](EventFixture.DIALCODE_METRIC_CREATE, EventFixture.DIALCODE_METRIC_DELETE))) when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeMetricIndex, jobConfig.dialcodeMetricIndexType) + val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeMetricIndex) val data = elasticUtil.getDocumentAsString("QR1234") data should be(null) BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(2) @@ -500,7 +522,7 @@ class SearchIndexerTaskTestSpec extends BaseTestSpec { when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CompositeSearchEventSource(List[String](EventFixture.DIALCODE_METRIC_UNKNOWN))) when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeMetricIndex, jobConfig.dialcodeMetricIndexType) + val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeMetricIndex) val data = elasticUtil.getDocumentAsString("QR1234") data should be(null) } @@ -522,7 +544,7 @@ class SearchIndexerTaskTestSpec extends BaseTestSpec { } "Composite Search Indexer" should " give error for the External Dialcode Data due to UNKNOWN objectType " in { - embeddedElastic.deleteIndices() + restClient.performRequest(new Request("DELETE", "/_all")) when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CompositeSearchEventSource(List[String](EventFixture.DATA_NODE_FAILED))) when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) intercept[Exception] { diff --git a/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/functions/AuditHistoryIndexer.scala b/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/functions/AuditHistoryIndexer.scala index 56378bd22..6c7a216e9 100644 --- a/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/functions/AuditHistoryIndexer.scala +++ b/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/functions/AuditHistoryIndexer.scala @@ -25,7 +25,7 @@ class AuditHistoryIndexer(config: TransactionEventProcessorConfig, var esUtil: E override def open(parameters: Configuration): Unit = { super.open(parameters) if (esUtil == null) { - esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.auditHistoryIndex, config.auditHistoryIndexType) + esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.auditHistoryIndex) } } diff --git a/transaction-event-processor/src/test/scala/org/sunbird/job/spec/TransactionEventProcessorTaskTestSpec.scala b/transaction-event-processor/src/test/scala/org/sunbird/job/spec/TransactionEventProcessorTaskTestSpec.scala index 4e3a98231..f481d49cd 100644 --- a/transaction-event-processor/src/test/scala/org/sunbird/job/spec/TransactionEventProcessorTaskTestSpec.scala +++ b/transaction-event-processor/src/test/scala/org/sunbird/job/spec/TransactionEventProcessorTaskTestSpec.scala @@ -129,9 +129,11 @@ class TransactionEventProcessorTaskTestSpec extends BaseTestSpec { "TransactionEventProcessorStreamTask" should "not generate audit history indexer event" in { server.start(9200) - server.enqueue(new MockResponse().setHeader( + server.enqueue(new MockResponse().setHeader("X-Elastic-Product", "Elasticsearch").setHeader( "Content-Type", "application/json" - ).setBody("""{"_index":"kp_audit_log_2018_7","_type":"ah","_id":"HLZ-1ngBtZ15DPx6ENjU","_version":1,"result":"created","_shards":{"total":2,"successful":1,"failed":0},"_seq_no":1,"_primary_term":1}""")) + ).setBody( + """{"name": "MacBook-Air.local","cluster_name": "elasticsearch","cluster_uuid": "9ra4wTGZSamseO3I99w","version": {"number": "7.17.13","build_flavor": "default","build_type": "tar","build_hash": "2b211dbb8bfd7f5b44d356bdfe54b1050c13","build_date": "2023-08-31T17:33:19.958690787Z","build_snapshot": false,"lucene_version": "8.11.1","minimum_wire_compatibility_version": "6.8.0","minimum_index_compatibility_version": "6.0.0-beta1"},"tagline": "You Know, for Search"} + |,{"_index":"kp_audit_log_2018_7","_type":"_doc","_id":"HLZ-1ngBtZ15DPx6ENjU","_version":1,"result":"created","_shards":{"total":2,"successful":0,"failed":1},"_seq_no":1,"_primary_term":1}""".stripMargin)) when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new AuditHistoryMapSource) if (jobConfig.auditHistoryIndexer) { @@ -145,9 +147,11 @@ class TransactionEventProcessorTaskTestSpec extends BaseTestSpec { } "TransactionEventProcessorStreamTask" should "generate audit history indexer event" in { - server.enqueue(new MockResponse().setHeader( + server.enqueue(new MockResponse().setHeader("X-Elastic-Product", "Elasticsearch").setHeader( "Content-Type", "application/json" - ).setBody("""{"_index":"kp_audit_log_2018_7","_type":"ah","_id":"HLZ-1ngBtZ15DPx6ENjU","_version":1,"result":"created","_shards":{"total":2,"successful":1,"failed":0},"_seq_no":1,"_primary_term":1}""")) + ).setBody( + """{"name": "MacBook-Air.local","cluster_name": "elasticsearch","cluster_uuid": "9ra4wTGZEFPeO3I99w","version": {"number": "7.17.13","build_flavor": "default","build_type": "tar","build_hash": "2b211dbb8bf7f5b44d356bdfe54b1050c13","build_date": "2023-08-31T17:33:19.958690787Z","build_snapshot": false,"lucene_version": "8.11.1","minimum_wire_compatibility_version": "6.8.0","minimum_index_compatibility_version": "6.0.0-beta1"},"tagline": "You Know, for Search"} + |,{"_index":"kp_audit_log_2018_7","_type":"_doc","_id":"HLZ-1ngBtZ15DPx6ENjU","_version":1,"result":"created","_shards":{"total":2,"successful":1,"failed":0},"_seq_no":1,"_primary_term":1}""".stripMargin)) when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new AuditHistoryMapSource) val setBoolean = config.withValue("job.audit-history-indexer", ConfigValueFactory.fromAnyRef(true))