Merge pull request #190 from guardian/pm-upgrade-es-sdk-717

Upgrade elasticsearch sdk to version 7.17
guardian · Jan 3, 2024 · 0db2c15 · 0db2c15
2 parents c2574ce + 759d569
commit 0db2c15
Show file tree

Hide file tree

Showing 8 changed files with 39 additions and 34 deletions.
diff --git a/backend/app/services/ElasticsearchSyntax.scala b/backend/app/services/ElasticsearchSyntax.scala
@@ -2,13 +2,14 @@ package services
 
 import com.sksamuel.elastic4s.{ElasticClient, ElasticRequest, Executor, Functor, Handler, HttpClient, RequestFailure, RequestSuccess}
 import com.sksamuel.elastic4s.ElasticDsl._
+import com.sksamuel.elastic4s.fields.{ObjectField, TextField}
 import com.sksamuel.elastic4s.http.{JavaClient, JavaClientExceptionWrapper}
 import com.sksamuel.elastic4s.requests.bulk.BulkCompatibleRequest
 import com.sksamuel.elastic4s.requests.common.RefreshPolicy
 import com.sksamuel.elastic4s.requests.indexes.CreateIndexResponse
 import com.sksamuel.elastic4s.requests.indexes.admin.IndexExistsResponse
 import com.sksamuel.elastic4s.requests.mappings.dynamictemplate.DynamicMapping
-import com.sksamuel.elastic4s.requests.mappings.{MappingDefinition, ObjectField, TextField}
+import com.sksamuel.elastic4s.requests.mappings.{MappingDefinition}
 import com.sksamuel.elastic4s.requests.update.{UpdateByQueryRequest, UpdateRequest}
 import model.Language
 import org.apache.http.{ContentTooLongException, HttpHost}
@@ -48,9 +49,9 @@ trait ElasticsearchSyntax { this: Logging =>
   def emptyMultiLanguageField(name: String): ObjectField = objectField(name)
 
   def multiLanguageField(name: String, language: Language): ObjectField = {
-    objectField(name).fields(
+    ObjectField(name, properties = Seq(
       singleLanguageField(language.key, language)
-    )
+    ))
   }
 
   def multiLanguageValue(languages: List[Language], value: Any): Map[String, Any] = languages.map { lang =>

diff --git a/backend/app/services/index/ElasticsearchPages.scala b/backend/app/services/index/ElasticsearchPages.scala
@@ -2,6 +2,7 @@ package services.index
 
 import com.sksamuel.elastic4s.ElasticClient
 import com.sksamuel.elastic4s.ElasticDsl._
+import com.sksamuel.elastic4s.fields.ObjectField
 import com.sksamuel.elastic4s.requests.searches.queries.Query
 import com.sksamuel.elastic4s.requests.searches.{HighlightField, MultisearchResponseItem}
 import model.index.{Page, PageResult, PagesSummary}
@@ -22,12 +23,12 @@ class ElasticsearchPages(val client: ElasticClient, indexNamePrefix: String)(imp
         keywordField(PagesFields.resourceId),
         intField(PagesFields.page),
         emptyMultiLanguageField(PagesFields.value),
-        objectField(PagesFields.dimensions).fields(
+        ObjectField(PagesFields.dimensions, properties = Seq(
           floatField(PagesFields.width),
           floatField(PagesFields.height),
           floatField(PagesFields.top),
           floatField(PagesFields.bottom)
-        )
+        ))
       )
     ).flatMap { _ =>
       Attempt.sequence(Languages.all.map(addLanguage))

diff --git a/backend/app/services/index/ElasticsearchResources.scala b/backend/app/services/index/ElasticsearchResources.scala
@@ -2,10 +2,12 @@ package services.index
 
 import com.sksamuel.elastic4s.ElasticClient
 import com.sksamuel.elastic4s.ElasticDsl._
+import com.sksamuel.elastic4s.fields.ObjectField
 import com.sksamuel.elastic4s.requests.common.FetchSourceContext
+import com.sksamuel.elastic4s.requests.script.Script
 import com.sksamuel.elastic4s.requests.searches.DateHistogramInterval
-import com.sksamuel.elastic4s.requests.searches.queries.BoolQuery
 import com.sksamuel.elastic4s.requests.searches.SearchResponse
+import com.sksamuel.elastic4s.requests.searches.queries.compound.BoolQuery
 import com.sksamuel.elastic4s.requests.update.UpdateByQueryRequest
 import extraction.EnrichedMetadata
 import model.frontend._
@@ -44,7 +46,7 @@ class ElasticsearchResources(override val client: ElasticClient, indexName: Stri
           keywordField(IndexFields.workspaces.workspaceNodeId),
           keywordField(IndexFields.workspaces.uri)
         ),
-        objectField(IndexFields.metadataField).fields(
+        ObjectField(IndexFields.metadataField, properties = Seq(
           // Normal Documents
           emptyMultiLanguageField(IndexFields.metadata.fileUris),
           textKeywordField(IndexFields.metadata.mimeTypes),
@@ -53,25 +55,25 @@ class ElasticsearchResources(override val client: ElasticClient, indexName: Stri
             textKeywordField(NestedField.key).termVector("with_positions_offsets"),
             textField(NestedField.values).termVector("with_positions_offsets")
           ),
-          objectField(IndexFields.metadata.enrichedMetadataField).fields(
+          ObjectField(IndexFields.metadata.enrichedMetadataField, properties = Seq(
             textKeywordField(IndexFields.metadata.enrichedMetadata.title),
             textKeywordField(IndexFields.metadata.enrichedMetadata.author),
             longField(IndexFields.metadata.enrichedMetadata.createdAt),
             longField(IndexFields.metadata.enrichedMetadata.lastModified),
             textKeywordField(IndexFields.metadata.enrichedMetadata.createdWith),
             intField(IndexFields.metadata.enrichedMetadata.pageCount),
             intField(IndexFields.metadata.enrichedMetadata.wordCount)
-          ),
+          )),
 
           // Emails Only
-          objectField(IndexFields.metadata.fromField).fields(
+          ObjectField(IndexFields.metadata.fromField, properties = Seq(
             emptyMultiLanguageField(IndexFields.metadata.from.name),
             textField(IndexFields.metadata.from.address).termVector("with_positions_offsets")
-          ),
-          objectField(IndexFields.metadata.recipientsField).fields(
+          )),
+          ObjectField(IndexFields.metadata.recipientsField, properties = Seq(
             emptyMultiLanguageField(IndexFields.metadata.recipients.name),
             textField(IndexFields.metadata.recipients.address).termVector("with_positions_offsets")
-          ),
+          )),
           textField(IndexFields.metadata.sentAt),
           textField(IndexFields.metadata.sensitivity).termVector("with_positions_offsets"),
           textField(IndexFields.metadata.priority).termVector("with_positions_offsets"),
@@ -81,8 +83,7 @@ class ElasticsearchResources(override val client: ElasticClient, indexName: Stri
           emptyMultiLanguageField(IndexFields.metadata.html),
           intField(IndexFields.metadata.attachmentCount)
         )
-      )
-    ).flatMap { _ =>
+    ))).flatMap { _ =>
       Attempt.sequence(Languages.all.map(addLanguage))
     }.map { _ =>
       this
@@ -95,17 +96,17 @@ class ElasticsearchResources(override val client: ElasticClient, indexName: Stri
         multiLanguageField(IndexFields.text, language),
         multiLanguageField(IndexFields.ocr, language),
         multiLanguageField(IndexFields.transcript, language),
-        objectField(IndexFields.metadataField).fields(
+        ObjectField(IndexFields.metadataField, properties = Seq(
           multiLanguageField(IndexFields.metadata.fileUris, language),
-          objectField(IndexFields.metadata.fromField).fields(
+          ObjectField(IndexFields.metadata.fromField, properties = Seq(
             multiLanguageField(IndexFields.metadata.from.name, language)
-          ),
-          objectField(IndexFields.metadata.recipientsField).fields(
+          )),
+          ObjectField(IndexFields.metadata.recipientsField, properties = Seq(
             multiLanguageField(IndexFields.metadata.recipients.name, language)
-          ),
+          )),
           multiLanguageField(IndexFields.metadata.subject, language),
           multiLanguageField(IndexFields.metadata.html, language)
-        )
+        ))
       )
     }
   }
@@ -149,7 +150,7 @@ class ElasticsearchResources(override val client: ElasticClient, indexName: Stri
     executeUpdate {
       updateById(indexName, uri.value)
         .script {
-          script(
+          Script(
             s"""
                |params.mimeTypes.removeIf(mime -> ctx._source.metadata.${IndexFields.metadata.mimeTypes}.contains(mime));
                |ctx._source.metadata.${IndexFields.metadata.mimeTypes}.addAll(params.mimeTypes);
@@ -326,7 +327,7 @@ class ElasticsearchResources(override val client: ElasticClient, indexName: Stri
     executeUpdate {
       updateById(indexName, email.uri.value)
         .script {
-          script(
+          Script(
             s"""
                |params.recipients
                |  .removeIf(recipient ->
@@ -475,7 +476,7 @@ class ElasticsearchResources(override val client: ElasticClient, indexName: Stri
     val mustQuery = must(matchCollectionIngestionQuery)
     val field = maybeIngestion.map(_ => "ingestion").getOrElse("collection")
     if (inMultiple)
-      mustQuery.filter(scriptQuery(script(s"doc['${field}.keyword'].length > 1")))
+      mustQuery.filter(scriptQuery(Script(s"doc['${field}.keyword'].length > 1")))
     else
       mustQuery
   }
@@ -591,7 +592,7 @@ class ElasticsearchResources(override val client: ElasticClient, indexName: Stri
   override def addResourceToWorkspace(blobUri: Uri, workspaceId: String, workspaceNodeId: String): Attempt[Unit] = {
     executeUpdateByQueryImmediateRefresh {
       buildUpdateWorkspaceQuery(blobUri).script(
-        script(
+        Script(
           s"""
              |if(ctx._source.${IndexFields.workspacesField} == null) {
              |  ctx._source.${IndexFields.workspacesField} = [[
@@ -620,7 +621,7 @@ class ElasticsearchResources(override val client: ElasticClient, indexName: Stri
   override def removeResourceFromWorkspace(blobUri: Uri, workspaceId: String, workspaceNodeId: String): Attempt[Unit] = {
     executeUpdateByQueryImmediateRefresh {
       buildUpdateWorkspaceQuery(blobUri).script(
-        script(
+        Script(
           s"""
              |if(ctx._source.${IndexFields.workspacesField} != null) {
              |  ctx._source.${IndexFields.workspacesField}.removeIf(entry ->
@@ -645,7 +646,7 @@ class ElasticsearchResources(override val client: ElasticClient, indexName: Stri
           termQuery(s"${IndexFields.workspacesField}.${IndexFields.workspaces.workspaceId}", workspaceId)
         )
       ).script(
-        script(
+        Script(
           s"""
              |if(ctx._source.${IndexFields.workspacesField} != null) {
              |  ctx._source.${IndexFields.workspacesField}.removeIf(entry ->

diff --git a/backend/app/services/index/SearchContext.scala b/backend/app/services/index/SearchContext.scala
@@ -1,7 +1,8 @@
 package services.index
 
 import com.sksamuel.elastic4s.ElasticDsl._
-import com.sksamuel.elastic4s.requests.searches.queries.{BoolQuery, Query}
+import com.sksamuel.elastic4s.requests.searches.queries.compound.BoolQuery
+import com.sksamuel.elastic4s.requests.searches.queries.{Query}
 import model.annotations.{WorkspaceEntry, WorkspaceLeaf}
 import model.frontend.{TreeEntry, TreeLeaf, TreeNode}
 import model.index.SearchParameters

diff --git a/backend/app/services/table/Table.scala b/backend/app/services/table/Table.scala
@@ -2,6 +2,7 @@ package services.table
 
 import com.sksamuel.elastic4s.ElasticDsl._
 import com.sksamuel.elastic4s.ElasticClient
+import com.sksamuel.elastic4s.fields.ObjectField
 import model.Uri
 import model.index.TableRow
 import services.ElasticsearchSyntax
@@ -23,10 +24,10 @@ class ElasticsearchTable(override val client: ElasticClient, tableIndexName: Str
         textField(TableRowFields.tableId),
         intField(TableRowFields.rowIndex),
         textField(TableRowFields.sheetName),
-        objectField(TableRowFields.cellsField).fields(
+        ObjectField(TableRowFields.cellsField, properties = Seq(
           textField(TableRowFields.cells.key),
           textField(TableRowFields.cells.value)
-        )
+        ))
       )
     ).map { _ => this }
 

diff --git a/backend/test/test/integration/DockerElasticsearchService.scala b/backend/test/test/integration/DockerElasticsearchService.scala
@@ -11,7 +11,7 @@ trait DockerElasticsearchService extends DockerKit {
 
   override val StartContainersTimeout: FiniteDuration = 10.minutes
 
-  val elasticsearchContainer = DockerContainer("docker.elastic.co/elasticsearch/elasticsearch:7.9.3")
+  val elasticsearchContainer = DockerContainer("docker.elastic.co/elasticsearch/elasticsearch:7.17.9")
     .withPorts(DefaultElasticsearchHttpPort -> Some(ExposedElasticsearchHttpPort))
     .withEnv("discovery.type=single-node", s"http.publish_port=$ExposedElasticsearchHttpPort", "xpack.security.enabled=false")
     .withReadyChecker(

diff --git a/build.sbt b/build.sbt
@@ -116,8 +116,8 @@ lazy val backend = (project in file("backend"))
       // dependency to tikka or another library
       "org.bouncycastle" % "bcutil-jdk15on" % "1.70",
       "commons-io" % "commons-io" % "2.6",
-      "com.sksamuel.elastic4s" %% "elastic4s-client-esjava" % "7.9.1",
-      "org.elasticsearch.client" % "elasticsearch-rest-client-sniffer" % "7.9.2",
+      "com.sksamuel.elastic4s" %% "elastic4s-client-esjava" % "7.17.4",
+      "org.elasticsearch.client" % "elasticsearch-rest-client-sniffer" % "7.17.9",
       "org.apache.pekko" %% "pekko-cluster-typed" % "1.0.1", // Needs to match pekko version in Play
       "org.neo4j.driver" % "neo4j-java-driver" % "1.6.3",
       "com.pff" % "java-libpst" % "0.9.3",

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -13,7 +13,7 @@ services:
       - 7687:7687
 
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.9.3
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.9
     container_name: pfi-elasticsearch
     environment:
       - discovery.type=single-node