Deploy v9.2.0 to production

.github/workflows/test.yml

@@ -138,7 +138,7 @@ jobs:
           - 389:389
           - 636:636
       opensearch:
-        image: opensearchproject/opensearch:1.3.1
+        image: opensearchproject/opensearch:2.11.1
         env:
           bootstrap.memory_lock: true
           OPENSEARCH_JAVA_OPTS: "-Xms256m -Xmx256m"

app/assets/package.json

@@ -21,7 +21,7 @@
     "@apollo/react-hooks": "^4.0.0",
     "@apollo/react-testing": "^4.0.0",
     "@appbaseio/reactivesearch": "3.23.1",
-    "@emotion/react": "^11.11.3",
+    "@emotion/react": "^11.11.4",
     "@fortawesome/fontawesome-svg-core": "^6.5.1",
     "@fortawesome/free-brands-svg-icons": "^6.5.1",
     "@fortawesome/free-solid-svg-icons": "^6.5.1",
@@ -30,28 +30,28 @@
     "@honeybadger-io/react": "^6.1.7",
     "@nulib/design-system": "^1.5.1",
     "@radix-ui/react-dialog": "^1.0.5",
-    "@samvera/clover-iiif": "^2.3.2",
+    "@samvera/clover-iiif": "2.4.0-rc.0",
     "@samvera/image-downloader": "^1.1.1",
     "bulma": "^0.9.4",
     "bulma-checkradio": "^2.1.3",
     "bulma-pageloader": "^0.3.0",
     "bulma-switch": "^2.0.0",
     "bulma-toast": "^2.4.3",
     "classnames": "^2.5.1",
-    "downshift": "^8.3.1",
+    "downshift": "^8.3.2",
     "edtf": "^4.6.0",
     "esbuild-plugin-svgr": "^2.1.0",
     "faker": "^5.5.3",
     "file-saver": "^2.0.5",
     "graphql": "^16.8.1",
     "graphql-tag": "^2.12.6",
-    "hls.js": "^1.5.3",
+    "hls.js": "^1.5.7",
     "inflection": "^3.0.0",
     "jest-environment-jsdom": "^29.7.0",
     "js-cookie": "^3.0.5",
     "moment": "^2.30.1",
     "node-webvtt": "^1.9.4",
-    "phoenix": "^1.7.10",
+    "phoenix": "^1.7.11",
     "prop-types": "^15.8.1",
     "rc-progress": "^3.5.1",
     "react-beautiful-dnd": "^13.1.0",
@@ -63,32 +63,32 @@
     "react-hook-form": "^7.33.1",
     "react-icons": "^5.0.1",
     "react-json-pretty": "^2.2.0",
-    "recharts": "^2.11.0",
+    "recharts": "^2.12.1",
     "use-clipboard-copy": "^0.2.0"
   },
   "devDependencies": {
-    "@babel/core": "^7.23.9",
+    "@babel/core": "^7.24.0",
     "@babel/plugin-proposal-class-properties": "^7.17.12",
     "@babel/plugin-proposal-object-rest-spread": "^7.17.3",
     "@babel/plugin-proposal-optional-chaining": "^7.16.7",
     "@babel/plugin-transform-async-to-generator": "^7.22.5",
-    "@babel/preset-env": "^7.23.9",
+    "@babel/preset-env": "^7.24.0",
     "@babel/preset-react": "^7.23.3",
     "@babel/preset-typescript": "^7.23.3",
-    "@babel/runtime": "^7.23.9",
+    "@babel/runtime": "^7.24.0",
     "@creativebulma/bulma-divider": "^1.1.0",
     "@creativebulma/bulma-tooltip": "^1.2.0",
     "@elastic/elasticsearch-mock": "^2.0.0",
-    "@graphql-codegen/cli": "^5.0.0",
-    "@graphql-codegen/client-preset": "^4.1.0",
+    "@graphql-codegen/cli": "^5.0.2",
+    "@graphql-codegen/client-preset": "^4.2.4",
     "@graphql-typed-document-node/core": "^3.2.0",
     "@nulib/dcapi-types": "^2.1.0",
     "@nulib/prettier-config": "^1.2.0",
     "@testing-library/dom": "^9.3.4",
     "@testing-library/jest-dom": "^5.17.0",
-    "@testing-library/react": "^14.1.0",
+    "@testing-library/react": "^14.2.1",
     "@testing-library/user-event": "^14.5.2",
-    "@types/jest": "^29.5.11",
+    "@types/jest": "^29.5.12",
     "@types/react-beautiful-dnd": "^13.1.8",
     "@types/react-router-dom": "^5.3.3",
     "babel-plugin-syntax-dynamic-import": "^6.18.0",
@@ -109,7 +109,7 @@
     "react-dom": "^18.2.0",
     "react-router-dom": "^5.3.0",
     "react-router-prop-types": "^1.0.5",
-    "sass": "^1.70.0",
+    "sass": "^1.71.1",
     "ts-node": "^10.9.2",
     "typescript": "^5.3.3",
     "use-phoenix-channel": "^1.1.1"

app/config/config.exs

@@ -52,7 +52,8 @@ config :meadow, Meadow.Search.Cluster,
       name: prefix("dc-v2-work"),
       settings: "priv/search/v2/settings/work.json",
       version: 2,
-      schemas: [Meadow.Data.Schemas.Work]
+      schemas: [Meadow.Data.Schemas.Work],
+      pipeline: prefix("dc-v2-work-pipeline")
     },
     %{
       name: prefix("dc-v2-file-set"),
@@ -73,7 +74,11 @@ config :meadow, Meadow.Search.Cluster,
   ],
   bulk_page_size: 200,
   bulk_wait_interval: 500,
-  json_encoder: Ecto.Jason
+  json_encoder: Ecto.Jason,
+  embedding_model_id: aws_secret("meadow",
+    dig: ["search", "embedding_model_id"],
+    default: nil
+  )
 
 config :meadow,
   ark: %{

app/config/releases.exs

@@ -71,7 +71,8 @@ config :meadow, Meadow.Search.Cluster,
       name: "dc-v2-work",
       settings: priv_path.("search/v2/settings/work.json"),
       version: 2,
-      schemas: [Meadow.Data.Schemas.Work]
+      schemas: [Meadow.Data.Schemas.Work],
+      pipeline: prefix("dc-v2-work-pipeline")
     },
     %{
       name: "dc-v2-file-set",

app/config/test.exs

@@ -14,7 +14,8 @@ config :meadow, Meadow.Search.Cluster,
       default: "http://localhost:9200"
     ),
   bulk_page_size: 3,
-  bulk_wait_interval: 2
+  bulk_wait_interval: 2,
+  embedding_model_id: nil
 
 config :meadow,
   index_interval: 1234,

app/lib/meadow/data/indexer.ex

@@ -35,9 +35,24 @@ defmodule Meadow.Data.Indexer do
   def reindex_all(version, schema) do
     SearchClient.hot_swap(schema, version, fn index ->
       synchronize_schema(schema, version, index, :all)
+      |> check_synchronize_result(schema, index)
     end)
   end
 
+  def check_synchronize_result(:ok, schema, index) do
+    expected = Repo.aggregate(schema, :count)
+    actual = SearchClient.indexed_doc_count(index)
+
+    if actual < expected * 0.9 do
+      error = Meadow.IndexerError.exception("Indexed #{actual} docs; expected #{expected}")
+      {:incomplete, error}
+    else
+      :ok
+    end
+  end
+
+  def check_synchronize_result(other, _, _), do: other
+
   def synchronize_index do
     SearchConfig.index_versions()
     |> Enum.each(&synchronize_index/1)

app/lib/meadow/data/shared_links.ex

@@ -107,10 +107,7 @@ defmodule Meadow.Data.SharedLinks do
   defp create_shared_link_docs([]), do: :noop
 
   defp create_shared_link_docs(payload) do
-    Elastix.Bulk.post(SearchConfig.cluster_url(), payload,
-      index: Config.shared_links_index(),
-      type: @type_name
-    )
+    Elastix.Bulk.post(SearchConfig.cluster_url(), payload, index: Config.shared_links_index())
   end
 
   @doc """
@@ -179,7 +176,7 @@ defmodule Meadow.Data.SharedLinks do
     case Elastix.Search.count(
            SearchConfig.cluster_url(),
            Config.shared_links_index(),
-           [@type_name],
+           [],
            %{
              query: %{match_all: %{}}
            }

app/lib/meadow/indexing/v2/work.ex

@@ -74,8 +74,60 @@ defmodule Meadow.Indexing.V2.Work do
       work_type: encode_label(work.work_type)
     }
     |> Meadow.Utils.Map.nillify_empty()
+    |> prepare_embedding_field()
   end
 
+  @embedding_keys [
+    :abstract,
+    :alternate_title,
+    :caption,
+    :collection,
+    :contributor,
+    :creator,
+    :date_created,
+    :description,
+    :genre,
+    # :keywords,
+    :language,
+    :location,
+    :physical_description_material,
+    :physical_description_size,
+    :publisher,
+    :scope_and_contents,
+    :source,
+    :subject,
+    :style_period,
+    :table_of_contents,
+    :title,
+    :technique
+  ]
+
+  defp prepare_embedding_field(map) do
+    value =
+      @embedding_keys
+      |> Enum.reduce([], fn field_name, acc ->
+        v = prepare_embedding_value(Map.get(map, field_name))
+        [v | acc]
+      end)
+      |> List.flatten()
+      |> Enum.reject(fn v ->
+        is_nil(v) or byte_size(v) == 0
+      end)
+      |> Enum.join("\n")
+
+    Map.put(map, :embedding_text, value)
+  end
+
+  defp prepare_embedding_value(%{label: v}), do: prepare_embedding_value(v)
+
+  defp prepare_embedding_value([]), do: []
+
+  defp prepare_embedding_value([v | list]),
+    do: [prepare_embedding_value(v) | prepare_embedding_value(list)]
+
+  defp prepare_embedding_value(v) when is_binary(v), do: v
+  defp prepare_embedding_value(_), do: nil
+
   def api_url, do: Application.get_env(:meadow, :dc_api) |> get_in([:v2, "base_url"])
   def dc_url, do: Application.get_env(:meadow, :digital_collections_url)
 

app/lib/meadow/search/client.ex

@@ -54,11 +54,19 @@ defmodule Meadow.Search.Client do
       {:ok, _} ->
         SearchAlias.update(alias, index)
 
+      {:incomplete, reason} ->
+        Error.log_and_report(
+          "Incomplete hot swap on #{alias}. Alias left unchanged.",
+          reason,
+          __MODULE__,
+          []
+        )
+
       {:error, reason} ->
         SearchIndex.delete(index)
 
         Error.log_and_report(
-          "Problem performing hot swap",
+          "Problem performing hot swap on #{alias}",
           reason,
           __MODULE__,
           []

app/lib/meadow/search/config.ex

@@ -3,6 +3,8 @@ defmodule Meadow.Search.Config do
   Convenience methods for retrieving search-specific configuration
   """
 
+  require Logger
+
   def index_configs do
     Application.get_env(:meadow, Meadow.Search.Cluster)
     |> Keyword.get(:indexes)
@@ -27,12 +29,29 @@ defmodule Meadow.Search.Config do
   end
 
   def settings_for(schema, version) do
-    case config_for(schema, version) do
-      %{settings: settings} -> File.read!(settings) |> Jason.decode!()
-      _ -> nil
+    case {config_for(schema, version), embedding_model_id()} do
+      {%{settings: settings, pipeline: pipeline}, nil} ->
+        Logger.warning("No embedding model id found in config, skipping pipeline: #{pipeline}")
+        File.read!(settings) |> Jason.decode!()
+
+      {%{settings: settings, pipeline: pipeline}, _embedding_model_id} ->
+        File.read!(settings)
+        |> Jason.decode!()
+        |> put_in(["settings", "default_pipeline"], pipeline)
+
+      {%{settings: settings}, _} ->
+        File.read!(settings) |> Jason.decode!()
+
+      _ ->
+        nil
     end
   end
 
+  def embedding_model_id do
+    Application.get_env(:meadow, Meadow.Search.Cluster)
+    |> Keyword.get(:embedding_model_id)
+  end
+
   def index_versions do
     index_configs()
     |> Enum.map(& &1.version)

app/lib/meadow/search/index.ex

@@ -21,6 +21,14 @@ defmodule Meadow.Search.Index do
   Returns `{:ok, {alias, index}}`
   """
   def create(alias, settings \\ %{}) when is_binary(alias) and is_map(settings) do
+    case get_in(settings, ["settings", "default_pipeline"]) do
+      nil ->
+        :noop
+
+      pipeline ->
+        create_vector_pipeline(pipeline, SearchConfig.embedding_model_id(), "embedding_text")
+    end
+
     with timestamp <- DateTime.utc_now() |> DateTime.to_unix(:millisecond),
          new_index <- [alias, to_string(timestamp)] |> Enum.join("-") do
       case Elastix.Index.create(SearchConfig.cluster_url(), new_index, settings) do
@@ -30,6 +38,35 @@ defmodule Meadow.Search.Index do
     end
   end
 
+  @doc """
+  Create a vector pipeline, takes a name, model id, source field, and a target field
+  Returns {:ok, {}}
+  """
+  def create_vector_pipeline(name, model_id, source_field, target_field \\ "embedding") do
+    pipeline = %{
+      "description" => "Vector pipeline for #{name}",
+      "processors" => [
+        %{
+          "text_embedding" => %{
+            "model_id" => model_id,
+            "field_map" => %{
+              source_field => target_field
+            },
+            "ignore_failure" => true
+          }
+        },
+        %{
+          "remove" => %{
+            "field" => source_field,
+            "ignore_failure" => true
+          }
+        }
+      ]
+    }
+
+    HTTP.put(["_ingest", "pipeline", name], pipeline)
+  end
+
   @doc """
   Delete an index
   """

app/lib/meadow/utils/sitemap.ex

@@ -8,18 +8,16 @@ defmodule Meadow.Utils.Sitemap do
   require Logger
 
   @doc """
-  Generate a sitemap, upload it to the configured bucket, and optionally ping
-  Google and Bing to tell them an update is available
+  Generate a sitemap, upload it to the configured bucket
   """
-  def generate(ping \\ false) do
+  def generate do
     with config <- config() do
       Repo.transaction(
         fn ->
           stream()
           |> Sitemapper.generate(config)
           |> Stream.map(fn {filename, data} -> {filename, IO.iodata_to_binary(data)} end)
           |> Stream.flat_map(&persist(&1, config))
-          |> maybe_ping(config, ping)
           |> Stream.run()
 
           if Keyword.get(config, :gzip, false) do
@@ -33,9 +31,6 @@ defmodule Meadow.Utils.Sitemap do
     end
   end
 
-  defp maybe_ping(stream, _, false), do: stream
-  defp maybe_ping(stream, config, true), do: Sitemapper.ping(stream, config)
-
   defp persist({filename, content}, config) do
     log_persist(filename, byte_size(content), Enum.into(config, %{}))
     Sitemapper.persist([{filename, content}], config)