From cd347cbf7683875bb8322732fc4ddcb2ea861a8c Mon Sep 17 00:00:00 2001
From: Jessica Dussault <jdussault4@gmail.com>
Date: Fri, 3 Jan 2020 13:22:22 -0600
Subject: [PATCH 01/12] adding support for filters with normalized values

BREAKING CHANGE

We need to support keyword sorting with normalization such as
- case insensitivity
- accents converted to ascii
- characters like quotation marks, html tags stripped out
In order to do that, we will normalize the keyword fields on ingest
but the stored field displays as sent. Example:
  sending "Willa Cather" will result in "willa cather" normalized version
  but that's not what we want to display to users
Therefore, the API needs to return the filter / aggregate used
  and the number of results AS WELL as the "top" original version
There could be many keywords combined into a specific filter which is
  why we are relying on elasticsearch to determine the top hit and return
  it for display
The breaking change is that instead of facets being a list of fields with
  keyword and number, now we have a list of fields with keyword listing
  the number and top source (non-normalized version)
---
 app/services/search_item_req.rb | 25 ++++++++++++---
 app/services/search_item_res.rb | 57 ++++++++++++++++++++-------------
 2 files changed, 55 insertions(+), 27 deletions(-)

diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb
index 117462d..e260263 100644
--- a/app/services/search_item_req.rb
+++ b/app/services/search_item_req.rb
@@ -117,6 +117,16 @@ def facets
                 "field" => f,
                 "order" => { type => dir },
                 "size" => size
+              },
+              "aggs" => {
+                "top_matches" => {
+                  "top_hits" => {
+                    "_source" => {
+                      "includes" => [ f ]
+                    },
+                    "size" => 1
+                  }
+                }
               }
             }
           }
@@ -124,14 +134,19 @@ def facets
       else
         aggs[f] = {
           "terms" => {
-            # TODO if dataset is large, can implement partitions?
-            # "include" => {
-            #   "partition" => 0,
-            #   "num_partitions" => 10
-            # },
             "field" => f,
             "order" => { type => dir },
             "size" => size
+          },
+          "aggs" => {
+            "top_matches" => {
+              "top_hits" => {
+                "_source" => {
+                  "includes" => [ f ]
+                },
+                "size" => 1
+              }
+            }
           }
         }
       end
diff --git a/app/services/search_item_res.rb b/app/services/search_item_res.rb
index 4a2c5a7..25b2830 100644
--- a/app/services/search_item_res.rb
+++ b/app/services/search_item_res.rb
@@ -4,6 +4,7 @@ class SearchItemRes
 
   @@count = ["hits", "total"]
   @@facets = ["aggregations"]
+  @@facets_label = ["top_matches", "hits", "hits", "_source"]
   @@item = ["hits", "hits", 0, "_source"]
   @@items = ["hits", "hits"]
 
@@ -38,33 +39,45 @@ def combine_highlights
     end
   end
 
+  def format_bucket_value(facets, field, bucket)
+    # dates return in wonktastic ways, so grab key_as_string instead of gibberish number
+    # but otherwise just grab the key if key_as_string unavailable
+    key = bucket.key?("key_as_string") ? bucket["key_as_string"] : bucket["key"]
+    val = bucket["doc_count"]
+    source = key
+    # top_matches is a top_hits aggregation which returns a list of terms
+    # which were used for the facet.
+    #   Example: "Willa Cather" and "WILLA CATHER"
+    # Those terms will both have been normalized as "willa cather" but
+    # we will want to display one of the non-normalized terms instead
+    matches = bucket.dig("top_matches", "hits", "hits")
+    if matches
+      # elasticsearch stores nested source results without the "path"
+      no_nesting = field.split(".").last
+      source = matches.first.dig("_source", no_nesting)
+    end
+    facets[field][key] = {
+      "num" => val,
+      "source" => source
+    }
+  end
+
   def reformat_facets
-    facets = @body.dig(*@@facets)
-    if facets
-      formatted = {}
-      facets.each do |field, info|
-        formatted[field] = {}
-        buckets = {}
-        # nested fields do not have buckets
-        # at this level in the response structure
-        if info.has_key?("buckets")
-          buckets = info["buckets"]
-        else
-          buckets = info.dig(field, "buckets")
-        end
+    raw_facets = @body.dig(*@@facets)
+    if raw_facets
+      facets = {}
+      raw_facets.each do |field, info|
+        facets[field] = {}
+        # nested fields do not have buckets at this level of response structure
+        buckets = info.key?("buckets") ? info["buckets"] : info.dig(field, "buckets")
+
         if buckets
-          buckets.each do |b|
-            # dates return in wonktastic ways, so grab key_as_string instead of gibberish number
-            # but otherwise just grab the key if key_as_string unavailable
-            key = b.has_key?("key_as_string") ? b["key_as_string"] : b["key"]
-            val = b["doc_count"]
-            formatted[field][key] = val
-          end
+          buckets.each { |b| format_bucket_value(facets, field, b) }
         else
-          formatted[field] = {}
+          facets[field] = {}
         end
       end
-      return formatted
+      return facets
     else
       return {}
     end

From 668990407a0dc0d7eccc00b6801e924e31ae6300 Mon Sep 17 00:00:00 2001
From: Jessica Dussault <jdussault4@gmail.com>
Date: Fri, 3 Jan 2020 13:26:23 -0600
Subject: [PATCH 02/12] removes unnecessarily returns

---
 app/services/search_coll_res.rb |  2 +-
 app/services/search_item_req.rb | 12 ++++++------
 app/services/search_item_res.rb | 10 +++++-----
 app/services/search_service.rb  |  6 +++---
 4 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/app/services/search_coll_res.rb b/app/services/search_coll_res.rb
index 1958f4b..342f472 100644
--- a/app/services/search_coll_res.rb
+++ b/app/services/search_coll_res.rb
@@ -23,7 +23,7 @@ def build_response
         }
       end
 
-      return {
+      {
         "code" => 200,
         "info" => {
           "count" => collections.length,
diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb
index e260263..e9260c0 100644
--- a/app/services/search_item_req.rb
+++ b/app/services/search_item_req.rb
@@ -151,7 +151,7 @@ def facets
         }
       end
     end
-    return aggs
+    aggs
   end
 
   def filters
@@ -221,7 +221,7 @@ def filters
         filter_list << { "term" => { filter[0] => filter[1].gsub(/\r/, "") } }
       end
     end
-    return filter_list
+    filter_list
   end
 
   def highlights
@@ -243,7 +243,7 @@ def highlights
         end
       end
     end
-    return hl
+    hl
   end
 
   def sort
@@ -290,7 +290,7 @@ def sort
 
     end
 
-    return sort_obj
+    sort_obj
   end
 
   def source
@@ -300,7 +300,7 @@ def source
     criteria = {}
     criteria["includes"] = wlist if !wlist.empty?
     criteria["excludes"] = blist if !blist.empty?
-    return criteria
+    criteria
   end
 
   def text_search
@@ -324,7 +324,7 @@ def text_search
     else
       must = { "match_all" => {} }
     end
-    return must
+    must
   end
 
 end
diff --git a/app/services/search_item_res.rb b/app/services/search_item_res.rb
index 25b2830..f30f61f 100644
--- a/app/services/search_item_res.rb
+++ b/app/services/search_item_res.rb
@@ -19,7 +19,7 @@ def build_response
     items = combine_highlights
     facets = reformat_facets
 
-    return {
+    {
       "code" => 200,
       "count" => count,
       "facets" => facets,
@@ -30,12 +30,12 @@ def build_response
   def combine_highlights
     hits = @body.dig(*@@items)
     if hits
-      return hits.map do |hit|
+      hits.map do |hit|
         hit["_source"]["highlight"] = hit["highlight"] || {}
         hit["_source"]
       end
     else
-      return []
+      []
     end
   end
 
@@ -77,9 +77,9 @@ def reformat_facets
           facets[field] = {}
         end
       end
-      return facets
+      facets
     else
-      return {}
+      {}
     end
   end
 
diff --git a/app/services/search_service.rb b/app/services/search_service.rb
index 8724644..dbd8877 100644
--- a/app/services/search_service.rb
+++ b/app/services/search_service.rb
@@ -12,9 +12,9 @@ def initialize(url, params={}, user_req)
 
   def post(url_ending, json)
     res = RestClient.post("#{@url}/#{url_ending}", json.to_json, { "content-type" => "json" } )
-    return JSON.parse(res.body)
+    JSON.parse(res.body)
   rescue => e
-    return e
+    e
   end
 
   def search_collections
@@ -108,7 +108,7 @@ def on_success(req, res)
     if @params["debug"].present?
       json["req"]["query_obj"] = req
     end
-    return json
+    json
   end
 
   def build_collections_response(res)

From 41765ebf12b8d1671029d64b4e669d64cfd77cab Mon Sep 17 00:00:00 2001
From: Jessica Dussault <jdussault4@gmail.com>
Date: Tue, 7 Jan 2020 14:16:43 -0600
Subject: [PATCH 03/12] adds version number to responses as api_version

---
 app/controllers/application_controller.rb | 1 +
 app/controllers/collection_controller.rb  | 1 +
 app/controllers/default_controller.rb     | 2 +-
 app/services/search_coll_res.rb           | 1 +
 app/services/search_item_res.rb           | 1 +
 5 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index 0404329..15ff0d8 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -17,6 +17,7 @@ def display_error(error, req_body)
     render(status: 500, json: JSON.pretty_generate({
       "res" => {
         "code" => 500,
+        "api_version" => Api::Application::VERSION,
         "message" => "TODO",
         "info" => {
           "documentation" => "TODO",
diff --git a/app/controllers/collection_controller.rb b/app/controllers/collection_controller.rb
index 154caf8..cd1f9b9 100644
--- a/app/controllers/collection_controller.rb
+++ b/app/controllers/collection_controller.rb
@@ -15,6 +15,7 @@ def show
         "query_string" => request.fullpath
       },
       "res" => {
+        "api_version" => Api::Application::VERSION,
         "code" => 200,
         "info" => {
           "collection" => {},
diff --git a/app/controllers/default_controller.rb b/app/controllers/default_controller.rb
index c217c06..601fca1 100644
--- a/app/controllers/default_controller.rb
+++ b/app/controllers/default_controller.rb
@@ -18,6 +18,7 @@ def root
       },
       "res" => {
         "code" => 200,
+        "api_version" => Api::Application::VERSION,
         "info" => {
           "api_updated" => METADATA["api_updated"],
           "contact" => METADATA["cdrh@unl.edu"],
@@ -26,7 +27,6 @@ def root
           "index_updated" => "TODO",
           "license" => METADATA["license"],
           "terms_of_service" => METADATA["terms_of_service"],
-          "version" => Api::Application::VERSION,
           # TODO should we be obtaining these from
           # Rails.application.routes or similar?
           "endpoints" => [
diff --git a/app/services/search_coll_res.rb b/app/services/search_coll_res.rb
index 342f472..6e44f60 100644
--- a/app/services/search_coll_res.rb
+++ b/app/services/search_coll_res.rb
@@ -16,6 +16,7 @@ def build_response
           "collection_name" => coll["key"],
           "description" => "TODO",
           "image_id" => "TODO",
+          "api_version" => Api::Application::VERSION,
           "uri" => "TODO",
           "collection" => coll["key"],
           "item_count" => coll["doc_count"],
diff --git a/app/services/search_item_res.rb b/app/services/search_item_res.rb
index f30f61f..d37df03 100644
--- a/app/services/search_item_res.rb
+++ b/app/services/search_item_res.rb
@@ -22,6 +22,7 @@ def build_response
     {
       "code" => 200,
       "count" => count,
+      "api_version" => Api::Application::VERSION,
       "facets" => facets,
       "items" => items,
     }

From fec8509cf02b7467be4ffa7406de538663172c1a Mon Sep 17 00:00:00 2001
From: Jessica Dussault <jdussault4@gmail.com>
Date: Tue, 7 Jan 2020 15:53:32 -0600
Subject: [PATCH 04/12] add to changelog file

---
 CHANGELOG.md | 39 ++++++++++++++++++++++++++++++++++-----
 README.md    |  3 ++-
 2 files changed, 36 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 38f4e61..b2f1dc4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,13 +2,16 @@
 
 All notable changes to Apium will be documented in this file.
 
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
-and this project adheres to [Semantic
-Versioning](https://semver.org/spec/v2.0.0.html).
+Starting from Open ONI v0.11, The format is based on [Keep a
+Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to
+[Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+Please respect the 80-character text margin and follow the [GitHub Flavored
+Markdown Spec](https://github.github.com/gfm/).
 
 <!-- Template - Please preserve this order of sections
 ## [Unreleased] - Brief description
-[Unreleased]: https://github.com/CDRH/api/compare/v#.#.#...dev
+[Unreleased]: https://github.com/CDRH/apium/compare/v#.#.#...dev
 
 ### Fixed
 
@@ -28,8 +31,30 @@ Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased] - updates in preparation for Habeas release
 [Unreleased]: https://github.com/CDRH/api/compare/v1.0.4...dev
 
+### Added
+- "api_version" added to all response "res" objects
+
 ### Changed
 - upgraded to Rails 6
+- Added support for aggregating buckets by normalized keyword and returning
+  the "top_hits" first document result for a non-normalized display
+- Changes response format of `facets` key
+  
+  From:
+  ```
+  "facets": {
+    "WILLA CATHER": 10,
+    "Willa Cather": 50
+  }
+  ```
+  To:
+  ```
+  "facets": {
+    "willa cather": { "num" : 60, source: "Willa Cather" }
+  }
+  ```
+  Not only is the response format itself different, but there may be fewer
+  facets returned since normalized values which match are combined
 
 ## [v1.0.4](https://github.com/CDRH/api/compare/v1.0....v1.0.4) - Updates & license
 
@@ -38,7 +63,6 @@ Versioning](https://semver.org/spec/v2.0.0.html).
 license added
 
 ### Added
-
 - Documentation on facets and highlighting
 
 ## [v1.0.3](https://github.com/CDRH/api/compare/v1.0.2...v1.0.3) - gem updates
@@ -67,3 +91,8 @@ license added
 - version moved to initializer
 
 ## [v1.0.0](https://github.com/CDRH/api/tree/v1.0.0) - Initial Launch
+
+### Contributors
+
+- Jessica Dussault (jduss4)
+
diff --git a/README.md b/README.md
index 94cb16b..c984d07 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,8 @@
 # Apium
 
-Apium is an API to access all public Center for Digital Research in the Humanities resources. It is also an invasive weed in Nebraska.
+Apium is an API to access all public Center for Digital Research in the Humanities resources. It is also a genus of plants which includes celery, fool's water cress, and lesser marshwort.
 
 **[Apium Documentation](docs/README.md)**
+**[Changelog](CHANGELOG.md)**
 
 This project is licensed under the terms of the [MIT license](LICENSE.md).

From 6aad1630ba954c4212867c1176268d047ff19da2 Mon Sep 17 00:00:00 2001
From: Jessica Dussault <jdussault4@gmail.com>
Date: Mon, 13 Jan 2020 12:10:05 -0600
Subject: [PATCH 05/12] forgot to change all the copy pasted text to Apium

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b2f1dc4..19e2f41 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,7 +2,7 @@
 
 All notable changes to Apium will be documented in this file.
 
-Starting from Open ONI v0.11, The format is based on [Keep a
+Starting from Apium v1.0.1, The format is based on [Keep a
 Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to
 [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 

From dc96b4c769898bf432a46f5114ee8f3cacdb3d14 Mon Sep 17 00:00:00 2001
From: Jessica Dussault <jdussault4@gmail.com>
Date: Mon, 13 Jan 2020 12:15:16 -0600
Subject: [PATCH 06/12] renaming variable for clarity

---
 app/services/search_item_res.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/services/search_item_res.rb b/app/services/search_item_res.rb
index d37df03..a399fa0 100644
--- a/app/services/search_item_res.rb
+++ b/app/services/search_item_res.rb
@@ -54,8 +54,8 @@ def format_bucket_value(facets, field, bucket)
     matches = bucket.dig("top_matches", "hits", "hits")
     if matches
       # elasticsearch stores nested source results without the "path"
-      no_nesting = field.split(".").last
-      source = matches.first.dig("_source", no_nesting)
+      nested_child = field.split(".").last
+      source = matches.first.dig("_source", nested_child)
     end
     facets[field][key] = {
       "num" => val,

From 27bac2d973b78c50554b58d153341e2b42fb9a58 Mon Sep 17 00:00:00 2001
From: Jessica Dussault <jdussault4@gmail.com>
Date: Mon, 13 Jan 2020 16:26:51 -0600
Subject: [PATCH 07/12] ARGH

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 19e2f41..ad8bf5d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,7 +11,7 @@ Markdown Spec](https://github.github.com/gfm/).
 
 <!-- Template - Please preserve this order of sections
 ## [Unreleased] - Brief description
-[Unreleased]: https://github.com/CDRH/apium/compare/v#.#.#...dev
+[Unreleased]: https://github.com/CDRH/api/compare/v#.#.#...dev
 
 ### Fixed
 

From c78f75fff4d1c88f961c3270ed1c8b89b3f5cc11 Mon Sep 17 00:00:00 2001
From: Jessica Dussault <jdussault4@gmail.com>
Date: Wed, 5 Feb 2020 12:09:53 -0600
Subject: [PATCH 08/12] recreates filtering elasticsearch does to keywords to
 find in array

the issue is that when there are multiple values coming back for
a field (for example: works = [ "Dune", "Gulp" ] )
ALL of the values are being returned as the _source, not just
whichever was used to create the normalized aggregate in question

there are script fields that you can create with elasticsearch
that may hold the answer to this problem, but when investigating
that option it looked as though using the _source in them is
not optimal due to being resource intensive. For now I decided
that IF there is an array, then this solution will work for now,
since though this is annoying for the API it won't be running
(typically) against huge sets of facet results or on each group of
facets

I would like to find a better solution, however
---
 app/services/search_item_res.rb | 38 ++++++++++++++++++++++++++++-----
 1 file changed, 33 insertions(+), 5 deletions(-)

diff --git a/app/services/search_item_res.rb b/app/services/search_item_res.rb
index a399fa0..093f430 100644
--- a/app/services/search_item_res.rb
+++ b/app/services/search_item_res.rb
@@ -40,6 +40,29 @@ def combine_highlights
     end
   end
 
+  def find_source_from_top_hits(top_hits, field, key)
+    # elasticsearch stores nested source results without the "path"
+    nested_child = field.split(".").last
+    hit = top_hits.first.dig("_source", nested_child)
+    # if this is a multivalued field (for example: works or places),
+    # ALL of the values come back as the source, but we only want
+    # the single value from which the key was derived
+    if hit.class == Array
+      # I don't love this, because we will have to match exactly the logic
+      # that got us the key to get this to work
+      match_index = hit
+        .map { |s| remove_nonword_chars(s) }
+        .index(remove_nonword_chars(key))
+      matches = hit.map { |s| remove_nonword_chars(s) }
+      # if nothing matches the original key, return the entire source hit
+      # should return a string, regardless
+      return match_index ? hit[match_index] : hit.join(" ")
+    else
+      # it must be single-valued and therefore we are good to go
+      return hit
+    end
+  end
+
   def format_bucket_value(facets, field, bucket)
     # dates return in wonktastic ways, so grab key_as_string instead of gibberish number
     # but otherwise just grab the key if key_as_string unavailable
@@ -51,11 +74,9 @@ def format_bucket_value(facets, field, bucket)
     #   Example: "Willa Cather" and "WILLA CATHER"
     # Those terms will both have been normalized as "willa cather" but
     # we will want to display one of the non-normalized terms instead
-    matches = bucket.dig("top_matches", "hits", "hits")
-    if matches
-      # elasticsearch stores nested source results without the "path"
-      nested_child = field.split(".").last
-      source = matches.first.dig("_source", nested_child)
+    top_hits = bucket.dig("top_matches", "hits", "hits")
+    if top_hits
+      source = find_source_from_top_hits(top_hits, field, key)
     end
     facets[field][key] = {
       "num" => val,
@@ -84,4 +105,11 @@ def reformat_facets
     end
   end
 
+  def remove_nonword_chars(term)
+    # transliterate to ascii (ø -> o)
+    transliterated = I18n.transliterate(term)
+    # remove html tags like em, u, and strong, then strip remaining non-alpha characters
+    transliterated.gsub(/<\/?(?:em|strong|u)>|\W/, "").downcase
+  end
+
 end

From d5072005c67b66b26765030b6a589aa787aa99eb Mon Sep 17 00:00:00 2001
From: Jessica Dussault <jdussault4@gmail.com>
Date: Thu, 6 Feb 2020 09:11:55 -0600
Subject: [PATCH 09/12] updates tests for API changes

---
 test/services/search_item_req_test.rb | 14 +++++++-------
 test/services/search_item_res_test.rb |  9 +++++++--
 2 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/test/services/search_item_req_test.rb b/test/services/search_item_req_test.rb
index 2a2c785..29bf323 100644
--- a/test/services/search_item_req_test.rb
+++ b/test/services/search_item_req_test.rb
@@ -33,7 +33,7 @@ def test_facets
     # normal with no pagination overrides
     facets = SearchItemReq.new({ "facet" => [ "title" ] }).facets
     assert_equal(
-      {"title"=>{"terms"=>{"field"=>"title", "order"=>{"_count"=>"desc"}, "size"=>20}}},
+      {"title"=>{"terms"=>{"field"=>"title", "order"=>{"_count"=>"desc"}, "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["title"]}, "size"=>1}}}}},
       facets
     )
 
@@ -44,7 +44,7 @@ def test_facets
       "facet" => [ "title", "subcategory" ]
     }).facets
     assert_equal(
-      {"title"=>{"terms"=>{"field"=>"title", "order"=>{"_term"=>"asc"}, "size"=>10}}, "subcategory"=>{"terms"=>{"field"=>"subcategory", "order"=>{"_term"=>"asc"}, "size"=>10}}},
+      {"title"=>{"terms"=>{"field"=>"title", "order"=>{"_term"=>"asc"}, "size"=>10}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["title"]}, "size"=>1}}}}, "subcategory"=>{"terms"=>{"field"=>"subcategory", "order"=>{"_term"=>"asc"}, "size"=>10}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["subcategory"]}, "size"=>1}}}}},
       facets
     ) 
 
@@ -69,35 +69,35 @@ def test_facets
       "facet" => [ "creator.name" ]
     }).facets
     assert_equal(
-      {"creator.name"=>{"nested"=>{"path"=>"creator"}, "aggs"=>{"creator.name"=>{"terms"=>{"field"=>"creator.name", "order"=>{"_term"=>"desc"}, "size"=>20}}}}},
+      {"creator.name"=>{"nested"=>{"path"=>"creator"}, "aggs"=>{"creator.name"=>{"terms"=>{"field"=>"creator.name", "order"=>{"_term"=>"desc"}, "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["creator.name"]}, "size"=>1}}}}}}},
       facets
     )
 
     # with non-array
     facets = SearchItemReq.new({ "facet" => "title" }).facets
     assert_equal(
-      {"title"=>{"terms"=>{"field"=>"title", "order"=>{"_count"=>"desc"}, "size"=>20}}},
+      {"title"=>{"terms"=>{"field"=>"title", "order"=>{"_count"=>"desc"}, "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["title"]}, "size"=>1}}}}},
       facets
     )
 
     # sort term order specified
     facets = SearchItemReq.new({ "facet" => ["title", "format"], "facet_sort" => "term|desc" }).facets
     assert_equal(
-      {"title"=>{"terms"=>{"field"=>"title", "order"=>{"_term"=>"desc"}, "size"=>20}}, "format"=>{"terms"=>{"field"=>"format", "order"=>{"_term"=>"desc"}, "size"=>20}}},
+      {"title"=>{"terms"=>{"field"=>"title", "order"=>{"_term"=>"desc"}, "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["title"]}, "size"=>1}}}}, "format"=>{"terms"=>{"field"=>"format", "order"=>{"_term"=>"desc"}, "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["format"]}, "size"=>1}}}}},
       facets
     )
 
     # sort term no order specified
     facets = SearchItemReq.new({ "facet" => ["title", "format"], "facet_sort" => "term" }).facets
     assert_equal(
-      {"title"=>{"terms"=>{"field"=>"title", "order"=>{"_term"=>"desc"}, "size"=>20}}, "format"=>{"terms"=>{"field"=>"format", "order"=>{"_term"=>"desc"}, "size"=>20}}},
+      {"title"=>{"terms"=>{"field"=>"title", "order"=>{"_term"=>"desc"}, "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["title"]}, "size"=>1}}}}, "format"=>{"terms"=>{"field"=>"format", "order"=>{"_term"=>"desc"}, "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["format"]}, "size"=>1}}}}},
       facets
     )
 
     # sort count, no order specified
     facets = SearchItemReq.new({ "facet" => ["title"], "facet_sort" => "count" }).facets
     assert_equal(
-      {"title"=>{"terms"=>{"field"=>"title", "order"=>{"_count"=>"desc"}, "size"=>20}}},
+      {"title"=>{"terms"=>{"field"=>"title", "order"=>{"_count"=>"desc"}, "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["title"]}, "size"=>1}}}}},
       facets
     )
 
diff --git a/test/services/search_item_res_test.rb b/test/services/search_item_res_test.rb
index e435b5b..33b7aa2 100644
--- a/test/services/search_item_res_test.rb
+++ b/test/services/search_item_res_test.rb
@@ -13,12 +13,17 @@ def setup
   def test_combine_highlights
     hl = SearchItemRes.new(@es).combine_highlights
     first = hl.dig(0, "highlight")
-    assert_equal first, {"text"=>["View of the <em>water</em> from S. Lucia street in Naples. Napoli - Strada S. Lucia 35 Ediz Artistica RICTER"], "annotations"=>["This is a fake one that I made up to match <em>water</em> changes to highlighting"]}
+    assert_equal(
+      {"text"=>["View of the <em>water</em> from S. Lucia street in Naples. Napoli - Strada S. Lucia 35 Ediz Artistica RICTER"], "annotations"=>["This is a fake one that I made up to match <em>water</em> changes to highlighting"]},
+      first
+    )
   end
 
   def test_reformat_facets
     facets = SearchItemRes.new(@es).reformat_facets
-    assert_equal facets, {"date.year"=>{"1896"=>4, "1920"=>4, "1934"=>4, "1908"=>3, "1938"=>3, "1942"=>3, "1916"=>2, "1929"=>2, "1933"=>2, "1936"=>2, "1941"=>2, "1899"=>1, "1905"=>1, "1909"=>1, "1911"=>1, "1917"=>1, "1918"=>1, "1925"=>1, "1930"=>1, "1931"=>1, "1935"=>1, "1940"=>1, "1944"=>1}, "person.name"=>{"Cather, Elsie"=>30, "Cather, Mary Virginia 'Jennie' Boak"=>22, "Cather, Roscoe"=>17, ""=>16, "Lewis, Edith"=>16, "Shannon, Margaret Cather"=>16, "Cather, Charles F."=>11, "Cather, Meta Schaper"=>8, "Gere, Mariel"=>8, "Auld, Jessica Cather"=>7, "Hambourg, Isabelle McClung"=>7, "Cather, Charles Douglass 'Douglass'"=>6, "Greenslet, Ferris"=>6, "Mellen, Mary Virginia Auld"=>6, "Sherwood, Carrie Miner"=>6, "Auld, William Thomas 'Tom, Will'"=>5, "Brockway, Virginia Cather"=>5, "Creighton, Mary Miner"=>5, "Hambourg, Jan"=>5, "Cather, James Donald"=>4}, "format"=>{"letter"=>50}}
+    assert_equal(
+{"date.year"=>{"1896"=>{"num"=>4, "source"=>"1896"}, "1920"=>{"num"=>4, "source"=>"1920"}, "1934"=>{"num"=>4, "source"=>"1934"}, "1908"=>{"num"=>3, "source"=>"1908"}, "1938"=>{"num"=>3, "source"=>"1938"}, "1942"=>{"num"=>3, "source"=>"1942"}, "1916"=>{"num"=>2, "source"=>"1916"}, "1929"=>{"num"=>2, "source"=>"1929"}, "1933"=>{"num"=>2, "source"=>"1933"}, "1936"=>{"num"=>2, "source"=>"1936"}, "1941"=>{"num"=>2, "source"=>"1941"}, "1899"=>{"num"=>1, "source"=>"1899"}, "1905"=>{"num"=>1, "source"=>"1905"}, "1909"=>{"num"=>1, "source"=>"1909"}, "1911"=>{"num"=>1, "source"=>"1911"}, "1917"=>{"num"=>1, "source"=>"1917"}, "1918"=>{"num"=>1, "source"=>"1918"}, "1925"=>{"num"=>1, "source"=>"1925"}, "1930"=>{"num"=>1, "source"=>"1930"}, "1931"=>{"num"=>1, "source"=>"1931"}, "1935"=>{"num"=>1, "source"=>"1935"}, "1940"=>{"num"=>1, "source"=>"1940"}, "1944"=>{"num"=>1, "source"=>"1944"}}, "person.name"=>{"Cather, Elsie"=>{"num"=>30, "source"=>"Cather, Elsie"}, "Cather, Mary Virginia 'Jennie' Boak"=>{"num"=>22, "source"=>"Cather, Mary Virginia 'Jennie' Boak"}, "Cather, Roscoe"=>{"num"=>17, "source"=>"Cather, Roscoe"}, ""=>{"num"=>16, "source"=>""}, "Lewis, Edith"=>{"num"=>16, "source"=>"Lewis, Edith"}, "Shannon, Margaret Cather"=>{"num"=>16, "source"=>"Shannon, Margaret Cather"}, "Cather, Charles F."=>{"num"=>11, "source"=>"Cather, Charles F."}, "Cather, Meta Schaper"=>{"num"=>8, "source"=>"Cather, Meta Schaper"}, "Gere, Mariel"=>{"num"=>8, "source"=>"Gere, Mariel"}, "Auld, Jessica Cather"=>{"num"=>7, "source"=>"Auld, Jessica Cather"}, "Hambourg, Isabelle McClung"=>{"num"=>7, "source"=>"Hambourg, Isabelle McClung"}, "Cather, Charles Douglass 'Douglass'"=>{"num"=>6, "source"=>"Cather, Charles Douglass 'Douglass'"}, "Greenslet, Ferris"=>{"num"=>6, "source"=>"Greenslet, Ferris"}, "Mellen, Mary Virginia Auld"=>{"num"=>6, "source"=>"Mellen, Mary Virginia Auld"}, "Sherwood, Carrie Miner"=>{"num"=>6, "source"=>"Sherwood, Carrie Miner"}, "Auld, William Thomas 'Tom, Will'"=>{"num"=>5, "source"=>"Auld, William Thomas 'Tom, Will'"}, "Brockway, Virginia Cather"=>{"num"=>5, "source"=>"Brockway, Virginia Cather"}, "Creighton, Mary Miner"=>{"num"=>5, "source"=>"Creighton, Mary Miner"}, "Hambourg, Jan"=>{"num"=>5, "source"=>"Hambourg, Jan"}, "Cather, James Donald"=>{"num"=>4, "source"=>"Cather, James Donald"}}, "format"=>{"letter"=>{"num"=>50, "source"=>"letter"}}},      facets
+    )
   end
 
 end

From 9c02de5ee1d5a3b67b85ff68e7f9c2deb51eb54e Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Wed, 18 May 2022 15:26:50 -0500
Subject: [PATCH 10/12] remove reference to API version  per greg's suggestion

---
 app/controllers/default_controller.rb | 1 -
 1 file changed, 1 deletion(-)

diff --git a/app/controllers/default_controller.rb b/app/controllers/default_controller.rb
index 601fca1..d9d47f0 100644
--- a/app/controllers/default_controller.rb
+++ b/app/controllers/default_controller.rb
@@ -18,7 +18,6 @@ def root
       },
       "res" => {
         "code" => 200,
-        "api_version" => Api::Application::VERSION,
         "info" => {
           "api_updated" => METADATA["api_updated"],
           "contact" => METADATA["cdrh@unl.edu"],

From 102de7b2b1cc3401ce13433d031f9647052ad11f Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Thu, 19 May 2022 09:53:51 -0500
Subject: [PATCH 11/12] use matches with chars removed, see PR discussion

---
 app/services/search_item_res.rb | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/app/services/search_item_res.rb b/app/services/search_item_res.rb
index 093f430..8921b8a 100644
--- a/app/services/search_item_res.rb
+++ b/app/services/search_item_res.rb
@@ -56,7 +56,7 @@ def find_source_from_top_hits(top_hits, field, key)
       matches = hit.map { |s| remove_nonword_chars(s) }
       # if nothing matches the original key, return the entire source hit
       # should return a string, regardless
-      return match_index ? hit[match_index] : hit.join(" ")
+      return match_index ? matches[match_index] : matches.join(" ")
     else
       # it must be single-valued and therefore we are good to go
       return hit
@@ -92,7 +92,6 @@ def reformat_facets
         facets[field] = {}
         # nested fields do not have buckets at this level of response structure
         buckets = info.key?("buckets") ? info["buckets"] : info.dig(field, "buckets")
-
         if buckets
           buckets.each { |b| format_bucket_value(facets, field, b) }
         else

From 4b92510af2dede1a9501b7ac6362dd3b40d2f01d Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Thu, 19 May 2022 09:53:51 -0500
Subject: [PATCH 12/12] remove matches var as unused, see PR discussion

---
 app/services/search_item_res.rb | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/app/services/search_item_res.rb b/app/services/search_item_res.rb
index 8921b8a..a82a199 100644
--- a/app/services/search_item_res.rb
+++ b/app/services/search_item_res.rb
@@ -53,10 +53,9 @@ def find_source_from_top_hits(top_hits, field, key)
       match_index = hit
         .map { |s| remove_nonword_chars(s) }
         .index(remove_nonword_chars(key))
-      matches = hit.map { |s| remove_nonword_chars(s) }
       # if nothing matches the original key, return the entire source hit
       # should return a string, regardless
-      return match_index ? matches[match_index] : matches.join(" ")
+      return match_index ? hit[match_index] : hit.join(" ")
     else
       # it must be single-valued and therefore we are good to go
       return hit