From 561aeeb7012e64fff6f2df7f6d0920607cdf9fb6 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 19 May 2022 13:04:06 -0500 Subject: [PATCH 01/22] upgrade Puma --- Gemfile | 2 +- Gemfile.lock | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index 88ac8b9..80c970e 100644 --- a/Gemfile +++ b/Gemfile @@ -11,7 +11,7 @@ gem 'rails', '~> 6.0.2' # Use sqlite3 as the database for Active Record gem 'sqlite3' # Use Puma as the app server -gem 'puma', '~> 3.7' +gem 'puma', '~> 4.3' # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder # gem 'jbuilder', '~> 2.5' # Use Redis adapter to run Action Cable in production diff --git a/Gemfile.lock b/Gemfile.lock index c38a145..d94298a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -96,7 +96,8 @@ GEM nokogiri (1.13.6) mini_portile2 (~> 2.8.0) racc (~> 1.4) - puma (3.12.6) + puma (4.3.12) + nio4r (~> 2.0) racc (1.6.0) rack (2.2.3) rack-test (1.1.0) @@ -168,7 +169,7 @@ DEPENDENCIES bootsnap byebug listen (>= 3.0.5, < 3.2) - puma (~> 3.7) + puma (~> 4.3) rails (~> 6.0.2) rest-client (>= 2.1.0.rc1, < 2.2) spring From deb704ac4e5273980a620adf52b59fafa5e8bee5 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 19 May 2022 13:13:40 -0500 Subject: [PATCH 02/22] add specific version of puma to avoid security warnings --- Gemfile | 2 +- Gemfile.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index 80c970e..6d7da95 100644 --- a/Gemfile +++ b/Gemfile @@ -11,7 +11,7 @@ gem 'rails', '~> 6.0.2' # Use sqlite3 as the database for Active Record gem 'sqlite3' # Use Puma as the app server -gem 'puma', '~> 4.3' +gem 'puma', '>= 4.3.12' # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder # gem 'jbuilder', '~> 2.5' # Use Redis adapter to run Action Cable in production diff --git a/Gemfile.lock b/Gemfile.lock index d94298a..3c04aa9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -169,7 +169,7 @@ DEPENDENCIES bootsnap byebug listen (>= 3.0.5, < 3.2) - puma (~> 4.3) + puma (>= 4.3.12) rails (~> 6.0.2) rest-client (>= 2.1.0.rc1, < 2.2) spring From 065857a7bef10fe1152b769d6ae3e59fc1dcc3c3 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 25 May 2022 10:46:36 -0500 Subject: [PATCH 03/22] update .ruby-version --- .ruby-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ruby-version b/.ruby-version index 324db8d..4e34c4d 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -ruby-2.6.8 +ruby-2.7.6 From 38b02b7a32e1a343d5ec714aff18df7cb930f382 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 25 May 2022 10:50:33 -0500 Subject: [PATCH 04/22] update to later version of puma --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 6d7da95..4170559 100644 --- a/Gemfile +++ b/Gemfile @@ -11,7 +11,7 @@ gem 'rails', '~> 6.0.2' # Use sqlite3 as the database for Active Record gem 'sqlite3' # Use Puma as the app server -gem 'puma', '>= 4.3.12' +gem 'puma', '>= 5.6' # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder # gem 'jbuilder', '~> 2.5' # Use Redis adapter to run Action Cable in production From e763bb1f0c9f98873a0cf7a004deff586214440c Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 25 May 2022 10:51:14 -0500 Subject: [PATCH 05/22] another round of gem updates --- Gemfile.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 3c04aa9..3f1cd9c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -69,7 +69,7 @@ GEM globalid (1.0.0) activesupport (>= 5.0) http-accept (1.7.0) - http-cookie (1.0.4) + http-cookie (1.0.5) domain_name (~> 0.5) i18n (1.10.0) concurrent-ruby (~> 1.0) @@ -96,7 +96,7 @@ GEM nokogiri (1.13.6) mini_portile2 (~> 2.8.0) racc (~> 1.4) - puma (4.3.12) + puma (5.6.4) nio4r (~> 2.0) racc (1.6.0) rack (2.2.3) @@ -169,7 +169,7 @@ DEPENDENCIES bootsnap byebug listen (>= 3.0.5, < 3.2) - puma (>= 4.3.12) + puma (>= 5.6) rails (~> 6.0.2) rest-client (>= 2.1.0.rc1, < 2.2) spring From 2769319a13dfa6fe3877fcd6cc6c2dddbc7ae3bc Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 24 May 2022 13:40:03 -0500 Subject: [PATCH 06/22] add facet for matching nested facet --- app/services/search_item_req.rb | 39 +++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index e9260c0..64c683b 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -104,6 +104,45 @@ def facets "order" => { f_type => dir }, } } + elsif f.include?("[") + #or nest it inside the next one? + #this will be the same + facet = f.split("[") + path = facet.split(".").first + condition = f[/(?<=\[).+?(?=\])/] + subject = condition.split(".").first + predicate = condition.split(".").last + aggs[f] = { + "nested" => { + "path" => path + }, + "aggs" => { + "query" => { + "term" => { + subject => predicate + } + }, + "aggs" => { + f => { + "terms" => { + "field" => facet, + "order" => { type => dir }, + "size" => size + }, + "aggs" => { + "top_matches" => { + "top_hits" => { + "_source" => { + "includes" => [ f ] + }, + "size" => 1 + } + } + } + } + } + } + } # if nested, has extra syntax elsif f.include?(".") path = f.split(".").first From 3b33d4afcf38391e9825e3f81b41dbe68837c6d9 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 24 May 2022 14:31:49 -0500 Subject: [PATCH 07/22] add filter for matching nested facet --- app/services/search_item_req.rb | 37 ++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index 64c683b..c58036d 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -104,9 +104,8 @@ def facets "order" => { f_type => dir }, } } + #nested facet, matching on another nested facet elsif f.include?("[") - #or nest it inside the next one? - #this will be the same facet = f.split("[") path = facet.split(".").first condition = f[/(?<=\[).+?(?=\])/] @@ -143,7 +142,7 @@ def facets } } } - # if nested, has extra syntax + # ordinary nested facet elsif f.include?(".") path = f.split(".").first aggs[f] = { @@ -200,8 +199,36 @@ def filters # (type 2 will only be used for dates) filters = fields.map {|f| f.split(@@filter_separator, 3) } filters.each do |filter| - # NESTED FIELD FILTER - if filter[0].include?(".") + # NESTED matching + if filter[0].include?("[") + facet = f.split("[") + path = facet.split(".").first + condition = f[/(?<=\[).+?(?=\])/] + subject = condition.split(".").first + predicate = condition.split(".").last + # this is a nested field and must be treated differently + nested = { + "nested" => { + + "path" => path, + "query" => { + "bool" => { + "must" => { + "term" => { + # "person.name" => "oliver wendell holmes" + # Remove CR's added by hidden input field values with returns + facet => filter[1].gsub(/\r/, "") + # "person.role" => "judge" + subject => predicate + } + } + } + } + } + } + filter_list << nested + #ordinary nested facet + elsif filter[0].include?(".") path = filter[0].split(".").first # this is a nested field and must be treated differently nested = { From fa5c245dfeba7a951d335d95c94948e64566622b Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 24 May 2022 17:26:08 -0500 Subject: [PATCH 08/22] change split character, add missing comma --- app/services/search_item_req.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index c58036d..f79894e 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -109,8 +109,8 @@ def facets facet = f.split("[") path = facet.split(".").first condition = f[/(?<=\[).+?(?=\])/] - subject = condition.split(".").first - predicate = condition.split(".").last + subject = condition.split("|").first + predicate = condition.split("|").last aggs[f] = { "nested" => { "path" => path @@ -204,8 +204,8 @@ def filters facet = f.split("[") path = facet.split(".").first condition = f[/(?<=\[).+?(?=\])/] - subject = condition.split(".").first - predicate = condition.split(".").last + subject = condition.split("|").first + predicate = condition.split("|").last # this is a nested field and must be treated differently nested = { "nested" => { @@ -217,7 +217,7 @@ def filters "term" => { # "person.name" => "oliver wendell holmes" # Remove CR's added by hidden input field values with returns - facet => filter[1].gsub(/\r/, "") + facet => filter[1].gsub(/\r/, ""), # "person.role" => "judge" subject => predicate } From 21310ef0548c6b224da50c2b0f79f5ddd98747c2 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 26 May 2022 11:24:05 -0500 Subject: [PATCH 09/22] parse the array for matching nested fields --- app/services/search_item_req.rb | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index f79894e..d63b755 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -84,7 +84,6 @@ def facets Array.wrap(@params["facet"]).each do |f| # histograms use a different ordering terminology than normal aggs f_type = type == "_term" ? "_key" : "_count" - if f.include?("date") || f[/_d$/] # NOTE: if nested fields will ever have dates we will # need to refactor this to be available to both @@ -105,13 +104,18 @@ def facets } } #nested facet, matching on another nested facet + elsif f.include?("[") - facet = f.split("[") - path = facet.split(".").first - condition = f[/(?<=\[).+?(?=\])/] + # will be an array including the original, and an alternate aggregation name + options = JSON.parse(f) + agg_name = options[1] + original = options[2] + facet = original.split("[") + path = original.split(".").first + condition = original[/(?<=\[).+?(?=\])/] subject = condition.split("|").first predicate = condition.split("|").last - aggs[f] = { + aggs[agg_name] = { "nested" => { "path" => path }, @@ -122,7 +126,7 @@ def facets } }, "aggs" => { - f => { + agg_name => { "terms" => { "field" => facet, "order" => { type => dir }, @@ -132,7 +136,7 @@ def facets "top_matches" => { "top_hits" => { "_source" => { - "includes" => [ f ] + "includes" => [ agg_name ] }, "size" => 1 } @@ -201,9 +205,11 @@ def filters filters.each do |filter| # NESTED matching if filter[0].include?("[") - facet = f.split("[") - path = facet.split(".").first - condition = f[/(?<=\[).+?(?=\])/] + options = JSON.parse(f) + original = options[2] + facet = original.split("[") + path = original.split(".").first + condition = original[/(?<=\[).+?(?=\])/] subject = condition.split("|").first predicate = condition.split("|").last # this is a nested field and must be treated differently From 4656dc6e104292fbd2982673d62321b738378710 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 26 May 2022 16:21:47 -0500 Subject: [PATCH 10/22] change how compound facet name is parsed --- app/services/search_item_req.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index d63b755..e8231a0 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -108,10 +108,10 @@ def facets elsif f.include?("[") # will be an array including the original, and an alternate aggregation name options = JSON.parse(f) + original = options[0] agg_name = options[1] - original = options[2] - facet = original.split("[") - path = original.split(".").first + facet = original.split("[")[0] + path = facet.split(".").first condition = original[/(?<=\[).+?(?=\])/] subject = condition.split("|").first predicate = condition.split("|").last @@ -206,9 +206,9 @@ def filters # NESTED matching if filter[0].include?("[") options = JSON.parse(f) - original = options[2] - facet = original.split("[") - path = original.split(".").first + original = options[1] + facet = original.split("[")[0] + path = facet.split(".").first condition = original[/(?<=\[).+?(?=\])/] subject = condition.split("|").first predicate = condition.split("|").last From 2b3af8e3e5a7b8ce6df764504dea4710e833bf48 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 26 May 2022 16:22:44 -0500 Subject: [PATCH 11/22] use facet name as agg name this is not the most semantically correct, but it doesn't break anything and orchid seems to expect it --- app/services/search_item_req.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index e8231a0..c30c9ea 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -115,7 +115,7 @@ def facets condition = original[/(?<=\[).+?(?=\])/] subject = condition.split("|").first predicate = condition.split("|").last - aggs[agg_name] = { + aggs[f] = { "nested" => { "path" => path }, @@ -126,7 +126,7 @@ def facets } }, "aggs" => { - agg_name => { + f => { "terms" => { "field" => facet, "order" => { type => dir }, From 100ac901e8271799ed4319edfdc0ab498ab391ad Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 26 May 2022 16:23:20 -0500 Subject: [PATCH 12/22] change query to filter this is necessary to make this sort of query work --- app/services/search_item_req.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index c30c9ea..0981253 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -120,7 +120,7 @@ def facets "path" => path }, "aggs" => { - "query" => { + "filter" => { "term" => { subject => predicate } From 3dd66085004562bad1060ed06073d73145354238 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Fri, 27 May 2022 12:31:31 -0500 Subject: [PATCH 13/22] fix nested filter aggregation so it doesn't cause 400 error --- app/services/search_item_req.rb | 44 +++++++++++++++++---------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index 0981253..a3aa58d 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -115,30 +115,32 @@ def facets condition = original[/(?<=\[).+?(?=\])/] subject = condition.split("|").first predicate = condition.split("|").last - aggs[f] = { + aggs[agg_name] = { "nested" => { "path" => path }, "aggs" => { - "filter" => { - "term" => { - subject => predicate - } - }, - "aggs" => { - f => { - "terms" => { - "field" => facet, - "order" => { type => dir }, - "size" => size - }, - "aggs" => { - "top_matches" => { - "top_hits" => { - "_source" => { - "includes" => [ agg_name ] - }, - "size" => 1 + agg_name => { + "filter" => { + "term" => { + subject => predicate + } + }, + "aggs" => { + agg_name => { + "terms" => { + "field" => facet, + "order" => { type => dir }, + "size" => size + }, + "aggs" => { + "top_matches" => { + "top_hits" => { + "_source" => { + "includes" => [ agg_name ] + }, + "size" => 1 + } } } } @@ -220,7 +222,7 @@ def filters "query" => { "bool" => { "must" => { - "term" => { + "terms" => { # "person.name" => "oliver wendell holmes" # Remove CR's added by hidden input field values with returns facet => filter[1].gsub(/\r/, ""), From fafb3a6db8d8c1746cf7e38de380f74635e54a72 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 31 May 2022 12:14:30 -0500 Subject: [PATCH 14/22] check for deeper nesting of buckets --- app/services/search_item_res.rb | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/app/services/search_item_res.rb b/app/services/search_item_res.rb index a82a199..04843d9 100644 --- a/app/services/search_item_res.rb +++ b/app/services/search_item_res.rb @@ -18,7 +18,6 @@ def build_response # strip out only the fields for the item response items = combine_highlights facets = reformat_facets - { "code" => 200, "count" => count, @@ -89,8 +88,7 @@ def reformat_facets facets = {} raw_facets.each do |field, info| facets[field] = {} - # nested fields do not have buckets at this level of response structure - buckets = info.key?("buckets") ? info["buckets"] : info.dig(field, "buckets") + buckets = get_buckets(info, field) if buckets buckets.each { |b| format_bucket_value(facets, field, b) } else @@ -110,4 +108,18 @@ def remove_nonword_chars(term) transliterated.gsub(/<\/?(?:em|strong|u)>|\W/, "").downcase end + def get_buckets(info, field) + buckets = nil + # ordinary facet + if info.key?("buckets") + buckets = info["buckets"] + # nested facet + elsif info.dig(field, "buckets") + buckets = info.dig(field, "buckets") + # filtered facet + else + buckets = info.dig(field, field, "buckets") + end + buckets + end end From 2ade65b6311e07d9654c9e5d7d6a939b77d2d080 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 1 Jun 2022 09:35:50 -0500 Subject: [PATCH 15/22] Change separator --- app/services/search_item_req.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index a3aa58d..e647119 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -113,8 +113,8 @@ def facets facet = original.split("[")[0] path = facet.split(".").first condition = original[/(?<=\[).+?(?=\])/] - subject = condition.split("|").first - predicate = condition.split("|").last + subject = condition.split("#").first + predicate = condition.split("#").last aggs[agg_name] = { "nested" => { "path" => path @@ -212,8 +212,8 @@ def filters facet = original.split("[")[0] path = facet.split(".").first condition = original[/(?<=\[).+?(?=\])/] - subject = condition.split("|").first - predicate = condition.split("|").last + subject = condition.split("#").first + predicate = condition.split("#").last # this is a nested field and must be treated differently nested = { "nested" => { From 3c223e4d94b11baf998c5370ee71b1c928b87987 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 1 Jun 2022 09:36:21 -0500 Subject: [PATCH 16/22] Fix parsing and query for filter matching --- app/services/search_item_req.rb | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index e647119..2c39b5b 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -205,10 +205,9 @@ def filters # (type 2 will only be used for dates) filters = fields.map {|f| f.split(@@filter_separator, 3) } filters.each do |filter| - # NESTED matching + # filter aggregation with nesting if filter[0].include?("[") - options = JSON.parse(f) - original = options[1] + original = filter[0] facet = original.split("[")[0] path = facet.split(".").first condition = original[/(?<=\[).+?(?=\])/] @@ -217,17 +216,14 @@ def filters # this is a nested field and must be treated differently nested = { "nested" => { - "path" => path, "query" => { "bool" => { "must" => { - "terms" => { + "term" => { # "person.name" => "oliver wendell holmes" # Remove CR's added by hidden input field values with returns - facet => filter[1].gsub(/\r/, ""), - # "person.role" => "judge" - subject => predicate + facet => filter[1].gsub(/\r/, "") } } } From ef223076cee6732c649d2f6e44db86067b445611 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 2 Jun 2022 15:04:58 -0500 Subject: [PATCH 17/22] rewrite filtered aggregation to be either nested or not --- app/services/search_item_req.rb | 71 +++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 29 deletions(-) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index 2c39b5b..81bd0ae 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -107,48 +107,61 @@ def facets elsif f.include?("[") # will be an array including the original, and an alternate aggregation name + + options = JSON.parse(f) original = options[0] agg_name = options[1] facet = original.split("[")[0] - path = facet.split(".").first + # may or may not be nested + nested = facet.include?(".") + if nested + path = facet.split(".").first + end condition = original[/(?<=\[).+?(?=\])/] subject = condition.split("#").first predicate = condition.split("#").last - aggs[agg_name] = { - "nested" => { - "path" => path - }, - "aggs" => { - agg_name => { - "filter" => { - "term" => { - subject => predicate - } - }, - "aggs" => { - agg_name => { - "terms" => { - "field" => facet, - "order" => { type => dir }, - "size" => size - }, - "aggs" => { - "top_matches" => { - "top_hits" => { - "_source" => { - "includes" => [ agg_name ] - }, - "size" => 1 - } + aggregation = { + # common to nested and non-nested + "filter" => { + "term" => { + subject => predicate + } + }, + "aggs" => { + agg_name => { + "terms" => { + "field" => facet, + "order" => { type => dir }, + "size" => size + }, + "aggs" => { + "top_matches" => { + "top_hits" => { + "_source" => { + "includes" => [ agg_name ] + }, + "size" => 1 } } } } } } - } - # ordinary nested facet + #interpolate above hash into nested query + if nested + aggs[agg_name] = { + "nested" => { + "path" => path + }, + "aggs" => { + agg_name => aggregation + } + } + else + #otherwise it is the whole query + aggs[agg_name] = aggregation + end elsif f.include?(".") path = f.split(".").first aggs[f] = { From 16a490d5d4b4749b85825740c491a7590b583ed7 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 2 Jun 2022 16:05:44 -0500 Subject: [PATCH 18/22] filtering on a single item can either be nested or not --- app/services/search_item_req.rb | 35 +++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index 81bd0ae..cd51f8f 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -222,28 +222,33 @@ def filters if filter[0].include?("[") original = filter[0] facet = original.split("[")[0] - path = facet.split(".").first + nested = facet.include?(".") + if nested + path = facet.split(".").first + end condition = original[/(?<=\[).+?(?=\])/] subject = condition.split("#").first predicate = condition.split("#").last - # this is a nested field and must be treated differently - nested = { - "nested" => { - "path" => path, - "query" => { - "bool" => { - "must" => { - "term" => { - # "person.name" => "oliver wendell holmes" - # Remove CR's added by hidden input field values with returns - facet => filter[1].gsub(/\r/, "") - } + query = { + "term" => { + # "person.name" => "oliver wendell holmes" + # Remove CR's added by hidden input field values with returns + facet => filter[1].gsub(/\r/, "") + } + } + if nested + query = { + "nested" => { + "path" => path, + "query" => { + "bool" => { + "must" => query } } } } - } - filter_list << nested + end + filter_list << query #ordinary nested facet elsif filter[0].include?(".") path = filter[0].split(".").first From b038be028989937f356c8e4f6f79d089b497e63a Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 26 Sep 2022 10:15:19 -0500 Subject: [PATCH 19/22] update config for server --- .ruby-gemset | 2 +- config/environments/development.rb | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.ruby-gemset b/.ruby-gemset index eedd89b..fcf5595 100644 --- a/.ruby-gemset +++ b/.ruby-gemset @@ -1 +1 @@ -api +api-v2 diff --git a/config/environments/development.rb b/config/environments/development.rb index 1e22e09..14a0a40 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -61,4 +61,5 @@ # CDRH CONFIGURATION config.hosts << "cdrhdev1.unl.edu" + config.hosts << "whitman-dev.unl.edu" end From 6eaa38b74dedd8c2b84c1c98bb24549174518a7e Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 19 Oct 2022 12:03:51 -0500 Subject: [PATCH 20/22] revise query to match both the facet and the filter --- app/services/search_item_req.rb | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index cd51f8f..b34a75a 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -229,12 +229,13 @@ def filters condition = original[/(?<=\[).+?(?=\])/] subject = condition.split("#").first predicate = condition.split("#").last - query = { - "term" => { - # "person.name" => "oliver wendell holmes" - # Remove CR's added by hidden input field values with returns - facet => filter[1].gsub(/\r/, "") - } + term_match = { + # "person.name" => "oliver wendell holmes" + # Remove CR's added by hidden input field values with returns + facet => filter[1].gsub(/\r/, "") + } + term_filter = { + subject => predicate } if nested query = { @@ -242,7 +243,10 @@ def filters "path" => path, "query" => { "bool" => { - "must" => query + "must" => [ + { "match" => term_filter }, + { "match" => term_match } + ] } } } From 88a8f80915f7528ded47734ff9328d4a651aaf6b Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 20 Oct 2022 09:57:32 -0500 Subject: [PATCH 21/22] use reverse nested agg for correct item count --- app/services/search_item_req.rb | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index b34a75a..e298836 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -51,6 +51,8 @@ def build_request # add bool to request body req["query"]["bool"] = bool + # uncomment below line to log ES query for debugging + # puts req.to_json() return req end @@ -136,12 +138,17 @@ def facets "size" => size }, "aggs" => { - "top_matches" => { - "top_hits" => { - "_source" => { - "includes" => [ agg_name ] - }, - "size" => 1 + "field_to_item" => { + "reverse_nested" => {}, + "aggs" => { + "top_matches" => { + "top_hits" => { + "_source" => { + "includes" => [ agg_name ] + }, + "size" => 1 + } + } } } } From dd4bacdc941b14d8180e3307613b4099cf000621 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 20 Oct 2022 10:01:28 -0500 Subject: [PATCH 22/22] used doc_count from reverse nested if it exists --- app/services/search_item_res.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/search_item_res.rb b/app/services/search_item_res.rb index 04843d9..05210e7 100644 --- a/app/services/search_item_res.rb +++ b/app/services/search_item_res.rb @@ -65,7 +65,7 @@ def format_bucket_value(facets, field, bucket) # dates return in wonktastic ways, so grab key_as_string instead of gibberish number # but otherwise just grab the key if key_as_string unavailable key = bucket.key?("key_as_string") ? bucket["key_as_string"] : bucket["key"] - val = bucket["doc_count"] + val = bucket.key?("field_to_item") ? bucket["field_to_item"]["doc_count"] : bucket["doc_count"] source = key # top_matches is a top_hits aggregation which returns a list of terms # which were used for the facet.