From 561aeeb7012e64fff6f2df7f6d0920607cdf9fb6 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 19 May 2022 13:04:06 -0500 Subject: [PATCH 01/39] upgrade Puma --- Gemfile | 2 +- Gemfile.lock | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index 88ac8b9..80c970e 100644 --- a/Gemfile +++ b/Gemfile @@ -11,7 +11,7 @@ gem 'rails', '~> 6.0.2' # Use sqlite3 as the database for Active Record gem 'sqlite3' # Use Puma as the app server -gem 'puma', '~> 3.7' +gem 'puma', '~> 4.3' # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder # gem 'jbuilder', '~> 2.5' # Use Redis adapter to run Action Cable in production diff --git a/Gemfile.lock b/Gemfile.lock index c38a145..d94298a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -96,7 +96,8 @@ GEM nokogiri (1.13.6) mini_portile2 (~> 2.8.0) racc (~> 1.4) - puma (3.12.6) + puma (4.3.12) + nio4r (~> 2.0) racc (1.6.0) rack (2.2.3) rack-test (1.1.0) @@ -168,7 +169,7 @@ DEPENDENCIES bootsnap byebug listen (>= 3.0.5, < 3.2) - puma (~> 3.7) + puma (~> 4.3) rails (~> 6.0.2) rest-client (>= 2.1.0.rc1, < 2.2) spring From deb704ac4e5273980a620adf52b59fafa5e8bee5 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 19 May 2022 13:13:40 -0500 Subject: [PATCH 02/39] add specific version of puma to avoid security warnings --- Gemfile | 2 +- Gemfile.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index 80c970e..6d7da95 100644 --- a/Gemfile +++ b/Gemfile @@ -11,7 +11,7 @@ gem 'rails', '~> 6.0.2' # Use sqlite3 as the database for Active Record gem 'sqlite3' # Use Puma as the app server -gem 'puma', '~> 4.3' +gem 'puma', '>= 4.3.12' # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder # gem 'jbuilder', '~> 2.5' # Use Redis adapter to run Action Cable in production diff --git a/Gemfile.lock b/Gemfile.lock index d94298a..3c04aa9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -169,7 +169,7 @@ DEPENDENCIES bootsnap byebug listen (>= 3.0.5, < 3.2) - puma (~> 4.3) + puma (>= 4.3.12) rails (~> 6.0.2) rest-client (>= 2.1.0.rc1, < 2.2) spring From 065857a7bef10fe1152b769d6ae3e59fc1dcc3c3 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 25 May 2022 10:46:36 -0500 Subject: [PATCH 03/39] update .ruby-version --- .ruby-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ruby-version b/.ruby-version index 324db8d..4e34c4d 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -ruby-2.6.8 +ruby-2.7.6 From 38b02b7a32e1a343d5ec714aff18df7cb930f382 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 25 May 2022 10:50:33 -0500 Subject: [PATCH 04/39] update to later version of puma --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 6d7da95..4170559 100644 --- a/Gemfile +++ b/Gemfile @@ -11,7 +11,7 @@ gem 'rails', '~> 6.0.2' # Use sqlite3 as the database for Active Record gem 'sqlite3' # Use Puma as the app server -gem 'puma', '>= 4.3.12' +gem 'puma', '>= 5.6' # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder # gem 'jbuilder', '~> 2.5' # Use Redis adapter to run Action Cable in production From e763bb1f0c9f98873a0cf7a004deff586214440c Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 25 May 2022 10:51:14 -0500 Subject: [PATCH 05/39] another round of gem updates --- Gemfile.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 3c04aa9..3f1cd9c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -69,7 +69,7 @@ GEM globalid (1.0.0) activesupport (>= 5.0) http-accept (1.7.0) - http-cookie (1.0.4) + http-cookie (1.0.5) domain_name (~> 0.5) i18n (1.10.0) concurrent-ruby (~> 1.0) @@ -96,7 +96,7 @@ GEM nokogiri (1.13.6) mini_portile2 (~> 2.8.0) racc (~> 1.4) - puma (4.3.12) + puma (5.6.4) nio4r (~> 2.0) racc (1.6.0) rack (2.2.3) @@ -169,7 +169,7 @@ DEPENDENCIES bootsnap byebug listen (>= 3.0.5, < 3.2) - puma (>= 4.3.12) + puma (>= 5.6) rails (~> 6.0.2) rest-client (>= 2.1.0.rc1, < 2.2) spring From 2769319a13dfa6fe3877fcd6cc6c2dddbc7ae3bc Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 24 May 2022 13:40:03 -0500 Subject: [PATCH 06/39] add facet for matching nested facet --- app/services/search_item_req.rb | 39 +++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index e9260c0..64c683b 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -104,6 +104,45 @@ def facets "order" => { f_type => dir }, } } + elsif f.include?("[") + #or nest it inside the next one? + #this will be the same + facet = f.split("[") + path = facet.split(".").first + condition = f[/(?<=\[).+?(?=\])/] + subject = condition.split(".").first + predicate = condition.split(".").last + aggs[f] = { + "nested" => { + "path" => path + }, + "aggs" => { + "query" => { + "term" => { + subject => predicate + } + }, + "aggs" => { + f => { + "terms" => { + "field" => facet, + "order" => { type => dir }, + "size" => size + }, + "aggs" => { + "top_matches" => { + "top_hits" => { + "_source" => { + "includes" => [ f ] + }, + "size" => 1 + } + } + } + } + } + } + } # if nested, has extra syntax elsif f.include?(".") path = f.split(".").first From 3b33d4afcf38391e9825e3f81b41dbe68837c6d9 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 24 May 2022 14:31:49 -0500 Subject: [PATCH 07/39] add filter for matching nested facet --- app/services/search_item_req.rb | 37 ++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index 64c683b..c58036d 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -104,9 +104,8 @@ def facets "order" => { f_type => dir }, } } + #nested facet, matching on another nested facet elsif f.include?("[") - #or nest it inside the next one? - #this will be the same facet = f.split("[") path = facet.split(".").first condition = f[/(?<=\[).+?(?=\])/] @@ -143,7 +142,7 @@ def facets } } } - # if nested, has extra syntax + # ordinary nested facet elsif f.include?(".") path = f.split(".").first aggs[f] = { @@ -200,8 +199,36 @@ def filters # (type 2 will only be used for dates) filters = fields.map {|f| f.split(@@filter_separator, 3) } filters.each do |filter| - # NESTED FIELD FILTER - if filter[0].include?(".") + # NESTED matching + if filter[0].include?("[") + facet = f.split("[") + path = facet.split(".").first + condition = f[/(?<=\[).+?(?=\])/] + subject = condition.split(".").first + predicate = condition.split(".").last + # this is a nested field and must be treated differently + nested = { + "nested" => { + + "path" => path, + "query" => { + "bool" => { + "must" => { + "term" => { + # "person.name" => "oliver wendell holmes" + # Remove CR's added by hidden input field values with returns + facet => filter[1].gsub(/\r/, "") + # "person.role" => "judge" + subject => predicate + } + } + } + } + } + } + filter_list << nested + #ordinary nested facet + elsif filter[0].include?(".") path = filter[0].split(".").first # this is a nested field and must be treated differently nested = { From fa5c245dfeba7a951d335d95c94948e64566622b Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 24 May 2022 17:26:08 -0500 Subject: [PATCH 08/39] change split character, add missing comma --- app/services/search_item_req.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index c58036d..f79894e 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -109,8 +109,8 @@ def facets facet = f.split("[") path = facet.split(".").first condition = f[/(?<=\[).+?(?=\])/] - subject = condition.split(".").first - predicate = condition.split(".").last + subject = condition.split("|").first + predicate = condition.split("|").last aggs[f] = { "nested" => { "path" => path @@ -204,8 +204,8 @@ def filters facet = f.split("[") path = facet.split(".").first condition = f[/(?<=\[).+?(?=\])/] - subject = condition.split(".").first - predicate = condition.split(".").last + subject = condition.split("|").first + predicate = condition.split("|").last # this is a nested field and must be treated differently nested = { "nested" => { @@ -217,7 +217,7 @@ def filters "term" => { # "person.name" => "oliver wendell holmes" # Remove CR's added by hidden input field values with returns - facet => filter[1].gsub(/\r/, "") + facet => filter[1].gsub(/\r/, ""), # "person.role" => "judge" subject => predicate } From 21310ef0548c6b224da50c2b0f79f5ddd98747c2 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 26 May 2022 11:24:05 -0500 Subject: [PATCH 09/39] parse the array for matching nested fields --- app/services/search_item_req.rb | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index f79894e..d63b755 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -84,7 +84,6 @@ def facets Array.wrap(@params["facet"]).each do |f| # histograms use a different ordering terminology than normal aggs f_type = type == "_term" ? "_key" : "_count" - if f.include?("date") || f[/_d$/] # NOTE: if nested fields will ever have dates we will # need to refactor this to be available to both @@ -105,13 +104,18 @@ def facets } } #nested facet, matching on another nested facet + elsif f.include?("[") - facet = f.split("[") - path = facet.split(".").first - condition = f[/(?<=\[).+?(?=\])/] + # will be an array including the original, and an alternate aggregation name + options = JSON.parse(f) + agg_name = options[1] + original = options[2] + facet = original.split("[") + path = original.split(".").first + condition = original[/(?<=\[).+?(?=\])/] subject = condition.split("|").first predicate = condition.split("|").last - aggs[f] = { + aggs[agg_name] = { "nested" => { "path" => path }, @@ -122,7 +126,7 @@ def facets } }, "aggs" => { - f => { + agg_name => { "terms" => { "field" => facet, "order" => { type => dir }, @@ -132,7 +136,7 @@ def facets "top_matches" => { "top_hits" => { "_source" => { - "includes" => [ f ] + "includes" => [ agg_name ] }, "size" => 1 } @@ -201,9 +205,11 @@ def filters filters.each do |filter| # NESTED matching if filter[0].include?("[") - facet = f.split("[") - path = facet.split(".").first - condition = f[/(?<=\[).+?(?=\])/] + options = JSON.parse(f) + original = options[2] + facet = original.split("[") + path = original.split(".").first + condition = original[/(?<=\[).+?(?=\])/] subject = condition.split("|").first predicate = condition.split("|").last # this is a nested field and must be treated differently From 4656dc6e104292fbd2982673d62321b738378710 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 26 May 2022 16:21:47 -0500 Subject: [PATCH 10/39] change how compound facet name is parsed --- app/services/search_item_req.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index d63b755..e8231a0 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -108,10 +108,10 @@ def facets elsif f.include?("[") # will be an array including the original, and an alternate aggregation name options = JSON.parse(f) + original = options[0] agg_name = options[1] - original = options[2] - facet = original.split("[") - path = original.split(".").first + facet = original.split("[")[0] + path = facet.split(".").first condition = original[/(?<=\[).+?(?=\])/] subject = condition.split("|").first predicate = condition.split("|").last @@ -206,9 +206,9 @@ def filters # NESTED matching if filter[0].include?("[") options = JSON.parse(f) - original = options[2] - facet = original.split("[") - path = original.split(".").first + original = options[1] + facet = original.split("[")[0] + path = facet.split(".").first condition = original[/(?<=\[).+?(?=\])/] subject = condition.split("|").first predicate = condition.split("|").last From 2b3af8e3e5a7b8ce6df764504dea4710e833bf48 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 26 May 2022 16:22:44 -0500 Subject: [PATCH 11/39] use facet name as agg name this is not the most semantically correct, but it doesn't break anything and orchid seems to expect it --- app/services/search_item_req.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index e8231a0..c30c9ea 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -115,7 +115,7 @@ def facets condition = original[/(?<=\[).+?(?=\])/] subject = condition.split("|").first predicate = condition.split("|").last - aggs[agg_name] = { + aggs[f] = { "nested" => { "path" => path }, @@ -126,7 +126,7 @@ def facets } }, "aggs" => { - agg_name => { + f => { "terms" => { "field" => facet, "order" => { type => dir }, From 100ac901e8271799ed4319edfdc0ab498ab391ad Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 26 May 2022 16:23:20 -0500 Subject: [PATCH 12/39] change query to filter this is necessary to make this sort of query work --- app/services/search_item_req.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index c30c9ea..0981253 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -120,7 +120,7 @@ def facets "path" => path }, "aggs" => { - "query" => { + "filter" => { "term" => { subject => predicate } From 3dd66085004562bad1060ed06073d73145354238 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Fri, 27 May 2022 12:31:31 -0500 Subject: [PATCH 13/39] fix nested filter aggregation so it doesn't cause 400 error --- app/services/search_item_req.rb | 44 +++++++++++++++++---------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index 0981253..a3aa58d 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -115,30 +115,32 @@ def facets condition = original[/(?<=\[).+?(?=\])/] subject = condition.split("|").first predicate = condition.split("|").last - aggs[f] = { + aggs[agg_name] = { "nested" => { "path" => path }, "aggs" => { - "filter" => { - "term" => { - subject => predicate - } - }, - "aggs" => { - f => { - "terms" => { - "field" => facet, - "order" => { type => dir }, - "size" => size - }, - "aggs" => { - "top_matches" => { - "top_hits" => { - "_source" => { - "includes" => [ agg_name ] - }, - "size" => 1 + agg_name => { + "filter" => { + "term" => { + subject => predicate + } + }, + "aggs" => { + agg_name => { + "terms" => { + "field" => facet, + "order" => { type => dir }, + "size" => size + }, + "aggs" => { + "top_matches" => { + "top_hits" => { + "_source" => { + "includes" => [ agg_name ] + }, + "size" => 1 + } } } } @@ -220,7 +222,7 @@ def filters "query" => { "bool" => { "must" => { - "term" => { + "terms" => { # "person.name" => "oliver wendell holmes" # Remove CR's added by hidden input field values with returns facet => filter[1].gsub(/\r/, ""), From fafb3a6db8d8c1746cf7e38de380f74635e54a72 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 31 May 2022 12:14:30 -0500 Subject: [PATCH 14/39] check for deeper nesting of buckets --- app/services/search_item_res.rb | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/app/services/search_item_res.rb b/app/services/search_item_res.rb index a82a199..04843d9 100644 --- a/app/services/search_item_res.rb +++ b/app/services/search_item_res.rb @@ -18,7 +18,6 @@ def build_response # strip out only the fields for the item response items = combine_highlights facets = reformat_facets - { "code" => 200, "count" => count, @@ -89,8 +88,7 @@ def reformat_facets facets = {} raw_facets.each do |field, info| facets[field] = {} - # nested fields do not have buckets at this level of response structure - buckets = info.key?("buckets") ? info["buckets"] : info.dig(field, "buckets") + buckets = get_buckets(info, field) if buckets buckets.each { |b| format_bucket_value(facets, field, b) } else @@ -110,4 +108,18 @@ def remove_nonword_chars(term) transliterated.gsub(/<\/?(?:em|strong|u)>|\W/, "").downcase end + def get_buckets(info, field) + buckets = nil + # ordinary facet + if info.key?("buckets") + buckets = info["buckets"] + # nested facet + elsif info.dig(field, "buckets") + buckets = info.dig(field, "buckets") + # filtered facet + else + buckets = info.dig(field, field, "buckets") + end + buckets + end end From 2ade65b6311e07d9654c9e5d7d6a939b77d2d080 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 1 Jun 2022 09:35:50 -0500 Subject: [PATCH 15/39] Change separator --- app/services/search_item_req.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index a3aa58d..e647119 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -113,8 +113,8 @@ def facets facet = original.split("[")[0] path = facet.split(".").first condition = original[/(?<=\[).+?(?=\])/] - subject = condition.split("|").first - predicate = condition.split("|").last + subject = condition.split("#").first + predicate = condition.split("#").last aggs[agg_name] = { "nested" => { "path" => path @@ -212,8 +212,8 @@ def filters facet = original.split("[")[0] path = facet.split(".").first condition = original[/(?<=\[).+?(?=\])/] - subject = condition.split("|").first - predicate = condition.split("|").last + subject = condition.split("#").first + predicate = condition.split("#").last # this is a nested field and must be treated differently nested = { "nested" => { From 3c223e4d94b11baf998c5370ee71b1c928b87987 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 1 Jun 2022 09:36:21 -0500 Subject: [PATCH 16/39] Fix parsing and query for filter matching --- app/services/search_item_req.rb | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index e647119..2c39b5b 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -205,10 +205,9 @@ def filters # (type 2 will only be used for dates) filters = fields.map {|f| f.split(@@filter_separator, 3) } filters.each do |filter| - # NESTED matching + # filter aggregation with nesting if filter[0].include?("[") - options = JSON.parse(f) - original = options[1] + original = filter[0] facet = original.split("[")[0] path = facet.split(".").first condition = original[/(?<=\[).+?(?=\])/] @@ -217,17 +216,14 @@ def filters # this is a nested field and must be treated differently nested = { "nested" => { - "path" => path, "query" => { "bool" => { "must" => { - "terms" => { + "term" => { # "person.name" => "oliver wendell holmes" # Remove CR's added by hidden input field values with returns - facet => filter[1].gsub(/\r/, ""), - # "person.role" => "judge" - subject => predicate + facet => filter[1].gsub(/\r/, "") } } } From ef223076cee6732c649d2f6e44db86067b445611 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 2 Jun 2022 15:04:58 -0500 Subject: [PATCH 17/39] rewrite filtered aggregation to be either nested or not --- app/services/search_item_req.rb | 71 +++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 29 deletions(-) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index 2c39b5b..81bd0ae 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -107,48 +107,61 @@ def facets elsif f.include?("[") # will be an array including the original, and an alternate aggregation name + + options = JSON.parse(f) original = options[0] agg_name = options[1] facet = original.split("[")[0] - path = facet.split(".").first + # may or may not be nested + nested = facet.include?(".") + if nested + path = facet.split(".").first + end condition = original[/(?<=\[).+?(?=\])/] subject = condition.split("#").first predicate = condition.split("#").last - aggs[agg_name] = { - "nested" => { - "path" => path - }, - "aggs" => { - agg_name => { - "filter" => { - "term" => { - subject => predicate - } - }, - "aggs" => { - agg_name => { - "terms" => { - "field" => facet, - "order" => { type => dir }, - "size" => size - }, - "aggs" => { - "top_matches" => { - "top_hits" => { - "_source" => { - "includes" => [ agg_name ] - }, - "size" => 1 - } + aggregation = { + # common to nested and non-nested + "filter" => { + "term" => { + subject => predicate + } + }, + "aggs" => { + agg_name => { + "terms" => { + "field" => facet, + "order" => { type => dir }, + "size" => size + }, + "aggs" => { + "top_matches" => { + "top_hits" => { + "_source" => { + "includes" => [ agg_name ] + }, + "size" => 1 } } } } } } - } - # ordinary nested facet + #interpolate above hash into nested query + if nested + aggs[agg_name] = { + "nested" => { + "path" => path + }, + "aggs" => { + agg_name => aggregation + } + } + else + #otherwise it is the whole query + aggs[agg_name] = aggregation + end elsif f.include?(".") path = f.split(".").first aggs[f] = { From 16a490d5d4b4749b85825740c491a7590b583ed7 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 2 Jun 2022 16:05:44 -0500 Subject: [PATCH 18/39] filtering on a single item can either be nested or not --- app/services/search_item_req.rb | 35 +++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index 81bd0ae..cd51f8f 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -222,28 +222,33 @@ def filters if filter[0].include?("[") original = filter[0] facet = original.split("[")[0] - path = facet.split(".").first + nested = facet.include?(".") + if nested + path = facet.split(".").first + end condition = original[/(?<=\[).+?(?=\])/] subject = condition.split("#").first predicate = condition.split("#").last - # this is a nested field and must be treated differently - nested = { - "nested" => { - "path" => path, - "query" => { - "bool" => { - "must" => { - "term" => { - # "person.name" => "oliver wendell holmes" - # Remove CR's added by hidden input field values with returns - facet => filter[1].gsub(/\r/, "") - } + query = { + "term" => { + # "person.name" => "oliver wendell holmes" + # Remove CR's added by hidden input field values with returns + facet => filter[1].gsub(/\r/, "") + } + } + if nested + query = { + "nested" => { + "path" => path, + "query" => { + "bool" => { + "must" => query } } } } - } - filter_list << nested + end + filter_list << query #ordinary nested facet elsif filter[0].include?(".") path = filter[0].split(".").first From b038be028989937f356c8e4f6f79d089b497e63a Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 26 Sep 2022 10:15:19 -0500 Subject: [PATCH 19/39] update config for server --- .ruby-gemset | 2 +- config/environments/development.rb | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.ruby-gemset b/.ruby-gemset index eedd89b..fcf5595 100644 --- a/.ruby-gemset +++ b/.ruby-gemset @@ -1 +1 @@ -api +api-v2 diff --git a/config/environments/development.rb b/config/environments/development.rb index 1e22e09..14a0a40 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -61,4 +61,5 @@ # CDRH CONFIGURATION config.hosts << "cdrhdev1.unl.edu" + config.hosts << "whitman-dev.unl.edu" end From 6eaa38b74dedd8c2b84c1c98bb24549174518a7e Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 19 Oct 2022 12:03:51 -0500 Subject: [PATCH 20/39] revise query to match both the facet and the filter --- app/services/search_item_req.rb | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index cd51f8f..b34a75a 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -229,12 +229,13 @@ def filters condition = original[/(?<=\[).+?(?=\])/] subject = condition.split("#").first predicate = condition.split("#").last - query = { - "term" => { - # "person.name" => "oliver wendell holmes" - # Remove CR's added by hidden input field values with returns - facet => filter[1].gsub(/\r/, "") - } + term_match = { + # "person.name" => "oliver wendell holmes" + # Remove CR's added by hidden input field values with returns + facet => filter[1].gsub(/\r/, "") + } + term_filter = { + subject => predicate } if nested query = { @@ -242,7 +243,10 @@ def filters "path" => path, "query" => { "bool" => { - "must" => query + "must" => [ + { "match" => term_filter }, + { "match" => term_match } + ] } } } From 88a8f80915f7528ded47734ff9328d4a651aaf6b Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 20 Oct 2022 09:57:32 -0500 Subject: [PATCH 21/39] use reverse nested agg for correct item count --- app/services/search_item_req.rb | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index b34a75a..e298836 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -51,6 +51,8 @@ def build_request # add bool to request body req["query"]["bool"] = bool + # uncomment below line to log ES query for debugging + # puts req.to_json() return req end @@ -136,12 +138,17 @@ def facets "size" => size }, "aggs" => { - "top_matches" => { - "top_hits" => { - "_source" => { - "includes" => [ agg_name ] - }, - "size" => 1 + "field_to_item" => { + "reverse_nested" => {}, + "aggs" => { + "top_matches" => { + "top_hits" => { + "_source" => { + "includes" => [ agg_name ] + }, + "size" => 1 + } + } } } } From dd4bacdc941b14d8180e3307613b4099cf000621 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 20 Oct 2022 10:01:28 -0500 Subject: [PATCH 22/39] used doc_count from reverse nested if it exists --- app/services/search_item_res.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/search_item_res.rb b/app/services/search_item_res.rb index 04843d9..05210e7 100644 --- a/app/services/search_item_res.rb +++ b/app/services/search_item_res.rb @@ -65,7 +65,7 @@ def format_bucket_value(facets, field, bucket) # dates return in wonktastic ways, so grab key_as_string instead of gibberish number # but otherwise just grab the key if key_as_string unavailable key = bucket.key?("key_as_string") ? bucket["key_as_string"] : bucket["key"] - val = bucket["doc_count"] + val = bucket.key?("field_to_item") ? bucket["field_to_item"]["doc_count"] : bucket["doc_count"] source = key # top_matches is a top_hits aggregation which returns a list of terms # which were used for the facet. From bd739e0fce4155b0df1588c583b0b5213314fa13 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Fri, 21 Oct 2022 13:23:57 -0500 Subject: [PATCH 23/39] change key for new elasticsearch version --- app/services/search_item_req.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index e298836..497c84a 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -99,7 +99,7 @@ def facets aggs[f] = { "date_histogram" => { "field" => field, - "interval" => interval, + "calendar_interval" => interval, "format" => formatted, "min_doc_count" => 1, "order" => { f_type => dir }, From f0c3124cf43b982ea894a4bee00855944d65d589 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 24 Oct 2022 12:51:22 -0500 Subject: [PATCH 24/39] change order query to avoid deprecated '_term' --- app/services/search_item_req.rb | 10 +++++----- test/services/search_item_req_test.rb | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index 497c84a..56e6453 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -74,7 +74,7 @@ def facets dir = "desc" if @params["facet_sort"].present? sort_type, sort_dir = @params["facet_sort"].split(@@filter_separator) - type = "_term" if sort_type == "term" + type = "term" if sort_type == "term" dir = sort_dir if sort_dir == "asc" end @@ -85,7 +85,7 @@ def facets aggs = {} Array.wrap(@params["facet"]).each do |f| # histograms use a different ordering terminology than normal aggs - f_type = type == "_term" ? "_key" : "_count" + f_type = (type == "term") ? "_key" : "_count" if f.include?("date") || f[/_d$/] # NOTE: if nested fields will ever have dates we will # need to refactor this to be available to both @@ -134,7 +134,7 @@ def facets agg_name => { "terms" => { "field" => facet, - "order" => { type => dir }, + "order" => {f_type => dir}, "size" => size }, "aggs" => { @@ -179,7 +179,7 @@ def facets f => { "terms" => { "field" => f, - "order" => { type => dir }, + "order" => {f_type => dir}, "size" => size }, "aggs" => { @@ -199,7 +199,7 @@ def facets aggs[f] = { "terms" => { "field" => f, - "order" => { type => dir }, + "order" => { f_type => dir }, "size" => size }, "aggs" => { diff --git a/test/services/search_item_req_test.rb b/test/services/search_item_req_test.rb index 29bf323..c4d8197 100644 --- a/test/services/search_item_req_test.rb +++ b/test/services/search_item_req_test.rb @@ -44,7 +44,7 @@ def test_facets "facet" => [ "title", "subcategory" ] }).facets assert_equal( - {"title"=>{"terms"=>{"field"=>"title", "order"=>{"_term"=>"asc"}, "size"=>10}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["title"]}, "size"=>1}}}}, "subcategory"=>{"terms"=>{"field"=>"subcategory", "order"=>{"_term"=>"asc"}, "size"=>10}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["subcategory"]}, "size"=>1}}}}}, + {"title"=>{"terms"=>{"field"=>"title", "order"=>"asc", "size"=>10}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["title"]}, "size"=>1}}}}, "subcategory"=>{"terms"=>{"field"=>"subcategory", "order"=>"asc", "size"=>10}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["subcategory"]}, "size"=>1}}}}}, facets ) @@ -69,7 +69,7 @@ def test_facets "facet" => [ "creator.name" ] }).facets assert_equal( - {"creator.name"=>{"nested"=>{"path"=>"creator"}, "aggs"=>{"creator.name"=>{"terms"=>{"field"=>"creator.name", "order"=>{"_term"=>"desc"}, "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["creator.name"]}, "size"=>1}}}}}}}, + {"creator.name"=>{"nested"=>{"path"=>"creator"}, "aggs"=>{"creator.name"=>{"terms"=>{"field"=>"creator.name", "order"=>"desc", "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["creator.name"]}, "size"=>1}}}}}}}, facets ) @@ -83,14 +83,14 @@ def test_facets # sort term order specified facets = SearchItemReq.new({ "facet" => ["title", "format"], "facet_sort" => "term|desc" }).facets assert_equal( - {"title"=>{"terms"=>{"field"=>"title", "order"=>{"_term"=>"desc"}, "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["title"]}, "size"=>1}}}}, "format"=>{"terms"=>{"field"=>"format", "order"=>{"_term"=>"desc"}, "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["format"]}, "size"=>1}}}}}, + {"title"=>{"terms"=>{"field"=>"title", "order"=>"desc", "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["title"]}, "size"=>1}}}}, "format"=>{"terms"=>{"field"=>"format", "order"=>"desc", "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["format"]}, "size"=>1}}}}}, facets ) # sort term no order specified facets = SearchItemReq.new({ "facet" => ["title", "format"], "facet_sort" => "term" }).facets assert_equal( - {"title"=>{"terms"=>{"field"=>"title", "order"=>{"_term"=>"desc"}, "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["title"]}, "size"=>1}}}}, "format"=>{"terms"=>{"field"=>"format", "order"=>{"_term"=>"desc"}, "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["format"]}, "size"=>1}}}}}, + {"title"=>{"terms"=>{"field"=>"title", "order"=>"desc", "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["title"]}, "size"=>1}}}}, "format"=>{"terms"=>{"field"=>"format", "order"=>"desc", "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["format"]}, "size"=>1}}}}}, facets ) From 7441a9626ac5c223b5c6ac125d0538051b82ebac Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 26 Oct 2022 09:06:39 -0500 Subject: [PATCH 25/39] gitignore master key --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 76fb9c4..a3bd71e 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,5 @@ bower.json .byebug_history .DS_Store + +/config/master.key From 255f9db6ff73738cd528591d404f292f6fcce33f Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 26 Oct 2022 09:26:15 -0500 Subject: [PATCH 26/39] add basic auth to elasticsearch requests --- app/controllers/application_controller.rb | 3 ++- app/services/search_service.rb | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 15ff0d8..e97e006 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -3,7 +3,8 @@ class ApplicationController < ActionController::API def post_search(json, error_method=method(:display_error)) - res = RestClient.post("#{ES_URI}/_search", json.to_json, { "content-type" => "json" }) + auth_hash = { "Authorization" => "Basic #{Base64::encode64("#{ES_USER}:#{ES_PASSWORD}")}" } + res = RestClient.post("#{ES_URI}/_search", json.to_json, auth_hash.merge({ "content-type" => "json" })) raise return JSON.parse(res.body) rescue => e diff --git a/app/services/search_service.rb b/app/services/search_service.rb index dbd8877..319b97b 100644 --- a/app/services/search_service.rb +++ b/app/services/search_service.rb @@ -11,7 +11,8 @@ def initialize(url, params={}, user_req) end def post(url_ending, json) - res = RestClient.post("#{@url}/#{url_ending}", json.to_json, { "content-type" => "json" } ) + auth_hash = { "Authorization" => "Basic #{Base64::encode64("#{Rails.application.credentials.elasticsearch[:user]}:#{Rails.application.credentials.elasticsearch[:password]}")}" } + res = RestClient.post("#{@url}/#{url_ending}", json.to_json, auth_hash.merge({ "content-type" => "json" } )) JSON.parse(res.body) rescue => e e From b8ab6b0ab42219f7daca11226f37caa610cdfef3 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Fri, 28 Oct 2022 11:58:55 -0500 Subject: [PATCH 27/39] raise number of results per facet --- config/config.example.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.example.yml b/config/config.example.yml index a06b13c..c0ce5d4 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -18,7 +18,7 @@ default: &default # highlight fragment number hl_num: 3 # number of results returned per "page" - num: 20 + num: 5000 # default sorting field and direction (field|asc) sort_fl: "identifier|asc" # starting document in results list (0 == 1st result) From 2c68524853eaac7cc2d21e7ff64c6f217a45fd72 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 31 Oct 2022 11:40:32 -0500 Subject: [PATCH 28/39] use facet_limit instead of facet_num to match Orchid --- app/services/search_item_req.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index 56e6453..749af2f 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -79,8 +79,7 @@ def facets end # FACET_SETTINGS["start"] - size = SETTINGS["num"] - size = @params["facet_num"].blank? ? SETTINGS["num"] : @params["facet_num"] + size = @params["facet_limit"].blank? ? SETTINGS["num"] : @params["facet_limit"] aggs = {} Array.wrap(@params["facet"]).each do |f| From 303598f02ef88164194c09972eb80e2389b3609c Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 31 Oct 2022 11:40:50 -0500 Subject: [PATCH 29/39] revert, will set in Orchid --- config/config.example.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.example.yml b/config/config.example.yml index c0ce5d4..a06b13c 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -18,7 +18,7 @@ default: &default # highlight fragment number hl_num: 3 # number of results returned per "page" - num: 5000 + num: 20 # default sorting field and direction (field|asc) sort_fl: "identifier|asc" # starting document in results list (0 == 1st result) From 2eed5b27505e5623bb7f6bb4abee4f7a535cc096 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 1 Nov 2022 15:59:45 -0500 Subject: [PATCH 30/39] change facet_num to facet_limit --- test/services/search_item_req_test.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/services/search_item_req_test.rb b/test/services/search_item_req_test.rb index c4d8197..a5b3eab 100644 --- a/test/services/search_item_req_test.rb +++ b/test/services/search_item_req_test.rb @@ -39,7 +39,7 @@ def test_facets # normal with pagination overrides, multiple facets facets = SearchItemReq.new({ - "facet_num" => 10, + "facet_limit" => 10, "facet_sort" => "term|asc", "facet" => [ "title", "subcategory" ] }).facets @@ -50,7 +50,7 @@ def test_facets # should be blank if there are no facets provided facets = SearchItemReq.new({ - "facet_num" => 1, + "facet_limit" => 1, "facet_sort" => "nonterm|asc", "facet" => [] }).facets From 4d10eddcb334dc655312aa19731c122f91cfd2c9 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 9 Nov 2022 09:17:35 -0600 Subject: [PATCH 31/39] update ruby version --- .ruby-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ruby-version b/.ruby-version index 4e34c4d..7bde84d 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -ruby-2.7.6 +ruby-3.1.2 From ff4c1c1bc5e7472e2b57b4d218697526c015951b Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 9 Nov 2022 09:22:08 -0600 Subject: [PATCH 32/39] update Rails to 6.1.7 and associated gems --- Gemfile | 3 +- Gemfile.lock | 177 ++++++++++++++++++++++++++------------------------- 2 files changed, 93 insertions(+), 87 deletions(-) diff --git a/Gemfile b/Gemfile index 4170559..9c639fc 100644 --- a/Gemfile +++ b/Gemfile @@ -7,7 +7,7 @@ end # Bundle edge Rails instead: gem 'rails', github: 'rails/rails' -gem 'rails', '~> 6.0.2' +gem 'rails', '~> 6.1.7' # Use sqlite3 as the database for Active Record gem 'sqlite3' # Use Puma as the app server @@ -26,6 +26,7 @@ gem 'puma', '>= 5.6' # gem 'rack-cors' gem 'bootsnap', require: false +gem 'net-smtp' group :development, :test do # Call 'byebug' anywhere in the code to stop execution and get a debugger console diff --git a/Gemfile.lock b/Gemfile.lock index 3f1cd9c..5bc4909 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,62 +1,66 @@ GEM remote: https://rubygems.org/ specs: - actioncable (6.0.5) - actionpack (= 6.0.5) + actioncable (6.1.7) + actionpack (= 6.1.7) + activesupport (= 6.1.7) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailbox (6.0.5) - actionpack (= 6.0.5) - activejob (= 6.0.5) - activerecord (= 6.0.5) - activestorage (= 6.0.5) - activesupport (= 6.0.5) + actionmailbox (6.1.7) + actionpack (= 6.1.7) + activejob (= 6.1.7) + activerecord (= 6.1.7) + activestorage (= 6.1.7) + activesupport (= 6.1.7) mail (>= 2.7.1) - actionmailer (6.0.5) - actionpack (= 6.0.5) - actionview (= 6.0.5) - activejob (= 6.0.5) + actionmailer (6.1.7) + actionpack (= 6.1.7) + actionview (= 6.1.7) + activejob (= 6.1.7) + activesupport (= 6.1.7) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (6.0.5) - actionview (= 6.0.5) - activesupport (= 6.0.5) - rack (~> 2.0, >= 2.0.8) + actionpack (6.1.7) + actionview (= 6.1.7) + activesupport (= 6.1.7) + rack (~> 2.0, >= 2.0.9) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.2.0) - actiontext (6.0.5) - actionpack (= 6.0.5) - activerecord (= 6.0.5) - activestorage (= 6.0.5) - activesupport (= 6.0.5) + actiontext (6.1.7) + actionpack (= 6.1.7) + activerecord (= 6.1.7) + activestorage (= 6.1.7) + activesupport (= 6.1.7) nokogiri (>= 1.8.5) - actionview (6.0.5) - activesupport (= 6.0.5) + actionview (6.1.7) + activesupport (= 6.1.7) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.1, >= 1.2.0) - activejob (6.0.5) - activesupport (= 6.0.5) + activejob (6.1.7) + activesupport (= 6.1.7) globalid (>= 0.3.6) - activemodel (6.0.5) - activesupport (= 6.0.5) - activerecord (6.0.5) - activemodel (= 6.0.5) - activesupport (= 6.0.5) - activestorage (6.0.5) - actionpack (= 6.0.5) - activejob (= 6.0.5) - activerecord (= 6.0.5) + activemodel (6.1.7) + activesupport (= 6.1.7) + activerecord (6.1.7) + activemodel (= 6.1.7) + activesupport (= 6.1.7) + activestorage (6.1.7) + actionpack (= 6.1.7) + activejob (= 6.1.7) + activerecord (= 6.1.7) + activesupport (= 6.1.7) marcel (~> 1.0) - activesupport (6.0.5) + mini_mime (>= 1.1.0) + activesupport (6.1.7) concurrent-ruby (~> 1.0, >= 1.0.2) - i18n (>= 0.7, < 2) - minitest (~> 5.1) - tzinfo (~> 1.1) - zeitwerk (~> 2.2, >= 2.2.2) - bootsnap (1.11.1) + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0) + zeitwerk (~> 2.3) + bootsnap (1.13.0) msgpack (~> 1.2) builder (3.2.4) byebug (11.1.3) @@ -64,20 +68,19 @@ GEM crass (1.0.6) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - erubi (1.10.0) + erubi (1.11.0) ffi (1.15.5) globalid (1.0.0) activesupport (>= 5.0) http-accept (1.7.0) http-cookie (1.0.5) domain_name (~> 0.5) - i18n (1.10.0) + i18n (1.12.0) concurrent-ruby (~> 1.0) - listen (3.1.5) + listen (3.0.8) rb-fsevent (~> 0.9, >= 0.9.4) rb-inotify (~> 0.9, >= 0.9.7) - ruby_dep (~> 1.2) - loofah (2.18.0) + loofah (2.19.0) crass (~> 1.0.2) nokogiri (>= 1.5.9) mail (2.7.1) @@ -88,48 +91,50 @@ GEM mime-types-data (~> 3.2015) mime-types-data (3.2022.0105) mini_mime (1.1.2) - mini_portile2 (2.8.0) - minitest (5.15.0) - msgpack (1.5.1) + minitest (5.16.3) + msgpack (1.6.0) + net-protocol (0.1.3) + timeout + net-smtp (0.3.3) + net-protocol netrc (0.11.0) nio4r (2.5.8) - nokogiri (1.13.6) - mini_portile2 (~> 2.8.0) + nokogiri (1.13.9-x86_64-darwin) racc (~> 1.4) - puma (5.6.4) + puma (6.0.0) nio4r (~> 2.0) racc (1.6.0) - rack (2.2.3) - rack-test (1.1.0) - rack (>= 1.0, < 3) - rails (6.0.5) - actioncable (= 6.0.5) - actionmailbox (= 6.0.5) - actionmailer (= 6.0.5) - actionpack (= 6.0.5) - actiontext (= 6.0.5) - actionview (= 6.0.5) - activejob (= 6.0.5) - activemodel (= 6.0.5) - activerecord (= 6.0.5) - activestorage (= 6.0.5) - activesupport (= 6.0.5) - bundler (>= 1.3.0) - railties (= 6.0.5) + rack (2.2.4) + rack-test (2.0.2) + rack (>= 1.3) + rails (6.1.7) + actioncable (= 6.1.7) + actionmailbox (= 6.1.7) + actionmailer (= 6.1.7) + actionpack (= 6.1.7) + actiontext (= 6.1.7) + actionview (= 6.1.7) + activejob (= 6.1.7) + activemodel (= 6.1.7) + activerecord (= 6.1.7) + activestorage (= 6.1.7) + activesupport (= 6.1.7) + bundler (>= 1.15.0) + railties (= 6.1.7) sprockets-rails (>= 2.0.0) rails-dom-testing (2.0.3) activesupport (>= 4.2.0) nokogiri (>= 1.6) - rails-html-sanitizer (1.4.2) + rails-html-sanitizer (1.4.3) loofah (~> 2.3) - railties (6.0.5) - actionpack (= 6.0.5) - activesupport (= 6.0.5) + railties (6.1.7) + actionpack (= 6.1.7) + activesupport (= 6.1.7) method_source - rake (>= 0.8.7) - thor (>= 0.20.3, < 2.0) + rake (>= 12.2) + thor (~> 1.0) rake (13.0.6) - rb-fsevent (0.11.1) + rb-fsevent (0.11.2) rb-inotify (0.10.1) ffi (~> 1.0) rest-client (2.1.0) @@ -137,40 +142,40 @@ GEM http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) - ruby_dep (1.5.0) spring (2.1.1) spring-watcher-listen (2.0.1) listen (>= 2.7, < 4.0) spring (>= 1.2, < 3.0) - sprockets (4.0.3) + sprockets (4.1.1) concurrent-ruby (~> 1.0) rack (> 1, < 3) sprockets-rails (3.4.2) actionpack (>= 5.2) activesupport (>= 5.2) sprockets (>= 3.0.0) - sqlite3 (1.4.2) + sqlite3 (1.5.3-x86_64-darwin) thor (1.2.1) - thread_safe (0.3.6) - tzinfo (1.2.9) - thread_safe (~> 0.1) + timeout (0.3.0) + tzinfo (2.0.5) + concurrent-ruby (~> 1.0) unf (0.1.4) unf_ext - unf_ext (0.0.8.1) + unf_ext (0.0.8.2) websocket-driver (0.7.5) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) - zeitwerk (2.5.4) + zeitwerk (2.6.6) PLATFORMS - ruby + x86_64-darwin-21 DEPENDENCIES bootsnap byebug listen (>= 3.0.5, < 3.2) + net-smtp puma (>= 5.6) - rails (~> 6.0.2) + rails (~> 6.1.7) rest-client (>= 2.1.0.rc1, < 2.2) spring spring-watcher-listen (~> 2.0.0) @@ -178,4 +183,4 @@ DEPENDENCIES tzinfo-data BUNDLED WITH - 2.2.26 + 2.3.7 From db6b0e1e6c9b80eaa3c8fb623ff294f0af437583 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 9 Nov 2022 09:22:46 -0600 Subject: [PATCH 33/39] update bin files for rails 6.1.7 --- bin/rails | 5 +++-- bin/rake | 5 +++-- bin/setup | 6 +++--- bin/spring | 14 ++++++++++++++ 4 files changed, 23 insertions(+), 7 deletions(-) create mode 100755 bin/spring diff --git a/bin/rails b/bin/rails index 0739660..21d3e02 100755 --- a/bin/rails +++ b/bin/rails @@ -1,4 +1,5 @@ #!/usr/bin/env ruby +load File.expand_path("spring", __dir__) APP_PATH = File.expand_path('../config/application', __dir__) -require_relative '../config/boot' -require 'rails/commands' +require_relative "../config/boot" +require "rails/commands" diff --git a/bin/rake b/bin/rake index 1724048..7327f47 100755 --- a/bin/rake +++ b/bin/rake @@ -1,4 +1,5 @@ #!/usr/bin/env ruby -require_relative '../config/boot' -require 'rake' +load File.expand_path("spring", __dir__) +require_relative "../config/boot" +require "rake" Rake.application.run diff --git a/bin/setup b/bin/setup index 0e39e8c..5792302 100755 --- a/bin/setup +++ b/bin/setup @@ -1,5 +1,5 @@ #!/usr/bin/env ruby -require 'fileutils' +require "fileutils" # path to your application root. APP_ROOT = File.expand_path('..', __dir__) @@ -9,8 +9,8 @@ def system!(*args) end FileUtils.chdir APP_ROOT do - # This script is a way to setup or update your development environment automatically. - # This script is idempotent, so that you can run it at anytime and get an expectable outcome. + # This script is a way to set up or update your development environment automatically. + # This script is idempotent, so that you can run it at any time and get an expectable outcome. # Add necessary setup steps to this file. puts '== Installing dependencies ==' diff --git a/bin/spring b/bin/spring new file mode 100755 index 0000000..b4147e8 --- /dev/null +++ b/bin/spring @@ -0,0 +1,14 @@ +#!/usr/bin/env ruby +if !defined?(Spring) && [nil, "development", "test"].include?(ENV["RAILS_ENV"]) + gem "bundler" + require "bundler" + + # Load Spring without loading other gems in the Gemfile, for speed. + Bundler.locked_gems&.specs&.find { |spec| spec.name == "spring" }&.tap do |spring| + Gem.use_paths Gem.dir, Bundler.bundle_path.to_s, *Gem.path + gem "spring", spring.version + require "spring/binstub" + rescue Gem::LoadError + # Ignore when Spring is not installed. + end +end From 2db5e573de6b19b89747f4c6a8b5a66a9524876d Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 9 Nov 2022 09:34:06 -0600 Subject: [PATCH 34/39] update config files for rails 6.1.7 --- config.ru | 3 +- config/application.rb | 13 ++-- config/boot.rb | 4 +- config/environment.rb | 2 +- config/environments/development.rb | 20 +++++- config/environments/production.rb | 18 +++-- config/environments/test.rb | 14 +++- config/initializers/backtrace_silencers.rb | 7 +- .../initializers/filter_parameter_logging.rb | 4 +- .../new_framework_defaults_6_1.rb | 67 +++++++++++++++++++ config/puma.rb | 7 +- 11 files changed, 136 insertions(+), 23 deletions(-) create mode 100644 config/initializers/new_framework_defaults_6_1.rb diff --git a/config.ru b/config.ru index f7ba0b5..4a3c09a 100644 --- a/config.ru +++ b/config.ru @@ -1,5 +1,6 @@ # This file is used by Rack-based servers to start the application. -require_relative 'config/environment' +require_relative "config/environment" run Rails.application +Rails.application.load_server diff --git a/config/application.rb b/config/application.rb index b9c2970..03f5085 100644 --- a/config/application.rb +++ b/config/application.rb @@ -1,4 +1,4 @@ -require_relative 'boot' +require_relative "boot" require "rails" # Pick the frameworks you want: @@ -24,10 +24,13 @@ class Application < Rails::Application # Initialize configuration defaults for originally generated Rails version. config.load_defaults 5.0 - # Settings in config/environments/* take precedence over those specified here. - # Application configuration can go into files in config/initializers - # -- all .rb files in that directory are automatically loaded after loading - # the framework and any gems in your application. + # Configuration for the application, engines, and railties goes here. + # + # These settings can be overridden in specific environments using the files + # in config/environments, which are processed later. + # + # config.time_zone = "Central Time (US & Canada)" + # config.eager_load_paths << Rails.root.join("extras") # Only loads a smaller set of middleware suitable for API only apps. # Middleware like session, flash, cookies can be added back manually. diff --git a/config/boot.rb b/config/boot.rb index b9e460c..3cda23b 100644 --- a/config/boot.rb +++ b/config/boot.rb @@ -1,4 +1,4 @@ ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__) -require 'bundler/setup' # Set up gems listed in the Gemfile. -require 'bootsnap/setup' # Speed up boot time by caching expensive operations. +require "bundler/setup" # Set up gems listed in the Gemfile. +require "bootsnap/setup" # Speed up boot time by caching expensive operations. diff --git a/config/environment.rb b/config/environment.rb index 426333b..cac5315 100644 --- a/config/environment.rb +++ b/config/environment.rb @@ -1,5 +1,5 @@ # Load the Rails application. -require_relative 'application' +require_relative "application" # Initialize the Rails application. Rails.application.initialize! diff --git a/config/environments/development.rb b/config/environments/development.rb index 14a0a40..78b25c4 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -1,8 +1,10 @@ +require "active_support/core_ext/integer/time" + Rails.application.configure do # Settings specified here will take precedence over those in config/application.rb. - # In the development environment your application's code is reloaded on - # every request. This slows down response time but is perfect for development + # In the development environment your application's code is reloaded any time + # it changes. This slows down response time but is perfect for development # since you don't have to restart the web server when you make code changes. config.cache_classes = false @@ -36,6 +38,12 @@ # Print deprecation notices to the Rails logger. config.active_support.deprecation = :log + # Raise exceptions for disallowed deprecations. + config.active_support.disallowed_deprecation = :raise + + # Tell Active Support which deprecation messages to disallow. + config.active_support.disallowed_deprecation_warnings = [] + # Raise an error on page load if there are pending migrations. config.active_record.migration_error = :page_load @@ -44,11 +52,15 @@ # Raises error for missing translations. - # config.action_view.raise_on_missing_translations = true + # config.i18n.raise_on_missing_translations = true + + # Annotate rendered view with file names. + # config.action_view.annotate_rendered_view_with_filenames = true # Use an evented file watcher to asynchronously detect changes in source code, # routes, locales, etc. This feature depends on the listen gem. config.file_watcher = ActiveSupport::EventedFileUpdateChecker + # LOCAL # Custom dev env logger to empty log more frequently config.logger = ActiveSupport::TaggedLogging.new( @@ -62,4 +74,6 @@ config.hosts << "cdrhdev1.unl.edu" config.hosts << "whitman-dev.unl.edu" + # Uncomment if you wish to allow Action Cable access from any origin. + # config.action_cable.disable_request_forgery_protection = true end diff --git a/config/environments/production.rb b/config/environments/production.rb index 1af1588..50bdaf0 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -1,3 +1,5 @@ +require "active_support/core_ext/integer/time" + Rails.application.configure do # Settings specified here will take precedence over those in config/application.rb. @@ -22,7 +24,7 @@ config.public_file_server.enabled = ENV['RAILS_SERVE_STATIC_FILES'].present? # Enable serving of images, stylesheets, and JavaScripts from an asset server. - # config.action_controller.asset_host = 'http://assets.example.com' + # config.asset_host = 'http://assets.example.com' # Specifies the header that your server uses for sending files. # config.action_dispatch.x_sendfile_header = 'X-Sendfile' # for Apache @@ -39,9 +41,9 @@ # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. # config.force_ssl = true - # Use the lowest log level to ensure availability of diagnostic information - # when problems arise. - config.log_level = :debug + # Include generic and useful information about system operation, but avoid logging too much + # information to avoid inadvertent exposure of personally identifiable information (PII). + config.log_level = :info # Prepend all log lines with the following tags. config.log_tags = [ :request_id ] @@ -66,11 +68,17 @@ # Send deprecation notices to registered listeners. config.active_support.deprecation = :notify + # Log disallowed deprecations. + config.active_support.disallowed_deprecation = :log + + # Tell Active Support which deprecation messages to disallow. + config.active_support.disallowed_deprecation_warnings = [] + # Use default logging formatter so that PID and timestamp are not suppressed. config.log_formatter = ::Logger::Formatter.new # Use a different logger for distributed setups. - # require 'syslog/logger' + # require "syslog/logger" # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new 'app-name') if ENV["RAILS_LOG_TO_STDOUT"].present? diff --git a/config/environments/test.rb b/config/environments/test.rb index 1d62e91..93ed4f1 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -1,3 +1,5 @@ +require "active_support/core_ext/integer/time" + # The test environment is used exclusively to run your application's # test suite. You never need to work with it otherwise. Remember that # your test database is "scratch space" for the test suite and is wiped @@ -7,6 +9,7 @@ # Settings specified here will take precedence over those in config/application.rb. config.cache_classes = false + config.action_view.cache_template_loading = true # Do not eager load code on boot. This avoids loading your whole application # just for the purpose of running a single test. If you are using a tool that @@ -43,6 +46,15 @@ # Print deprecation notices to the stderr. config.active_support.deprecation = :stderr + # Raise exceptions for disallowed deprecations. + config.active_support.disallowed_deprecation = :raise + + # Tell Active Support which deprecation messages to disallow. + config.active_support.disallowed_deprecation_warnings = [] + # Raises error for missing translations. - # config.action_view.raise_on_missing_translations = true + # config.i18n.raise_on_missing_translations = true + + # Annotate rendered view with file names. + # config.action_view.annotate_rendered_view_with_filenames = true end diff --git a/config/initializers/backtrace_silencers.rb b/config/initializers/backtrace_silencers.rb index 59385cd..33699c3 100644 --- a/config/initializers/backtrace_silencers.rb +++ b/config/initializers/backtrace_silencers.rb @@ -1,7 +1,8 @@ # Be sure to restart your server when you modify this file. # You can add backtrace silencers for libraries that you're using but don't wish to see in your backtraces. -# Rails.backtrace_cleaner.add_silencer { |line| line =~ /my_noisy_library/ } +# Rails.backtrace_cleaner.add_silencer { |line| /my_noisy_library/.match?(line) } -# You can also remove all the silencers if you're trying to debug a problem that might stem from framework code. -# Rails.backtrace_cleaner.remove_silencers! +# You can also remove all the silencers if you're trying to debug a problem that might stem from framework code +# by setting BACKTRACE=1 before calling your invocation, like "BACKTRACE=1 ./bin/rails runner 'MyClass.perform'". +Rails.backtrace_cleaner.remove_silencers! if ENV["BACKTRACE"] diff --git a/config/initializers/filter_parameter_logging.rb b/config/initializers/filter_parameter_logging.rb index 4a994e1..4b34a03 100644 --- a/config/initializers/filter_parameter_logging.rb +++ b/config/initializers/filter_parameter_logging.rb @@ -1,4 +1,6 @@ # Be sure to restart your server when you modify this file. # Configure sensitive parameters which will be filtered from the log file. -Rails.application.config.filter_parameters += [:password] +Rails.application.config.filter_parameters += [ + :passw, :secret, :token, :_key, :crypt, :salt, :certificate, :otp, :ssn +] diff --git a/config/initializers/new_framework_defaults_6_1.rb b/config/initializers/new_framework_defaults_6_1.rb new file mode 100644 index 0000000..9526b83 --- /dev/null +++ b/config/initializers/new_framework_defaults_6_1.rb @@ -0,0 +1,67 @@ +# Be sure to restart your server when you modify this file. +# +# This file contains migration options to ease your Rails 6.1 upgrade. +# +# Once upgraded flip defaults one by one to migrate to the new default. +# +# Read the Guide for Upgrading Ruby on Rails for more info on each option. + +# Support for inversing belongs_to -> has_many Active Record associations. +# Rails.application.config.active_record.has_many_inversing = true + +# Track Active Storage variants in the database. +# Rails.application.config.active_storage.track_variants = true + +# Apply random variation to the delay when retrying failed jobs. +# Rails.application.config.active_job.retry_jitter = 0.15 + +# Stop executing `after_enqueue`/`after_perform` callbacks if +# `before_enqueue`/`before_perform` respectively halts with `throw :abort`. +# Rails.application.config.active_job.skip_after_callbacks_if_terminated = true + +# Specify cookies SameSite protection level: either :none, :lax, or :strict. +# +# This change is not backwards compatible with earlier Rails versions. +# It's best enabled when your entire app is migrated and stable on 6.1. +# Rails.application.config.action_dispatch.cookies_same_site_protection = :lax + +# Generate CSRF tokens that are encoded in URL-safe Base64. +# +# This change is not backwards compatible with earlier Rails versions. +# It's best enabled when your entire app is migrated and stable on 6.1. +# Rails.application.config.action_controller.urlsafe_csrf_tokens = true + +# Specify whether `ActiveSupport::TimeZone.utc_to_local` returns a time with an +# UTC offset or a UTC time. +# ActiveSupport.utc_to_local_returns_utc_offset_times = true + +# Change the default HTTP status code to `308` when redirecting non-GET/HEAD +# requests to HTTPS in `ActionDispatch::SSL` middleware. +# Rails.application.config.action_dispatch.ssl_default_redirect_status = 308 + +# Use new connection handling API. For most applications this won't have any +# effect. For applications using multiple databases, this new API provides +# support for granular connection swapping. +# Rails.application.config.active_record.legacy_connection_handling = false + +# Make `form_with` generate non-remote forms by default. +# Rails.application.config.action_view.form_with_generates_remote_forms = false + +# Set the default queue name for the analysis job to the queue adapter default. +# Rails.application.config.active_storage.queues.analysis = nil + +# Set the default queue name for the purge job to the queue adapter default. +# Rails.application.config.active_storage.queues.purge = nil + +# Set the default queue name for the incineration job to the queue adapter default. +# Rails.application.config.action_mailbox.queues.incineration = nil + +# Set the default queue name for the routing job to the queue adapter default. +# Rails.application.config.action_mailbox.queues.routing = nil + +# Set the default queue name for the mail deliver job to the queue adapter default. +# Rails.application.config.action_mailer.deliver_later_queue_name = nil + +# Generate a `Link` header that gives a hint to modern browsers about +# preloading assets when using `javascript_include_tag` and `stylesheet_link_tag`. +# Rails.application.config.action_view.preload_links_header = true diff --git a/config/puma.rb b/config/puma.rb index 5ed4437..d9b3e83 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -8,9 +8,14 @@ min_threads_count = ENV.fetch("RAILS_MIN_THREADS") { max_threads_count } threads min_threads_count, max_threads_count +# Specifies the `worker_timeout` threshold that Puma will use to wait before +# terminating a worker in development environments. +# +worker_timeout 3600 if ENV.fetch("RAILS_ENV", "development") == "development" + # Specifies the `port` that Puma will listen on to receive requests; default is 3000. # -port ENV.fetch("PORT") { 3000 } +port ENV.fetch("PORT") { 3000 } # Specifies the `environment` that Puma will run in. # From 86a1b14763c581010b8e109d201939c423a6f18c Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 9 Nov 2022 12:52:58 -0600 Subject: [PATCH 35/39] update documentation for new release --- CHANGELOG.md | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad8bf5d..03ef8d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,15 +27,16 @@ Markdown Spec](https://github.github.com/gfm/). ### Contributors --> - -## [Unreleased] - updates in preparation for Habeas release +## [2.0.0] - new nested bucket aggregation/query functionality for Habeas release [Unreleased]: https://github.com/CDRH/api/compare/v1.0.4...dev ### Added - "api_version" added to all response "res" objects - -### Changed -- upgraded to Rails 6 +- support for elasticsearch 8.5 +- user/password basic authentication with ES 8.5, when querying the index or posting from Datura +- better support for nested fields +- support for nested bucket aggregations, matching a nested value on another nested value. `person.name[person.role#judge]` will return all names of persons where role="judge". +- "api_version" added to all response "res" objects - Added support for aggregating buckets by normalized keyword and returning the "top_hits" first document result for a non-normalized display - Changes response format of `facets` key @@ -55,6 +56,23 @@ Markdown Spec](https://github.github.com/gfm/). ``` Not only is the response format itself different, but there may be fewer facets returned since normalized values which match are combined +### Changed +- upgraded to Rails 6.1.7 and Ruby 3 +- changes reflect new api schemas in Datura, which make heavy use of nested fields + +### Migration +- in Datura repos config `private.yml` api to `"api_version": "2.0"` to take advantage of new bucket aggregation functionality (or `"api_version": "1.0"` for legacy repos that have not been updated for the new schema). Please note that a running API index can only use one ES index at a time, and each ES index is restricted to one version of the schema. +- Use Elasticsearch 8.5 or later +- If you are using ES with security enabled, you must configure credentials with Rails in the API repo. See https://guides.rubyonrails.org/v6.1/security.html. Configure the VSCode editor. Run `EDITOR="code --wait" rails credentials:edit` and add +``` +elasticsearch: + user: username + password: ***** +``` +to the secrets file and then close the window to save. Do not commit `config/master.key` (it should be in `gitignore`) +- Orchid apps that connect to the API should use `facet_limit` instead of `facet_num` in options. +- Add nested facets as described above, if desired + ## [v1.0.4](https://github.com/CDRH/api/compare/v1.0....v1.0.4) - Updates & license From e73244b4fa11941fd49b88bbcbe697ee22e7e554 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 9 Nov 2022 12:53:09 -0600 Subject: [PATCH 36/39] update db files for Rails 6.1.7 --- ..._to_active_storage_blobs.active_storage.rb | 22 +++++++++++++++ ..._storage_variant_records.active_storage.rb | 27 +++++++++++++++++++ db/schema.rb | 15 +++++++++++ 3 files changed, 64 insertions(+) create mode 100644 db/migrate/20221109145721_add_service_name_to_active_storage_blobs.active_storage.rb create mode 100644 db/migrate/20221109145722_create_active_storage_variant_records.active_storage.rb create mode 100644 db/schema.rb diff --git a/db/migrate/20221109145721_add_service_name_to_active_storage_blobs.active_storage.rb b/db/migrate/20221109145721_add_service_name_to_active_storage_blobs.active_storage.rb new file mode 100644 index 0000000..a15c6ce --- /dev/null +++ b/db/migrate/20221109145721_add_service_name_to_active_storage_blobs.active_storage.rb @@ -0,0 +1,22 @@ +# This migration comes from active_storage (originally 20190112182829) +class AddServiceNameToActiveStorageBlobs < ActiveRecord::Migration[6.0] + def up + return unless table_exists?(:active_storage_blobs) + + unless column_exists?(:active_storage_blobs, :service_name) + add_column :active_storage_blobs, :service_name, :string + + if configured_service = ActiveStorage::Blob.service.name + ActiveStorage::Blob.unscoped.update_all(service_name: configured_service) + end + + change_column :active_storage_blobs, :service_name, :string, null: false + end + end + + def down + return unless table_exists?(:active_storage_blobs) + + remove_column :active_storage_blobs, :service_name + end +end diff --git a/db/migrate/20221109145722_create_active_storage_variant_records.active_storage.rb b/db/migrate/20221109145722_create_active_storage_variant_records.active_storage.rb new file mode 100644 index 0000000..94ac83a --- /dev/null +++ b/db/migrate/20221109145722_create_active_storage_variant_records.active_storage.rb @@ -0,0 +1,27 @@ +# This migration comes from active_storage (originally 20191206030411) +class CreateActiveStorageVariantRecords < ActiveRecord::Migration[6.0] + def change + return unless table_exists?(:active_storage_blobs) + + # Use Active Record's configured type for primary key + create_table :active_storage_variant_records, id: primary_key_type, if_not_exists: true do |t| + t.belongs_to :blob, null: false, index: false, type: blobs_primary_key_type + t.string :variation_digest, null: false + + t.index %i[ blob_id variation_digest ], name: "index_active_storage_variant_records_uniqueness", unique: true + t.foreign_key :active_storage_blobs, column: :blob_id + end + end + + private + def primary_key_type + config = Rails.configuration.generators + config.options[config.orm][:primary_key_type] || :primary_key + end + + def blobs_primary_key_type + pkey_name = connection.primary_key(:active_storage_blobs) + pkey_column = connection.columns(:active_storage_blobs).find { |c| c.name == pkey_name } + pkey_column.bigint? ? :bigint : pkey_column.type + end +end diff --git a/db/schema.rb b/db/schema.rb new file mode 100644 index 0000000..adc3722 --- /dev/null +++ b/db/schema.rb @@ -0,0 +1,15 @@ +# This file is auto-generated from the current state of the database. Instead +# of editing this file, please use the migrations feature of Active Record to +# incrementally modify your database, and then regenerate this schema definition. +# +# This file is the source Rails uses to define your schema when running `bin/rails +# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to +# be faster and is potentially less error prone than running all of your +# migrations from scratch. Old migrations may fail to apply correctly if those +# migrations use external dependencies or application code. +# +# It's strongly recommended that you check this file into your version control system. + +ActiveRecord::Schema.define(version: 2022_11_09_145722) do + +end From 456a1a9a8945b9c6ce3277a2fdb3d453c1941e1c Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 10 Nov 2022 09:56:24 -0600 Subject: [PATCH 37/39] update nested facets documentation --- docs/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/README.md b/docs/README.md index 8aa9ad6..cd2fc90 100644 --- a/docs/README.md +++ b/docs/README.md @@ -50,6 +50,12 @@ __Nested fields__ facet[]=creator.name facet[]=creator.name&facet[]=creator.role ``` +you can also match on another nested field with the new API schema +`facet[]=nested_field.keyword_field1[nested_field.keyword_field2#value]` +``` +facet[]=person.name[person.role#judge] +``` +the above will select all names of persons, where the role of that person is "judge". __Date ranges__ (currently supports days or years) From 48865e0169b9136cd75c86b4a28bf5091bb2852d Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 10 Nov 2022 13:55:56 -0600 Subject: [PATCH 38/39] add links to more detailed documentation --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03ef8d2..c160e0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,8 +61,8 @@ Markdown Spec](https://github.github.com/gfm/). - changes reflect new api schemas in Datura, which make heavy use of nested fields ### Migration -- in Datura repos config `private.yml` api to `"api_version": "2.0"` to take advantage of new bucket aggregation functionality (or `"api_version": "1.0"` for legacy repos that have not been updated for the new schema). Please note that a running API index can only use one ES index at a time, and each ES index is restricted to one version of the schema. -- Use Elasticsearch 8.5 or later +- in Datura repos config `private.yml` api to `"api_version": "2.0"` to take advantage of new bucket aggregation functionality (or `"api_version": "1.0"` for legacy repos that have not been updated for the new schema). Please note that a running API index can only use one ES index at a time, and each ES index is restricted to one version of the schema. See new schema (2.0) documentation [here](https://github.com/CDRH/datura/docs/schema_v2.md) +- Use Elasticsearch 8.5 or later. See [dev docs instructions](https://github.com/CDRH/cdrh_dev_docs/blob/update_elasticsearch_documentation/publishing/2_basic_requirements.md#downloading-elasticsearch). - If you are using ES with security enabled, you must configure credentials with Rails in the API repo. See https://guides.rubyonrails.org/v6.1/security.html. Configure the VSCode editor. Run `EDITOR="code --wait" rails credentials:edit` and add ``` elasticsearch: @@ -71,7 +71,7 @@ elasticsearch: ``` to the secrets file and then close the window to save. Do not commit `config/master.key` (it should be in `gitignore`) - Orchid apps that connect to the API should use `facet_limit` instead of `facet_num` in options. -- Add nested facets as described above, if desired +- Add nested facets as described above, if desired. ## [v1.0.4](https://github.com/CDRH/api/compare/v1.0....v1.0.4) - Updates & license From 1305266efa6191aff6b002610cfe0633d8718dc5 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 10 Nov 2022 14:24:33 -0600 Subject: [PATCH 39/39] clarify --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c160e0a..19a8639 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,7 +61,7 @@ Markdown Spec](https://github.github.com/gfm/). - changes reflect new api schemas in Datura, which make heavy use of nested fields ### Migration -- in Datura repos config `private.yml` api to `"api_version": "2.0"` to take advantage of new bucket aggregation functionality (or `"api_version": "1.0"` for legacy repos that have not been updated for the new schema). Please note that a running API index can only use one ES index at a time, and each ES index is restricted to one version of the schema. See new schema (2.0) documentation [here](https://github.com/CDRH/datura/docs/schema_v2.md) +- in the config files of your Datura repos, (`private.yml` or `public.yml`, set the api to `"api_version": "2.0"` to take advantage of new bucket aggregation functionality (or `"api_version": "1.0"` for legacy repos that have not been updated for the new schema). Please note that a running API index can only use one ES index at a time, and each ES index is restricted to one version of the schema. See new schema (2.0) documentation [here](https://github.com/CDRH/datura/docs/schema_v2.md). - Use Elasticsearch 8.5 or later. See [dev docs instructions](https://github.com/CDRH/cdrh_dev_docs/blob/update_elasticsearch_documentation/publishing/2_basic_requirements.md#downloading-elasticsearch). - If you are using ES with security enabled, you must configure credentials with Rails in the API repo. See https://guides.rubyonrails.org/v6.1/security.html. Configure the VSCode editor. Run `EDITOR="code --wait" rails credentials:edit` and add ```