diff --git a/.gitignore b/.gitignore index 76fb9c4..a3bd71e 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,5 @@ bower.json .byebug_history .DS_Store + +/config/master.key diff --git a/.ruby-gemset b/.ruby-gemset index eedd89b..fcf5595 100644 --- a/.ruby-gemset +++ b/.ruby-gemset @@ -1 +1 @@ -api +api-v2 diff --git a/.ruby-version b/.ruby-version index 324db8d..7bde84d 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -ruby-2.6.8 +ruby-3.1.2 diff --git a/CHANGELOG.md b/CHANGELOG.md index ad8bf5d..19a8639 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,15 +27,16 @@ Markdown Spec](https://github.github.com/gfm/). ### Contributors --> - -## [Unreleased] - updates in preparation for Habeas release +## [2.0.0] - new nested bucket aggregation/query functionality for Habeas release [Unreleased]: https://github.com/CDRH/api/compare/v1.0.4...dev ### Added - "api_version" added to all response "res" objects - -### Changed -- upgraded to Rails 6 +- support for elasticsearch 8.5 +- user/password basic authentication with ES 8.5, when querying the index or posting from Datura +- better support for nested fields +- support for nested bucket aggregations, matching a nested value on another nested value. `person.name[person.role#judge]` will return all names of persons where role="judge". +- "api_version" added to all response "res" objects - Added support for aggregating buckets by normalized keyword and returning the "top_hits" first document result for a non-normalized display - Changes response format of `facets` key @@ -55,6 +56,23 @@ Markdown Spec](https://github.github.com/gfm/). ``` Not only is the response format itself different, but there may be fewer facets returned since normalized values which match are combined +### Changed +- upgraded to Rails 6.1.7 and Ruby 3 +- changes reflect new api schemas in Datura, which make heavy use of nested fields + +### Migration +- in the config files of your Datura repos, (`private.yml` or `public.yml`, set the api to `"api_version": "2.0"` to take advantage of new bucket aggregation functionality (or `"api_version": "1.0"` for legacy repos that have not been updated for the new schema). Please note that a running API index can only use one ES index at a time, and each ES index is restricted to one version of the schema. See new schema (2.0) documentation [here](https://github.com/CDRH/datura/docs/schema_v2.md). +- Use Elasticsearch 8.5 or later. See [dev docs instructions](https://github.com/CDRH/cdrh_dev_docs/blob/update_elasticsearch_documentation/publishing/2_basic_requirements.md#downloading-elasticsearch). +- If you are using ES with security enabled, you must configure credentials with Rails in the API repo. See https://guides.rubyonrails.org/v6.1/security.html. Configure the VSCode editor. Run `EDITOR="code --wait" rails credentials:edit` and add +``` +elasticsearch: + user: username + password: ***** +``` +to the secrets file and then close the window to save. Do not commit `config/master.key` (it should be in `gitignore`) +- Orchid apps that connect to the API should use `facet_limit` instead of `facet_num` in options. +- Add nested facets as described above, if desired. + ## [v1.0.4](https://github.com/CDRH/api/compare/v1.0....v1.0.4) - Updates & license diff --git a/Gemfile b/Gemfile index 88ac8b9..9c639fc 100644 --- a/Gemfile +++ b/Gemfile @@ -7,11 +7,11 @@ end # Bundle edge Rails instead: gem 'rails', github: 'rails/rails' -gem 'rails', '~> 6.0.2' +gem 'rails', '~> 6.1.7' # Use sqlite3 as the database for Active Record gem 'sqlite3' # Use Puma as the app server -gem 'puma', '~> 3.7' +gem 'puma', '>= 5.6' # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder # gem 'jbuilder', '~> 2.5' # Use Redis adapter to run Action Cable in production @@ -26,6 +26,7 @@ gem 'puma', '~> 3.7' # gem 'rack-cors' gem 'bootsnap', require: false +gem 'net-smtp' group :development, :test do # Call 'byebug' anywhere in the code to stop execution and get a debugger console diff --git a/Gemfile.lock b/Gemfile.lock index c38a145..5bc4909 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,62 +1,66 @@ GEM remote: https://rubygems.org/ specs: - actioncable (6.0.5) - actionpack (= 6.0.5) + actioncable (6.1.7) + actionpack (= 6.1.7) + activesupport (= 6.1.7) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailbox (6.0.5) - actionpack (= 6.0.5) - activejob (= 6.0.5) - activerecord (= 6.0.5) - activestorage (= 6.0.5) - activesupport (= 6.0.5) + actionmailbox (6.1.7) + actionpack (= 6.1.7) + activejob (= 6.1.7) + activerecord (= 6.1.7) + activestorage (= 6.1.7) + activesupport (= 6.1.7) mail (>= 2.7.1) - actionmailer (6.0.5) - actionpack (= 6.0.5) - actionview (= 6.0.5) - activejob (= 6.0.5) + actionmailer (6.1.7) + actionpack (= 6.1.7) + actionview (= 6.1.7) + activejob (= 6.1.7) + activesupport (= 6.1.7) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (6.0.5) - actionview (= 6.0.5) - activesupport (= 6.0.5) - rack (~> 2.0, >= 2.0.8) + actionpack (6.1.7) + actionview (= 6.1.7) + activesupport (= 6.1.7) + rack (~> 2.0, >= 2.0.9) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.2.0) - actiontext (6.0.5) - actionpack (= 6.0.5) - activerecord (= 6.0.5) - activestorage (= 6.0.5) - activesupport (= 6.0.5) + actiontext (6.1.7) + actionpack (= 6.1.7) + activerecord (= 6.1.7) + activestorage (= 6.1.7) + activesupport (= 6.1.7) nokogiri (>= 1.8.5) - actionview (6.0.5) - activesupport (= 6.0.5) + actionview (6.1.7) + activesupport (= 6.1.7) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.1, >= 1.2.0) - activejob (6.0.5) - activesupport (= 6.0.5) + activejob (6.1.7) + activesupport (= 6.1.7) globalid (>= 0.3.6) - activemodel (6.0.5) - activesupport (= 6.0.5) - activerecord (6.0.5) - activemodel (= 6.0.5) - activesupport (= 6.0.5) - activestorage (6.0.5) - actionpack (= 6.0.5) - activejob (= 6.0.5) - activerecord (= 6.0.5) + activemodel (6.1.7) + activesupport (= 6.1.7) + activerecord (6.1.7) + activemodel (= 6.1.7) + activesupport (= 6.1.7) + activestorage (6.1.7) + actionpack (= 6.1.7) + activejob (= 6.1.7) + activerecord (= 6.1.7) + activesupport (= 6.1.7) marcel (~> 1.0) - activesupport (6.0.5) + mini_mime (>= 1.1.0) + activesupport (6.1.7) concurrent-ruby (~> 1.0, >= 1.0.2) - i18n (>= 0.7, < 2) - minitest (~> 5.1) - tzinfo (~> 1.1) - zeitwerk (~> 2.2, >= 2.2.2) - bootsnap (1.11.1) + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0) + zeitwerk (~> 2.3) + bootsnap (1.13.0) msgpack (~> 1.2) builder (3.2.4) byebug (11.1.3) @@ -64,20 +68,19 @@ GEM crass (1.0.6) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - erubi (1.10.0) + erubi (1.11.0) ffi (1.15.5) globalid (1.0.0) activesupport (>= 5.0) http-accept (1.7.0) - http-cookie (1.0.4) + http-cookie (1.0.5) domain_name (~> 0.5) - i18n (1.10.0) + i18n (1.12.0) concurrent-ruby (~> 1.0) - listen (3.1.5) + listen (3.0.8) rb-fsevent (~> 0.9, >= 0.9.4) rb-inotify (~> 0.9, >= 0.9.7) - ruby_dep (~> 1.2) - loofah (2.18.0) + loofah (2.19.0) crass (~> 1.0.2) nokogiri (>= 1.5.9) mail (2.7.1) @@ -88,47 +91,50 @@ GEM mime-types-data (~> 3.2015) mime-types-data (3.2022.0105) mini_mime (1.1.2) - mini_portile2 (2.8.0) - minitest (5.15.0) - msgpack (1.5.1) + minitest (5.16.3) + msgpack (1.6.0) + net-protocol (0.1.3) + timeout + net-smtp (0.3.3) + net-protocol netrc (0.11.0) nio4r (2.5.8) - nokogiri (1.13.6) - mini_portile2 (~> 2.8.0) + nokogiri (1.13.9-x86_64-darwin) racc (~> 1.4) - puma (3.12.6) + puma (6.0.0) + nio4r (~> 2.0) racc (1.6.0) - rack (2.2.3) - rack-test (1.1.0) - rack (>= 1.0, < 3) - rails (6.0.5) - actioncable (= 6.0.5) - actionmailbox (= 6.0.5) - actionmailer (= 6.0.5) - actionpack (= 6.0.5) - actiontext (= 6.0.5) - actionview (= 6.0.5) - activejob (= 6.0.5) - activemodel (= 6.0.5) - activerecord (= 6.0.5) - activestorage (= 6.0.5) - activesupport (= 6.0.5) - bundler (>= 1.3.0) - railties (= 6.0.5) + rack (2.2.4) + rack-test (2.0.2) + rack (>= 1.3) + rails (6.1.7) + actioncable (= 6.1.7) + actionmailbox (= 6.1.7) + actionmailer (= 6.1.7) + actionpack (= 6.1.7) + actiontext (= 6.1.7) + actionview (= 6.1.7) + activejob (= 6.1.7) + activemodel (= 6.1.7) + activerecord (= 6.1.7) + activestorage (= 6.1.7) + activesupport (= 6.1.7) + bundler (>= 1.15.0) + railties (= 6.1.7) sprockets-rails (>= 2.0.0) rails-dom-testing (2.0.3) activesupport (>= 4.2.0) nokogiri (>= 1.6) - rails-html-sanitizer (1.4.2) + rails-html-sanitizer (1.4.3) loofah (~> 2.3) - railties (6.0.5) - actionpack (= 6.0.5) - activesupport (= 6.0.5) + railties (6.1.7) + actionpack (= 6.1.7) + activesupport (= 6.1.7) method_source - rake (>= 0.8.7) - thor (>= 0.20.3, < 2.0) + rake (>= 12.2) + thor (~> 1.0) rake (13.0.6) - rb-fsevent (0.11.1) + rb-fsevent (0.11.2) rb-inotify (0.10.1) ffi (~> 1.0) rest-client (2.1.0) @@ -136,40 +142,40 @@ GEM http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) - ruby_dep (1.5.0) spring (2.1.1) spring-watcher-listen (2.0.1) listen (>= 2.7, < 4.0) spring (>= 1.2, < 3.0) - sprockets (4.0.3) + sprockets (4.1.1) concurrent-ruby (~> 1.0) rack (> 1, < 3) sprockets-rails (3.4.2) actionpack (>= 5.2) activesupport (>= 5.2) sprockets (>= 3.0.0) - sqlite3 (1.4.2) + sqlite3 (1.5.3-x86_64-darwin) thor (1.2.1) - thread_safe (0.3.6) - tzinfo (1.2.9) - thread_safe (~> 0.1) + timeout (0.3.0) + tzinfo (2.0.5) + concurrent-ruby (~> 1.0) unf (0.1.4) unf_ext - unf_ext (0.0.8.1) + unf_ext (0.0.8.2) websocket-driver (0.7.5) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) - zeitwerk (2.5.4) + zeitwerk (2.6.6) PLATFORMS - ruby + x86_64-darwin-21 DEPENDENCIES bootsnap byebug listen (>= 3.0.5, < 3.2) - puma (~> 3.7) - rails (~> 6.0.2) + net-smtp + puma (>= 5.6) + rails (~> 6.1.7) rest-client (>= 2.1.0.rc1, < 2.2) spring spring-watcher-listen (~> 2.0.0) @@ -177,4 +183,4 @@ DEPENDENCIES tzinfo-data BUNDLED WITH - 2.2.26 + 2.3.7 diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 15ff0d8..e97e006 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -3,7 +3,8 @@ class ApplicationController < ActionController::API def post_search(json, error_method=method(:display_error)) - res = RestClient.post("#{ES_URI}/_search", json.to_json, { "content-type" => "json" }) + auth_hash = { "Authorization" => "Basic #{Base64::encode64("#{ES_USER}:#{ES_PASSWORD}")}" } + res = RestClient.post("#{ES_URI}/_search", json.to_json, auth_hash.merge({ "content-type" => "json" })) raise return JSON.parse(res.body) rescue => e diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb index e9260c0..749af2f 100644 --- a/app/services/search_item_req.rb +++ b/app/services/search_item_req.rb @@ -51,6 +51,8 @@ def build_request # add bool to request body req["query"]["bool"] = bool + # uncomment below line to log ES query for debugging + # puts req.to_json() return req end @@ -72,19 +74,17 @@ def facets dir = "desc" if @params["facet_sort"].present? sort_type, sort_dir = @params["facet_sort"].split(@@filter_separator) - type = "_term" if sort_type == "term" + type = "term" if sort_type == "term" dir = sort_dir if sort_dir == "asc" end # FACET_SETTINGS["start"] - size = SETTINGS["num"] - size = @params["facet_num"].blank? ? SETTINGS["num"] : @params["facet_num"] + size = @params["facet_limit"].blank? ? SETTINGS["num"] : @params["facet_limit"] aggs = {} Array.wrap(@params["facet"]).each do |f| # histograms use a different ordering terminology than normal aggs - f_type = type == "_term" ? "_key" : "_count" - + f_type = (type == "term") ? "_key" : "_count" if f.include?("date") || f[/_d$/] # NOTE: if nested fields will ever have dates we will # need to refactor this to be available to both @@ -98,13 +98,76 @@ def facets aggs[f] = { "date_histogram" => { "field" => field, - "interval" => interval, + "calendar_interval" => interval, "format" => formatted, "min_doc_count" => 1, "order" => { f_type => dir }, } } - # if nested, has extra syntax + #nested facet, matching on another nested facet + + elsif f.include?("[") + # will be an array including the original, and an alternate aggregation name + + + options = JSON.parse(f) + original = options[0] + agg_name = options[1] + facet = original.split("[")[0] + # may or may not be nested + nested = facet.include?(".") + if nested + path = facet.split(".").first + end + condition = original[/(?<=\[).+?(?=\])/] + subject = condition.split("#").first + predicate = condition.split("#").last + aggregation = { + # common to nested and non-nested + "filter" => { + "term" => { + subject => predicate + } + }, + "aggs" => { + agg_name => { + "terms" => { + "field" => facet, + "order" => {f_type => dir}, + "size" => size + }, + "aggs" => { + "field_to_item" => { + "reverse_nested" => {}, + "aggs" => { + "top_matches" => { + "top_hits" => { + "_source" => { + "includes" => [ agg_name ] + }, + "size" => 1 + } + } + } + } + } + } + } + } + #interpolate above hash into nested query + if nested + aggs[agg_name] = { + "nested" => { + "path" => path + }, + "aggs" => { + agg_name => aggregation + } + } + else + #otherwise it is the whole query + aggs[agg_name] = aggregation + end elsif f.include?(".") path = f.split(".").first aggs[f] = { @@ -115,7 +178,7 @@ def facets f => { "terms" => { "field" => f, - "order" => { type => dir }, + "order" => {f_type => dir}, "size" => size }, "aggs" => { @@ -135,7 +198,7 @@ def facets aggs[f] = { "terms" => { "field" => f, - "order" => { type => dir }, + "order" => { f_type => dir }, "size" => size }, "aggs" => { @@ -161,8 +224,43 @@ def filters # (type 2 will only be used for dates) filters = fields.map {|f| f.split(@@filter_separator, 3) } filters.each do |filter| - # NESTED FIELD FILTER - if filter[0].include?(".") + # filter aggregation with nesting + if filter[0].include?("[") + original = filter[0] + facet = original.split("[")[0] + nested = facet.include?(".") + if nested + path = facet.split(".").first + end + condition = original[/(?<=\[).+?(?=\])/] + subject = condition.split("#").first + predicate = condition.split("#").last + term_match = { + # "person.name" => "oliver wendell holmes" + # Remove CR's added by hidden input field values with returns + facet => filter[1].gsub(/\r/, "") + } + term_filter = { + subject => predicate + } + if nested + query = { + "nested" => { + "path" => path, + "query" => { + "bool" => { + "must" => [ + { "match" => term_filter }, + { "match" => term_match } + ] + } + } + } + } + end + filter_list << query + #ordinary nested facet + elsif filter[0].include?(".") path = filter[0].split(".").first # this is a nested field and must be treated differently nested = { diff --git a/app/services/search_item_res.rb b/app/services/search_item_res.rb index a82a199..05210e7 100644 --- a/app/services/search_item_res.rb +++ b/app/services/search_item_res.rb @@ -18,7 +18,6 @@ def build_response # strip out only the fields for the item response items = combine_highlights facets = reformat_facets - { "code" => 200, "count" => count, @@ -66,7 +65,7 @@ def format_bucket_value(facets, field, bucket) # dates return in wonktastic ways, so grab key_as_string instead of gibberish number # but otherwise just grab the key if key_as_string unavailable key = bucket.key?("key_as_string") ? bucket["key_as_string"] : bucket["key"] - val = bucket["doc_count"] + val = bucket.key?("field_to_item") ? bucket["field_to_item"]["doc_count"] : bucket["doc_count"] source = key # top_matches is a top_hits aggregation which returns a list of terms # which were used for the facet. @@ -89,8 +88,7 @@ def reformat_facets facets = {} raw_facets.each do |field, info| facets[field] = {} - # nested fields do not have buckets at this level of response structure - buckets = info.key?("buckets") ? info["buckets"] : info.dig(field, "buckets") + buckets = get_buckets(info, field) if buckets buckets.each { |b| format_bucket_value(facets, field, b) } else @@ -110,4 +108,18 @@ def remove_nonword_chars(term) transliterated.gsub(/<\/?(?:em|strong|u)>|\W/, "").downcase end + def get_buckets(info, field) + buckets = nil + # ordinary facet + if info.key?("buckets") + buckets = info["buckets"] + # nested facet + elsif info.dig(field, "buckets") + buckets = info.dig(field, "buckets") + # filtered facet + else + buckets = info.dig(field, field, "buckets") + end + buckets + end end diff --git a/app/services/search_service.rb b/app/services/search_service.rb index dbd8877..319b97b 100644 --- a/app/services/search_service.rb +++ b/app/services/search_service.rb @@ -11,7 +11,8 @@ def initialize(url, params={}, user_req) end def post(url_ending, json) - res = RestClient.post("#{@url}/#{url_ending}", json.to_json, { "content-type" => "json" } ) + auth_hash = { "Authorization" => "Basic #{Base64::encode64("#{Rails.application.credentials.elasticsearch[:user]}:#{Rails.application.credentials.elasticsearch[:password]}")}" } + res = RestClient.post("#{@url}/#{url_ending}", json.to_json, auth_hash.merge({ "content-type" => "json" } )) JSON.parse(res.body) rescue => e e diff --git a/bin/rails b/bin/rails index 0739660..21d3e02 100755 --- a/bin/rails +++ b/bin/rails @@ -1,4 +1,5 @@ #!/usr/bin/env ruby +load File.expand_path("spring", __dir__) APP_PATH = File.expand_path('../config/application', __dir__) -require_relative '../config/boot' -require 'rails/commands' +require_relative "../config/boot" +require "rails/commands" diff --git a/bin/rake b/bin/rake index 1724048..7327f47 100755 --- a/bin/rake +++ b/bin/rake @@ -1,4 +1,5 @@ #!/usr/bin/env ruby -require_relative '../config/boot' -require 'rake' +load File.expand_path("spring", __dir__) +require_relative "../config/boot" +require "rake" Rake.application.run diff --git a/bin/setup b/bin/setup index 0e39e8c..5792302 100755 --- a/bin/setup +++ b/bin/setup @@ -1,5 +1,5 @@ #!/usr/bin/env ruby -require 'fileutils' +require "fileutils" # path to your application root. APP_ROOT = File.expand_path('..', __dir__) @@ -9,8 +9,8 @@ def system!(*args) end FileUtils.chdir APP_ROOT do - # This script is a way to setup or update your development environment automatically. - # This script is idempotent, so that you can run it at anytime and get an expectable outcome. + # This script is a way to set up or update your development environment automatically. + # This script is idempotent, so that you can run it at any time and get an expectable outcome. # Add necessary setup steps to this file. puts '== Installing dependencies ==' diff --git a/bin/spring b/bin/spring new file mode 100755 index 0000000..b4147e8 --- /dev/null +++ b/bin/spring @@ -0,0 +1,14 @@ +#!/usr/bin/env ruby +if !defined?(Spring) && [nil, "development", "test"].include?(ENV["RAILS_ENV"]) + gem "bundler" + require "bundler" + + # Load Spring without loading other gems in the Gemfile, for speed. + Bundler.locked_gems&.specs&.find { |spec| spec.name == "spring" }&.tap do |spring| + Gem.use_paths Gem.dir, Bundler.bundle_path.to_s, *Gem.path + gem "spring", spring.version + require "spring/binstub" + rescue Gem::LoadError + # Ignore when Spring is not installed. + end +end diff --git a/config.ru b/config.ru index f7ba0b5..4a3c09a 100644 --- a/config.ru +++ b/config.ru @@ -1,5 +1,6 @@ # This file is used by Rack-based servers to start the application. -require_relative 'config/environment' +require_relative "config/environment" run Rails.application +Rails.application.load_server diff --git a/config/application.rb b/config/application.rb index b9c2970..03f5085 100644 --- a/config/application.rb +++ b/config/application.rb @@ -1,4 +1,4 @@ -require_relative 'boot' +require_relative "boot" require "rails" # Pick the frameworks you want: @@ -24,10 +24,13 @@ class Application < Rails::Application # Initialize configuration defaults for originally generated Rails version. config.load_defaults 5.0 - # Settings in config/environments/* take precedence over those specified here. - # Application configuration can go into files in config/initializers - # -- all .rb files in that directory are automatically loaded after loading - # the framework and any gems in your application. + # Configuration for the application, engines, and railties goes here. + # + # These settings can be overridden in specific environments using the files + # in config/environments, which are processed later. + # + # config.time_zone = "Central Time (US & Canada)" + # config.eager_load_paths << Rails.root.join("extras") # Only loads a smaller set of middleware suitable for API only apps. # Middleware like session, flash, cookies can be added back manually. diff --git a/config/boot.rb b/config/boot.rb index b9e460c..3cda23b 100644 --- a/config/boot.rb +++ b/config/boot.rb @@ -1,4 +1,4 @@ ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__) -require 'bundler/setup' # Set up gems listed in the Gemfile. -require 'bootsnap/setup' # Speed up boot time by caching expensive operations. +require "bundler/setup" # Set up gems listed in the Gemfile. +require "bootsnap/setup" # Speed up boot time by caching expensive operations. diff --git a/config/environment.rb b/config/environment.rb index 426333b..cac5315 100644 --- a/config/environment.rb +++ b/config/environment.rb @@ -1,5 +1,5 @@ # Load the Rails application. -require_relative 'application' +require_relative "application" # Initialize the Rails application. Rails.application.initialize! diff --git a/config/environments/development.rb b/config/environments/development.rb index 1e22e09..78b25c4 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -1,8 +1,10 @@ +require "active_support/core_ext/integer/time" + Rails.application.configure do # Settings specified here will take precedence over those in config/application.rb. - # In the development environment your application's code is reloaded on - # every request. This slows down response time but is perfect for development + # In the development environment your application's code is reloaded any time + # it changes. This slows down response time but is perfect for development # since you don't have to restart the web server when you make code changes. config.cache_classes = false @@ -36,6 +38,12 @@ # Print deprecation notices to the Rails logger. config.active_support.deprecation = :log + # Raise exceptions for disallowed deprecations. + config.active_support.disallowed_deprecation = :raise + + # Tell Active Support which deprecation messages to disallow. + config.active_support.disallowed_deprecation_warnings = [] + # Raise an error on page load if there are pending migrations. config.active_record.migration_error = :page_load @@ -44,11 +52,15 @@ # Raises error for missing translations. - # config.action_view.raise_on_missing_translations = true + # config.i18n.raise_on_missing_translations = true + + # Annotate rendered view with file names. + # config.action_view.annotate_rendered_view_with_filenames = true # Use an evented file watcher to asynchronously detect changes in source code, # routes, locales, etc. This feature depends on the listen gem. config.file_watcher = ActiveSupport::EventedFileUpdateChecker + # LOCAL # Custom dev env logger to empty log more frequently config.logger = ActiveSupport::TaggedLogging.new( @@ -61,4 +73,7 @@ # CDRH CONFIGURATION config.hosts << "cdrhdev1.unl.edu" + config.hosts << "whitman-dev.unl.edu" + # Uncomment if you wish to allow Action Cable access from any origin. + # config.action_cable.disable_request_forgery_protection = true end diff --git a/config/environments/production.rb b/config/environments/production.rb index 1af1588..50bdaf0 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -1,3 +1,5 @@ +require "active_support/core_ext/integer/time" + Rails.application.configure do # Settings specified here will take precedence over those in config/application.rb. @@ -22,7 +24,7 @@ config.public_file_server.enabled = ENV['RAILS_SERVE_STATIC_FILES'].present? # Enable serving of images, stylesheets, and JavaScripts from an asset server. - # config.action_controller.asset_host = 'http://assets.example.com' + # config.asset_host = 'http://assets.example.com' # Specifies the header that your server uses for sending files. # config.action_dispatch.x_sendfile_header = 'X-Sendfile' # for Apache @@ -39,9 +41,9 @@ # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. # config.force_ssl = true - # Use the lowest log level to ensure availability of diagnostic information - # when problems arise. - config.log_level = :debug + # Include generic and useful information about system operation, but avoid logging too much + # information to avoid inadvertent exposure of personally identifiable information (PII). + config.log_level = :info # Prepend all log lines with the following tags. config.log_tags = [ :request_id ] @@ -66,11 +68,17 @@ # Send deprecation notices to registered listeners. config.active_support.deprecation = :notify + # Log disallowed deprecations. + config.active_support.disallowed_deprecation = :log + + # Tell Active Support which deprecation messages to disallow. + config.active_support.disallowed_deprecation_warnings = [] + # Use default logging formatter so that PID and timestamp are not suppressed. config.log_formatter = ::Logger::Formatter.new # Use a different logger for distributed setups. - # require 'syslog/logger' + # require "syslog/logger" # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new 'app-name') if ENV["RAILS_LOG_TO_STDOUT"].present? diff --git a/config/environments/test.rb b/config/environments/test.rb index 1d62e91..93ed4f1 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -1,3 +1,5 @@ +require "active_support/core_ext/integer/time" + # The test environment is used exclusively to run your application's # test suite. You never need to work with it otherwise. Remember that # your test database is "scratch space" for the test suite and is wiped @@ -7,6 +9,7 @@ # Settings specified here will take precedence over those in config/application.rb. config.cache_classes = false + config.action_view.cache_template_loading = true # Do not eager load code on boot. This avoids loading your whole application # just for the purpose of running a single test. If you are using a tool that @@ -43,6 +46,15 @@ # Print deprecation notices to the stderr. config.active_support.deprecation = :stderr + # Raise exceptions for disallowed deprecations. + config.active_support.disallowed_deprecation = :raise + + # Tell Active Support which deprecation messages to disallow. + config.active_support.disallowed_deprecation_warnings = [] + # Raises error for missing translations. - # config.action_view.raise_on_missing_translations = true + # config.i18n.raise_on_missing_translations = true + + # Annotate rendered view with file names. + # config.action_view.annotate_rendered_view_with_filenames = true end diff --git a/config/initializers/backtrace_silencers.rb b/config/initializers/backtrace_silencers.rb index 59385cd..33699c3 100644 --- a/config/initializers/backtrace_silencers.rb +++ b/config/initializers/backtrace_silencers.rb @@ -1,7 +1,8 @@ # Be sure to restart your server when you modify this file. # You can add backtrace silencers for libraries that you're using but don't wish to see in your backtraces. -# Rails.backtrace_cleaner.add_silencer { |line| line =~ /my_noisy_library/ } +# Rails.backtrace_cleaner.add_silencer { |line| /my_noisy_library/.match?(line) } -# You can also remove all the silencers if you're trying to debug a problem that might stem from framework code. -# Rails.backtrace_cleaner.remove_silencers! +# You can also remove all the silencers if you're trying to debug a problem that might stem from framework code +# by setting BACKTRACE=1 before calling your invocation, like "BACKTRACE=1 ./bin/rails runner 'MyClass.perform'". +Rails.backtrace_cleaner.remove_silencers! if ENV["BACKTRACE"] diff --git a/config/initializers/filter_parameter_logging.rb b/config/initializers/filter_parameter_logging.rb index 4a994e1..4b34a03 100644 --- a/config/initializers/filter_parameter_logging.rb +++ b/config/initializers/filter_parameter_logging.rb @@ -1,4 +1,6 @@ # Be sure to restart your server when you modify this file. # Configure sensitive parameters which will be filtered from the log file. -Rails.application.config.filter_parameters += [:password] +Rails.application.config.filter_parameters += [ + :passw, :secret, :token, :_key, :crypt, :salt, :certificate, :otp, :ssn +] diff --git a/config/initializers/new_framework_defaults_6_1.rb b/config/initializers/new_framework_defaults_6_1.rb new file mode 100644 index 0000000..9526b83 --- /dev/null +++ b/config/initializers/new_framework_defaults_6_1.rb @@ -0,0 +1,67 @@ +# Be sure to restart your server when you modify this file. +# +# This file contains migration options to ease your Rails 6.1 upgrade. +# +# Once upgraded flip defaults one by one to migrate to the new default. +# +# Read the Guide for Upgrading Ruby on Rails for more info on each option. + +# Support for inversing belongs_to -> has_many Active Record associations. +# Rails.application.config.active_record.has_many_inversing = true + +# Track Active Storage variants in the database. +# Rails.application.config.active_storage.track_variants = true + +# Apply random variation to the delay when retrying failed jobs. +# Rails.application.config.active_job.retry_jitter = 0.15 + +# Stop executing `after_enqueue`/`after_perform` callbacks if +# `before_enqueue`/`before_perform` respectively halts with `throw :abort`. +# Rails.application.config.active_job.skip_after_callbacks_if_terminated = true + +# Specify cookies SameSite protection level: either :none, :lax, or :strict. +# +# This change is not backwards compatible with earlier Rails versions. +# It's best enabled when your entire app is migrated and stable on 6.1. +# Rails.application.config.action_dispatch.cookies_same_site_protection = :lax + +# Generate CSRF tokens that are encoded in URL-safe Base64. +# +# This change is not backwards compatible with earlier Rails versions. +# It's best enabled when your entire app is migrated and stable on 6.1. +# Rails.application.config.action_controller.urlsafe_csrf_tokens = true + +# Specify whether `ActiveSupport::TimeZone.utc_to_local` returns a time with an +# UTC offset or a UTC time. +# ActiveSupport.utc_to_local_returns_utc_offset_times = true + +# Change the default HTTP status code to `308` when redirecting non-GET/HEAD +# requests to HTTPS in `ActionDispatch::SSL` middleware. +# Rails.application.config.action_dispatch.ssl_default_redirect_status = 308 + +# Use new connection handling API. For most applications this won't have any +# effect. For applications using multiple databases, this new API provides +# support for granular connection swapping. +# Rails.application.config.active_record.legacy_connection_handling = false + +# Make `form_with` generate non-remote forms by default. +# Rails.application.config.action_view.form_with_generates_remote_forms = false + +# Set the default queue name for the analysis job to the queue adapter default. +# Rails.application.config.active_storage.queues.analysis = nil + +# Set the default queue name for the purge job to the queue adapter default. +# Rails.application.config.active_storage.queues.purge = nil + +# Set the default queue name for the incineration job to the queue adapter default. +# Rails.application.config.action_mailbox.queues.incineration = nil + +# Set the default queue name for the routing job to the queue adapter default. +# Rails.application.config.action_mailbox.queues.routing = nil + +# Set the default queue name for the mail deliver job to the queue adapter default. +# Rails.application.config.action_mailer.deliver_later_queue_name = nil + +# Generate a `Link` header that gives a hint to modern browsers about +# preloading assets when using `javascript_include_tag` and `stylesheet_link_tag`. +# Rails.application.config.action_view.preload_links_header = true diff --git a/config/puma.rb b/config/puma.rb index 5ed4437..d9b3e83 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -8,9 +8,14 @@ min_threads_count = ENV.fetch("RAILS_MIN_THREADS") { max_threads_count } threads min_threads_count, max_threads_count +# Specifies the `worker_timeout` threshold that Puma will use to wait before +# terminating a worker in development environments. +# +worker_timeout 3600 if ENV.fetch("RAILS_ENV", "development") == "development" + # Specifies the `port` that Puma will listen on to receive requests; default is 3000. # -port ENV.fetch("PORT") { 3000 } +port ENV.fetch("PORT") { 3000 } # Specifies the `environment` that Puma will run in. # diff --git a/db/migrate/20221109145721_add_service_name_to_active_storage_blobs.active_storage.rb b/db/migrate/20221109145721_add_service_name_to_active_storage_blobs.active_storage.rb new file mode 100644 index 0000000..a15c6ce --- /dev/null +++ b/db/migrate/20221109145721_add_service_name_to_active_storage_blobs.active_storage.rb @@ -0,0 +1,22 @@ +# This migration comes from active_storage (originally 20190112182829) +class AddServiceNameToActiveStorageBlobs < ActiveRecord::Migration[6.0] + def up + return unless table_exists?(:active_storage_blobs) + + unless column_exists?(:active_storage_blobs, :service_name) + add_column :active_storage_blobs, :service_name, :string + + if configured_service = ActiveStorage::Blob.service.name + ActiveStorage::Blob.unscoped.update_all(service_name: configured_service) + end + + change_column :active_storage_blobs, :service_name, :string, null: false + end + end + + def down + return unless table_exists?(:active_storage_blobs) + + remove_column :active_storage_blobs, :service_name + end +end diff --git a/db/migrate/20221109145722_create_active_storage_variant_records.active_storage.rb b/db/migrate/20221109145722_create_active_storage_variant_records.active_storage.rb new file mode 100644 index 0000000..94ac83a --- /dev/null +++ b/db/migrate/20221109145722_create_active_storage_variant_records.active_storage.rb @@ -0,0 +1,27 @@ +# This migration comes from active_storage (originally 20191206030411) +class CreateActiveStorageVariantRecords < ActiveRecord::Migration[6.0] + def change + return unless table_exists?(:active_storage_blobs) + + # Use Active Record's configured type for primary key + create_table :active_storage_variant_records, id: primary_key_type, if_not_exists: true do |t| + t.belongs_to :blob, null: false, index: false, type: blobs_primary_key_type + t.string :variation_digest, null: false + + t.index %i[ blob_id variation_digest ], name: "index_active_storage_variant_records_uniqueness", unique: true + t.foreign_key :active_storage_blobs, column: :blob_id + end + end + + private + def primary_key_type + config = Rails.configuration.generators + config.options[config.orm][:primary_key_type] || :primary_key + end + + def blobs_primary_key_type + pkey_name = connection.primary_key(:active_storage_blobs) + pkey_column = connection.columns(:active_storage_blobs).find { |c| c.name == pkey_name } + pkey_column.bigint? ? :bigint : pkey_column.type + end +end diff --git a/db/schema.rb b/db/schema.rb new file mode 100644 index 0000000..adc3722 --- /dev/null +++ b/db/schema.rb @@ -0,0 +1,15 @@ +# This file is auto-generated from the current state of the database. Instead +# of editing this file, please use the migrations feature of Active Record to +# incrementally modify your database, and then regenerate this schema definition. +# +# This file is the source Rails uses to define your schema when running `bin/rails +# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to +# be faster and is potentially less error prone than running all of your +# migrations from scratch. Old migrations may fail to apply correctly if those +# migrations use external dependencies or application code. +# +# It's strongly recommended that you check this file into your version control system. + +ActiveRecord::Schema.define(version: 2022_11_09_145722) do + +end diff --git a/docs/README.md b/docs/README.md index 8aa9ad6..cd2fc90 100644 --- a/docs/README.md +++ b/docs/README.md @@ -50,6 +50,12 @@ __Nested fields__ facet[]=creator.name facet[]=creator.name&facet[]=creator.role ``` +you can also match on another nested field with the new API schema +`facet[]=nested_field.keyword_field1[nested_field.keyword_field2#value]` +``` +facet[]=person.name[person.role#judge] +``` +the above will select all names of persons, where the role of that person is "judge". __Date ranges__ (currently supports days or years) diff --git a/test/services/search_item_req_test.rb b/test/services/search_item_req_test.rb index 29bf323..a5b3eab 100644 --- a/test/services/search_item_req_test.rb +++ b/test/services/search_item_req_test.rb @@ -39,18 +39,18 @@ def test_facets # normal with pagination overrides, multiple facets facets = SearchItemReq.new({ - "facet_num" => 10, + "facet_limit" => 10, "facet_sort" => "term|asc", "facet" => [ "title", "subcategory" ] }).facets assert_equal( - {"title"=>{"terms"=>{"field"=>"title", "order"=>{"_term"=>"asc"}, "size"=>10}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["title"]}, "size"=>1}}}}, "subcategory"=>{"terms"=>{"field"=>"subcategory", "order"=>{"_term"=>"asc"}, "size"=>10}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["subcategory"]}, "size"=>1}}}}}, + {"title"=>{"terms"=>{"field"=>"title", "order"=>"asc", "size"=>10}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["title"]}, "size"=>1}}}}, "subcategory"=>{"terms"=>{"field"=>"subcategory", "order"=>"asc", "size"=>10}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["subcategory"]}, "size"=>1}}}}}, facets ) # should be blank if there are no facets provided facets = SearchItemReq.new({ - "facet_num" => 1, + "facet_limit" => 1, "facet_sort" => "nonterm|asc", "facet" => [] }).facets @@ -69,7 +69,7 @@ def test_facets "facet" => [ "creator.name" ] }).facets assert_equal( - {"creator.name"=>{"nested"=>{"path"=>"creator"}, "aggs"=>{"creator.name"=>{"terms"=>{"field"=>"creator.name", "order"=>{"_term"=>"desc"}, "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["creator.name"]}, "size"=>1}}}}}}}, + {"creator.name"=>{"nested"=>{"path"=>"creator"}, "aggs"=>{"creator.name"=>{"terms"=>{"field"=>"creator.name", "order"=>"desc", "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["creator.name"]}, "size"=>1}}}}}}}, facets ) @@ -83,14 +83,14 @@ def test_facets # sort term order specified facets = SearchItemReq.new({ "facet" => ["title", "format"], "facet_sort" => "term|desc" }).facets assert_equal( - {"title"=>{"terms"=>{"field"=>"title", "order"=>{"_term"=>"desc"}, "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["title"]}, "size"=>1}}}}, "format"=>{"terms"=>{"field"=>"format", "order"=>{"_term"=>"desc"}, "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["format"]}, "size"=>1}}}}}, + {"title"=>{"terms"=>{"field"=>"title", "order"=>"desc", "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["title"]}, "size"=>1}}}}, "format"=>{"terms"=>{"field"=>"format", "order"=>"desc", "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["format"]}, "size"=>1}}}}}, facets ) # sort term no order specified facets = SearchItemReq.new({ "facet" => ["title", "format"], "facet_sort" => "term" }).facets assert_equal( - {"title"=>{"terms"=>{"field"=>"title", "order"=>{"_term"=>"desc"}, "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["title"]}, "size"=>1}}}}, "format"=>{"terms"=>{"field"=>"format", "order"=>{"_term"=>"desc"}, "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["format"]}, "size"=>1}}}}}, + {"title"=>{"terms"=>{"field"=>"title", "order"=>"desc", "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["title"]}, "size"=>1}}}}, "format"=>{"terms"=>{"field"=>"format", "order"=>"desc", "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["format"]}, "size"=>1}}}}}, facets )