Facet display

.gitignore

@@ -48,3 +48,5 @@ bower.json
 .byebug_history
 
 .DS_Store
+
+/config/master.key

.ruby-gemset

@@ -1 +1 @@
-api
+api-v2

.ruby-version

@@ -1 +1 @@
-ruby-2.6.8
+ruby-3.1.2

CHANGELOG.md

@@ -27,15 +27,16 @@ Markdown Spec](https://github.github.com/gfm/).
 
 ### Contributors
 -->
-
-## [Unreleased] - updates in preparation for Habeas release
+## [2.0.0] - new nested bucket aggregation/query functionality for Habeas release
 [Unreleased]: https://github.com/CDRH/api/compare/v1.0.4...dev
 
 ### Added
 - "api_version" added to all response "res" objects
-
-### Changed
-- upgraded to Rails 6
+- support for elasticsearch 8.5
+- user/password basic authentication with ES 8.5, when querying the index or posting from Datura
+- better support for nested fields
+- support for nested bucket aggregations, matching a nested value on another nested value. `person.name[person.role#judge]` will return all names of persons where role="judge".
+- "api_version" added to all response "res" objects
 - Added support for aggregating buckets by normalized keyword and returning
   the "top_hits" first document result for a non-normalized display
 - Changes response format of `facets` key
@@ -55,6 +56,23 @@ Markdown Spec](https://github.github.com/gfm/).
   ```
   Not only is the response format itself different, but there may be fewer
   facets returned since normalized values which match are combined
+### Changed
+- upgraded to Rails 6.1.7 and Ruby 3
+- changes reflect new api schemas in Datura, which make heavy use of nested fields
+
+### Migration
+- in the config files of your Datura repos, (`private.yml` or `public.yml`, set the api to `"api_version": "2.0"` to take advantage of new bucket aggregation functionality (or `"api_version": "1.0"` for legacy repos that have not been updated for the new schema). Please note that a running API index can only use one ES index at a time, and each ES index is restricted to one version of the schema. See new schema (2.0) documentation [here](https://github.com/CDRH/datura/docs/schema_v2.md).
+- Use Elasticsearch 8.5 or later. See [dev docs instructions](https://github.com/CDRH/cdrh_dev_docs/blob/update_elasticsearch_documentation/publishing/2_basic_requirements.md#downloading-elasticsearch).
+- If you are using ES with security enabled, you must configure credentials with Rails in the API repo. See https://guides.rubyonrails.org/v6.1/security.html. Configure the VSCode editor. Run `EDITOR="code --wait" rails credentials:edit` and add
+```
+elasticsearch:
+  user: username
+  password: *****
+```
+to the secrets file and then close the window to save. Do not commit `config/master.key` (it should be in `gitignore`)
+- Orchid apps that connect to the API should use `facet_limit` instead of `facet_num` in options.
+- Add nested facets as described above, if desired.
+
 
 ## [v1.0.4](https://github.com/CDRH/api/compare/v1.0....v1.0.4) - Updates & license
 

Gemfile

@@ -7,11 +7,11 @@ end
 
 
 # Bundle edge Rails instead: gem 'rails', github: 'rails/rails'
-gem 'rails', '~> 6.0.2'
+gem 'rails', '~> 6.1.7'
 # Use sqlite3 as the database for Active Record
 gem 'sqlite3'
 # Use Puma as the app server
-gem 'puma', '~> 3.7'
+gem 'puma', '>= 5.6'
 # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder
 # gem 'jbuilder', '~> 2.5'
 # Use Redis adapter to run Action Cable in production
@@ -26,6 +26,7 @@ gem 'puma', '~> 3.7'
 # gem 'rack-cors'
 
 gem 'bootsnap', require: false
+gem 'net-smtp'
 
 group :development, :test do
   # Call 'byebug' anywhere in the code to stop execution and get a debugger console

Gemfile.lock

@@ -1,83 +1,86 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (6.0.5)
-      actionpack (= 6.0.5)
+    actioncable (6.1.7)
+      actionpack (= 6.1.7)
+      activesupport (= 6.1.7)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailbox (6.0.5)
-      actionpack (= 6.0.5)
-      activejob (= 6.0.5)
-      activerecord (= 6.0.5)
-      activestorage (= 6.0.5)
-      activesupport (= 6.0.5)
+    actionmailbox (6.1.7)
+      actionpack (= 6.1.7)
+      activejob (= 6.1.7)
+      activerecord (= 6.1.7)
+      activestorage (= 6.1.7)
+      activesupport (= 6.1.7)
       mail (>= 2.7.1)
-    actionmailer (6.0.5)
-      actionpack (= 6.0.5)
-      actionview (= 6.0.5)
-      activejob (= 6.0.5)
+    actionmailer (6.1.7)
+      actionpack (= 6.1.7)
+      actionview (= 6.1.7)
+      activejob (= 6.1.7)
+      activesupport (= 6.1.7)
       mail (~> 2.5, >= 2.5.4)
       rails-dom-testing (~> 2.0)
-    actionpack (6.0.5)
-      actionview (= 6.0.5)
-      activesupport (= 6.0.5)
-      rack (~> 2.0, >= 2.0.8)
+    actionpack (6.1.7)
+      actionview (= 6.1.7)
+      activesupport (= 6.1.7)
+      rack (~> 2.0, >= 2.0.9)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (6.0.5)
-      actionpack (= 6.0.5)
-      activerecord (= 6.0.5)
-      activestorage (= 6.0.5)
-      activesupport (= 6.0.5)
+    actiontext (6.1.7)
+      actionpack (= 6.1.7)
+      activerecord (= 6.1.7)
+      activestorage (= 6.1.7)
+      activesupport (= 6.1.7)
       nokogiri (>= 1.8.5)
-    actionview (6.0.5)
-      activesupport (= 6.0.5)
+    actionview (6.1.7)
+      activesupport (= 6.1.7)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    activejob (6.0.5)
-      activesupport (= 6.0.5)
+    activejob (6.1.7)
+      activesupport (= 6.1.7)
       globalid (>= 0.3.6)
-    activemodel (6.0.5)
-      activesupport (= 6.0.5)
-    activerecord (6.0.5)
-      activemodel (= 6.0.5)
-      activesupport (= 6.0.5)
-    activestorage (6.0.5)
-      actionpack (= 6.0.5)
-      activejob (= 6.0.5)
-      activerecord (= 6.0.5)
+    activemodel (6.1.7)
+      activesupport (= 6.1.7)
+    activerecord (6.1.7)
+      activemodel (= 6.1.7)
+      activesupport (= 6.1.7)
+    activestorage (6.1.7)
+      actionpack (= 6.1.7)
+      activejob (= 6.1.7)
+      activerecord (= 6.1.7)
+      activesupport (= 6.1.7)
       marcel (~> 1.0)
-    activesupport (6.0.5)
+      mini_mime (>= 1.1.0)
+    activesupport (6.1.7)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 0.7, < 2)
-      minitest (~> 5.1)
-      tzinfo (~> 1.1)
-      zeitwerk (~> 2.2, >= 2.2.2)
-    bootsnap (1.11.1)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    bootsnap (1.13.0)
       msgpack (~> 1.2)
     builder (3.2.4)
     byebug (11.1.3)
     concurrent-ruby (1.1.10)
     crass (1.0.6)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    erubi (1.10.0)
+    erubi (1.11.0)
     ffi (1.15.5)
     globalid (1.0.0)
       activesupport (>= 5.0)
     http-accept (1.7.0)
-    http-cookie (1.0.4)
+    http-cookie (1.0.5)
       domain_name (~> 0.5)
-    i18n (1.10.0)
+    i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    listen (3.1.5)
+    listen (3.0.8)
       rb-fsevent (~> 0.9, >= 0.9.4)
       rb-inotify (~> 0.9, >= 0.9.7)
-      ruby_dep (~> 1.2)
-    loofah (2.18.0)
+    loofah (2.19.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     mail (2.7.1)
@@ -88,93 +91,96 @@ GEM
       mime-types-data (~> 3.2015)
     mime-types-data (3.2022.0105)
     mini_mime (1.1.2)
-    mini_portile2 (2.8.0)
-    minitest (5.15.0)
-    msgpack (1.5.1)
+    minitest (5.16.3)
+    msgpack (1.6.0)
+    net-protocol (0.1.3)
+      timeout
+    net-smtp (0.3.3)
+      net-protocol
     netrc (0.11.0)
     nio4r (2.5.8)
-    nokogiri (1.13.6)
-      mini_portile2 (~> 2.8.0)
+    nokogiri (1.13.9-x86_64-darwin)
       racc (~> 1.4)
-    puma (3.12.6)
+    puma (6.0.0)
+      nio4r (~> 2.0)
     racc (1.6.0)
-    rack (2.2.3)
-    rack-test (1.1.0)
-      rack (>= 1.0, < 3)
-    rails (6.0.5)
-      actioncable (= 6.0.5)
-      actionmailbox (= 6.0.5)
-      actionmailer (= 6.0.5)
-      actionpack (= 6.0.5)
-      actiontext (= 6.0.5)
-      actionview (= 6.0.5)
-      activejob (= 6.0.5)
-      activemodel (= 6.0.5)
-      activerecord (= 6.0.5)
-      activestorage (= 6.0.5)
-      activesupport (= 6.0.5)
-      bundler (>= 1.3.0)
-      railties (= 6.0.5)
+    rack (2.2.4)
+    rack-test (2.0.2)
+      rack (>= 1.3)
+    rails (6.1.7)
+      actioncable (= 6.1.7)
+      actionmailbox (= 6.1.7)
+      actionmailer (= 6.1.7)
+      actionpack (= 6.1.7)
+      actiontext (= 6.1.7)
+      actionview (= 6.1.7)
+      activejob (= 6.1.7)
+      activemodel (= 6.1.7)
+      activerecord (= 6.1.7)
+      activestorage (= 6.1.7)
+      activesupport (= 6.1.7)
+      bundler (>= 1.15.0)
+      railties (= 6.1.7)
       sprockets-rails (>= 2.0.0)
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.4.2)
+    rails-html-sanitizer (1.4.3)
       loofah (~> 2.3)
-    railties (6.0.5)
-      actionpack (= 6.0.5)
-      activesupport (= 6.0.5)
+    railties (6.1.7)
+      actionpack (= 6.1.7)
+      activesupport (= 6.1.7)
       method_source
-      rake (>= 0.8.7)
-      thor (>= 0.20.3, < 2.0)
+      rake (>= 12.2)
+      thor (~> 1.0)
     rake (13.0.6)
-    rb-fsevent (0.11.1)
+    rb-fsevent (0.11.2)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
-    ruby_dep (1.5.0)
     spring (2.1.1)
     spring-watcher-listen (2.0.1)
       listen (>= 2.7, < 4.0)
       spring (>= 1.2, < 3.0)
-    sprockets (4.0.3)
+    sprockets (4.1.1)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
     sprockets-rails (3.4.2)
       actionpack (>= 5.2)
       activesupport (>= 5.2)
       sprockets (>= 3.0.0)
-    sqlite3 (1.4.2)
+    sqlite3 (1.5.3-x86_64-darwin)
     thor (1.2.1)
-    thread_safe (0.3.6)
-    tzinfo (1.2.9)
-      thread_safe (~> 0.1)
+    timeout (0.3.0)
+    tzinfo (2.0.5)
+      concurrent-ruby (~> 1.0)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.8.1)
+    unf_ext (0.0.8.2)
     websocket-driver (0.7.5)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
-    zeitwerk (2.5.4)
+    zeitwerk (2.6.6)
 
 PLATFORMS
-  ruby
+  x86_64-darwin-21
 
 DEPENDENCIES
   bootsnap
   byebug
   listen (>= 3.0.5, < 3.2)
-  puma (~> 3.7)
-  rails (~> 6.0.2)
+  net-smtp
+  puma (>= 5.6)
+  rails (~> 6.1.7)
   rest-client (>= 2.1.0.rc1, < 2.2)
   spring
   spring-watcher-listen (~> 2.0.0)
   sqlite3
   tzinfo-data
 
 BUNDLED WITH
-   2.2.26
+   2.3.7

app/controllers/application_controller.rb

@@ -3,7 +3,8 @@
 class ApplicationController < ActionController::API
 
   def post_search(json, error_method=method(:display_error))
-    res = RestClient.post("#{ES_URI}/_search", json.to_json, { "content-type" => "json" })
+    auth_hash = { "Authorization" => "Basic #{Base64::encode64("#{ES_USER}:#{ES_PASSWORD}")}" }
+    res = RestClient.post("#{ES_URI}/_search", json.to_json, auth_hash.merge({ "content-type" => "json" }))
     raise
     return JSON.parse(res.body)
   rescue => e