Feature: add API monitoring

Dockerfile

@@ -1,13 +1,11 @@
-ARG RUBY_VERSION=3.0
+ARG RUBY_VERSION=3.1.0
 ARG DISTRO_NAME=bullseye
 
 FROM ruby:$RUBY_VERSION-$DISTRO_NAME
 
 RUN apt-get update -yqq && apt-get install -yqq --no-install-recommends \
   openjdk-11-jre-headless \
   raptor2-utils \
-  wait-for-it \
-  libraptor2-dev \
   && rm -rf /var/lib/apt/lists/*
 
 RUN mkdir -p /srv/ontoportal/ontologies_api
@@ -16,7 +14,7 @@ COPY Gemfile* /srv/ontoportal/ontologies_api/
 
 WORKDIR /srv/ontoportal/ontologies_api
 
-RUN gem update --system 3.4.22 # the 3.4.22 can be removed if we support Ruby version > 3.0
+RUN gem update --system
 RUN gem install bundler
 ENV BUNDLE_PATH=/srv/ontoportal/bundle
 RUN bundle install

Gemfile

@@ -17,13 +17,14 @@ end
 
 gem 'request_store'
 gem 'parallel'
+gem 'connection_pool'
 gem 'google-protobuf'
 gem 'net-ftp'
 gem 'json-ld', '~> 3.2.0'
 gem 'rdf-raptor', github:'ruby-rdf/rdf-raptor', ref: '6392ceabf71c3233b0f7f0172f662bd4a22cd534' # use version 3.3.0 when available
 
 # Rack middleware
-gem 'ffi'
+gem 'ffi', '~> 1.15.0'
 gem 'rack-accept'
 gem 'rack-attack', require: 'rack/attack'
 gem 'rack-cache'
@@ -57,15 +58,18 @@ gem 'goo', github: 'ontoportal-lirmm/goo', branch: 'feature/migrate-ruby-3.2'
 gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'development'
 
 
+gem 'rackup'
+gem 'puma', "~> 6.4"
+gem 'listen', '~> 3.8'
+
 group :development do
   # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42
-  gem 'shotgun', github: 'palexander/shotgun', branch: 'ncbo'
   gem 'rubocop'
 end
 
 group :deployment do
   # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42
-  gem 'bcrypt_pbkdf', '>= 1.0', '< 2.0', require: false
+  gem 'bcrypt_pbkdf',  require: false
   gem 'capistrano', '~> 3', require: false
   gem 'capistrano-bundler', require: false
   gem 'capistrano-locally', require: false

Gemfile.lock

@@ -92,14 +92,6 @@ GIT
     rack-post-body-to-params (0.1.8)
       activesupport (>= 2.3)
 
-GIT
-  remote: https://github.com/palexander/shotgun.git
-  revision: db198224aaab2e4cb9b049adccb30e387d88bc3b
-  branch: ncbo
-  specs:
-    shotgun (0.9)
-      rack (>= 1.0)
-
 GIT
   remote: https://github.com/ruby-rdf/rdf-raptor.git
   revision: 6392ceabf71c3233b0f7f0172f662bd4a22cd534
@@ -111,7 +103,7 @@ GIT
 
 GIT
   remote: https://github.com/sinatra/sinatra.git
-  revision: c4b7c04e6d23ef8e17404d64cc731bece268acea
+  revision: c235249abaafa2780b540aca1813dfcf3d17c2dd
   specs:
     rack-protection (4.1.1)
       base64 (>= 0.1.0)
@@ -127,7 +119,7 @@ GIT
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (8.0.1)
+    activesupport (7.2.2.1)
       base64
       benchmark (>= 0.3)
       bigdecimal
@@ -139,7 +131,6 @@ GEM
       minitest (>= 5.1)
       securerandom (>= 0.3)
       tzinfo (~> 2.0, >= 2.0.5)
-      uri (>= 0.13.1)
     addressable (2.8.7)
       public_suffix (>= 2.0.2, < 7.0)
     airbrussh (1.5.3)
@@ -188,8 +179,7 @@ GEM
       net-http (>= 0.5.0)
     faraday-retry (2.2.1)
       faraday (~> 2.0)
-    ffi (1.17.1-arm64-darwin)
-    ffi (1.17.1-x86_64-linux-gnu)
+    ffi (1.15.5)
     fugit (1.11.1)
       et-orbi (~> 1, >= 1.2.11)
       raabro (~> 1.4)
@@ -228,6 +218,9 @@ GEM
       faraday (>= 1.0, < 3.a)
     google-cloud-errors (1.4.0)
     google-logging-utils (0.1.0)
+    google-protobuf (4.29.3-aarch64-linux)
+      bigdecimal
+      rake (>= 13)
     google-protobuf (4.29.3-arm64-darwin)
       bigdecimal
       rake (>= 13)
@@ -248,6 +241,9 @@ GEM
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
       signet (>= 0.16, < 2.a)
+    grpc (1.70.1-aarch64-linux)
+      google-protobuf (>= 3.25, < 5.0)
+      googleapis-common-protos-types (~> 1.0)
     grpc (1.70.1-arm64-darwin)
       google-protobuf (>= 3.25, < 5.0)
       googleapis-common-protos-types (~> 1.0)
@@ -266,7 +262,7 @@ GEM
     httpclient (2.8.3)
     i18n (1.14.7)
       concurrent-ruby (~> 1.0)
-    json (2.9.1)
+    json (2.10.1)
     json-canonicalization (0.4.0)
     json-ld (3.2.5)
       htmlentities (~> 4.3)
@@ -284,6 +280,9 @@ GEM
     language_server-protocol (3.17.0.4)
     libxml-ruby (5.0.3)
     link_header (0.0.8)
+    listen (3.9.0)
+      rb-fsevent (~> 0.10, >= 0.10.3)
+      rb-inotify (~> 0.9, >= 0.9.10)
     logger (1.6.5)
     macaddr (1.7.2)
       systemu (~> 2.6.5)
@@ -322,7 +321,7 @@ GEM
       uri
     net-http-persistent (4.0.5)
       connection_pool (~> 2.2)
-    net-imap (0.5.5)
+    net-imap (0.5.6)
       date
       net-protocol
     net-pop (0.1.2)
@@ -338,6 +337,7 @@ GEM
     net-ssh (7.3.0)
     netrc (0.11.0)
     newrelic_rpm (9.17.0)
+    nio4r (2.7.4)
     oj (3.16.9)
       bigdecimal (>= 3.0)
       ostruct (>= 0.2)
@@ -356,9 +356,11 @@ GEM
       coderay (~> 1.1)
       method_source (~> 1.0)
     public_suffix (6.0.1)
+    puma (6.6.0)
+      nio4r (~> 2.0)
     raabro (1.4.0)
     racc (1.8.1)
-    rack (3.1.9)
+    rack (3.1.10)
     rack-accept (0.4.5)
       rack (>= 0.4)
     rack-attack (6.7.0)
@@ -380,6 +382,9 @@ GEM
     rainbow (3.1.1)
     raindrops (0.20.1)
     rake (13.2.1)
+    rb-fsevent (0.11.2)
+    rb-inotify (0.11.1)
+      ffi (~> 1.0)
     rdf (3.3.2)
       bcp47_spec (~> 0.2)
       bigdecimal (~> 3.1, >= 3.1.5)
@@ -461,8 +466,9 @@ GEM
       rack-protection (= 4.1.1)
       rack-session (>= 2.0.0, < 3)
       tilt (~> 2.0)
-    sshkit (1.23.2)
+    sshkit (1.24.0)
       base64
+      logger
       net-scp (>= 1.1.2)
       net-sftp (>= 2.1.2)
       net-ssh (>= 2.8.0)
@@ -497,25 +503,28 @@ GEM
     webrick (1.9.1)
 
 PLATFORMS
+  aarch64-linux
   arm64-darwin-24
   x86_64-linux
 
 DEPENDENCIES
   activesupport
-  bcrypt_pbkdf (>= 1.0, < 2.0)
+  bcrypt_pbkdf
   bigdecimal
   capistrano (~> 3)
   capistrano-bundler
   capistrano-locally
   capistrano-rbenv
+  connection_pool
   crack (= 0.4.5)
   ed25519 (>= 1.2, < 2.0)
-  ffi
+  ffi (~> 1.15.0)
   goo!
   google-protobuf
   haml
   json-ld (~> 3.2.0)
   json-schema
+  listen (~> 3.8)
   minitest
   minitest-fail-fast
   minitest-hooks
@@ -531,6 +540,7 @@ DEPENDENCIES
   ontologies_linked_data!
   parallel
   parseconfig
+  puma (~> 6.4)
   rack
   rack-accept
   rack-attack
@@ -550,7 +560,6 @@ DEPENDENCIES
   request_store
   rexml
   rubocop
-  shotgun!
   simplecov
   simplecov-cobertura
   sinatra

app.rb

@@ -46,32 +46,32 @@
 # Setup root and static public directory
 set :root, File.dirname(__FILE__)
 use Rack::Static,
-  :urls => ["/static"],
-  :root => "public"
+    :urls => ["/static"],
+    :root => "public"
 
 # Setup the environment
 environment = settings.environment.nil? ? :development : settings.environment
 require_relative "config/config"
 
 if ENV['OVERRIDE_CONFIG'] == 'true'
   LinkedData.config do |config|
-    config.goo_backend_name  = ENV['GOO_BACKEND_NAME']
-    config.goo_host          = ENV['GOO_HOST']
-    config.goo_port          = ENV['GOO_PORT'].to_i
-    config.goo_path_query    = ENV['GOO_PATH_QUERY']
-    config.goo_path_data     = ENV['GOO_PATH_DATA']
-    config.goo_path_update   = ENV['GOO_PATH_UPDATE']
-    config.goo_redis_host    = ENV['REDIS_HOST']
-    config.goo_redis_port    = ENV['REDIS_PORT']
-    config.http_redis_host   = ENV['REDIS_HOST']
-    config.http_redis_port   = ENV['REDIS_PORT']
+    config.goo_backend_name = ENV['GOO_BACKEND_NAME']
+    config.goo_host = ENV['GOO_HOST']
+    config.goo_port = ENV['GOO_PORT'].to_i
+    config.goo_path_query = ENV['GOO_PATH_QUERY']
+    config.goo_path_data = ENV['GOO_PATH_DATA']
+    config.goo_path_update = ENV['GOO_PATH_UPDATE']
+    config.goo_redis_host = ENV['REDIS_HOST']
+    config.goo_redis_port = ENV['REDIS_PORT']
+    config.http_redis_host = ENV['REDIS_HOST']
+    config.http_redis_port = ENV['REDIS_PORT']
   end
 
   Annotator.config do |config|
     config.annotator_redis_host = ENV['ANNOTATOR_REDIS_HOST']
     config.annotator_redis_port = ENV['ANNOTATOR_REDIS_PORT']
-    config.mgrep_host           = ENV['MGREP_HOST']
-    config.mgrep_port           = ENV['MGREP_PORT']
+    config.mgrep_host = ENV['MGREP_HOST']
+    config.mgrep_port = ENV['MGREP_PORT']
   end
 end
 
@@ -86,17 +86,13 @@
   set :show_exceptions, false
 end
 
-
 use Rack::Cors do
   allow do
     origins '*'
     resource '*', :headers => :any, :methods => [:get, :post, :put, :patch, :delete, :options]
   end
 end
 
-
-
-
 # Show exceptions after timeout
 if LinkedData::OntologiesAPI.settings.enable_req_timeout
   use Rack::Timeout; Rack::Timeout.timeout = LinkedData::OntologiesAPI.settings.req_timeout # seconds, shorter than unicorn timeout
@@ -123,28 +119,49 @@
   redis_host_port = "#{LinkedData::OntologiesAPI.settings.http_redis_host}:#{LinkedData::OntologiesAPI.settings.http_redis_port}"
   verbose = environment == :development
   use Rack::Cache,
-    verbose: verbose,
-    allow_reload: true,
-    metastore: "redis://#{redis_host_port}/0/metastore",
-    entitystore: "redis://#{redis_host_port}/0/entitystore"
+      verbose: verbose,
+      allow_reload: true,
+      metastore: "redis://#{redis_host_port}/0/metastore",
+      entitystore: "redis://#{redis_host_port}/0/entitystore"
 end
 
-# Initialize unicorn Worker killer to mitigate unicorn worker memory bloat
-if LinkedData::OntologiesAPI.settings.enable_unicorn_workerkiller
-  require 'unicorn'
-  require_relative 'config/unicorn_workerkiller'
-end
+# # Initialize unicorn Worker killer to mitigate unicorn worker memory bloat
+# if LinkedData::OntologiesAPI.settings.enable_unicorn_workerkiller
+#   require 'unicorn'
+#   require_relative 'config/unicorn_workerkiller'
+# end
 
 # Add New Relic last to allow Rack middleware instrumentation
 require 'newrelic_rpm'
 
 # Initialize the app
 require_relative 'init'
 
+require_relative 'lib/api_monitor'
+configure do
+  set :monitor, APIMonitor.new(pool_size: ENV.fetch('REDIS_POOL_SIZE', 5).to_i)
+end
+
+require "sinatra/reloader"
+
+configure :development do
+  register Sinatra::Reloader
+end
+
+require 'sinatra/reloader' if development?
+require 'listen'
+set :bind, '0.0.0.0'
+set :port, 3000
+
 # Enter console mode
 if settings.environment == :console
   require 'rack/test'
-  include Rack::Test::Methods; def app() Sinatra::Application end
+  include Rack::Test::Methods;
+
+  def app()
+    Sinatra::Application
+  end
+
   Pry.start binding, :quiet => true
   exit
 end

bin/ontoportal

@@ -218,7 +218,7 @@ run_command() {
 dev() {
   echo "Starting OntoPortal API development server..."
 
-  local custom_command="bundle exec shotgun --host 0.0.0.0  --env=development --port 9393"
+  local custom_command="bundle exec rackup config.ru"
   run_command "$custom_command" "$@"
 }