Standardize and add SSL settings (#1118)

This commit made the plugin SSL settings consistent with the naming convention defined in the meta issue: elastic/logstash#14905.

It added the following SSL settings:
ssl_truststore_type: The format of the truststore file
ssl_keystore_type: The format of the keystore file
ssl_certificate: OpenSSL-style X.509 certificate file to authenticate the client
ssl_key: OpenSSL-style RSA private key that corresponds to the ssl_certificate
ssl_cipher_suites: The list of cipher suites

And deprecated:
ssl in favor of ssl_enabled
cacert in favor of ssl_certificate_authorities
keystore in favor of ssl_keystore_path
keystore_password in favor of ssl_keystore_password
truststore in favor of ssl_truststore_path
truststore_password in favor of ssl_truststore_password
ssl_certificate_verification in favor of ssl_verification_mode

Author: edmocosta

CHANGELOG.md

@@ -1,3 +1,19 @@
+## 11.14.0
+ - Added SSL settings for: [#1115](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1115)
+   - `ssl_truststore_type`: The format of the truststore file
+   - `ssl_keystore_type`: The format of the keystore file
+   - `ssl_certificate`: OpenSSL-style X.509 certificate file to authenticate the client
+   - `ssl_key`: OpenSSL-style RSA private key that corresponds to the `ssl_certificate`
+   - `ssl_cipher_suites`: The list of cipher suites
+ - Reviewed and deprecated SSL settings to comply with Logstash's naming convention
+   - Deprecated `ssl` in favor of `ssl_enabled`
+   - Deprecated `cacert` in favor of `ssl_certificate_authorities`
+   - Deprecated `keystore` in favor of `ssl_keystore_path`
+   - Deprecated `keystore_password` in favor of `ssl_keystore_password`
+   - Deprecated `truststore` in favor of `ssl_truststore_path`
+   - Deprecated `truststore_password` in favor of `ssl_truststore_password`
+   - Deprecated `ssl_certificate_verification` in favor of `ssl_verification_mode`
+
 ## 11.13.1
  - Avoid crash by ensuring ILM settings are injected in the correct location depending on the default (or custom) template format, template_api setting and ES version [#1102](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1102)
 

docs/index.asciidoc

@@ -96,10 +96,14 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   require "logstash/outputs/elasticsearch/data_stream_support"
   require 'logstash/plugin_mixins/ecs_compatibility_support'
   require 'logstash/plugin_mixins/deprecation_logger_support'
+  require 'logstash/plugin_mixins/normalize_config_support'
 
   # Protocol agnostic methods
   include(LogStash::PluginMixins::ElasticSearch::Common)
 
+  # Config normalization helpers
+  include(LogStash::PluginMixins::NormalizeConfigSupport)
+
   # Methods for ILM support
   include(LogStash::Outputs::ElasticSearch::Ilm)
 
@@ -282,6 +286,8 @@ def initialize(*params)
   end
 
   def register
+    setup_ssl_params!
+
     if !failure_type_logging_whitelist.empty?
       log_message = "'failure_type_logging_whitelist' is deprecated and in a future version of Elasticsearch " +
         "output plugin will be removed, please use 'silence_errors_in_log' instead."
@@ -622,6 +628,52 @@ def setup_template_manager_defaults(data_stream_enabled)
     end
   end
 
+  def setup_ssl_params!
+    @ssl_enabled = normalize_config(:ssl_enabled) do |normalize|
+      normalize.with_deprecated_alias(:ssl)
+    end
+
+    @ssl_certificate_authorities = normalize_config(:ssl_certificate_authorities) do |normalize|
+      normalize.with_deprecated_mapping(:cacert) do |cacert|
+        [cacert]
+      end
+    end
+
+    @ssl_keystore_path =  normalize_config(:ssl_keystore_path) do |normalize|
+      normalize.with_deprecated_alias(:keystore)
+    end
+
+    @ssl_keystore_password = normalize_config(:ssl_keystore_password) do |normalize|
+      normalize.with_deprecated_alias(:keystore_password)
+    end
+
+    @ssl_truststore_path = normalize_config(:ssl_truststore_path) do |normalize|
+      normalize.with_deprecated_alias(:truststore)
+    end
+
+    @ssl_truststore_password =  normalize_config(:ssl_truststore_password) do |normalize|
+      normalize.with_deprecated_alias(:truststore_password)
+    end
+
+    @ssl_verification_mode = normalize_config(:ssl_verification_mode) do |normalize|
+      normalize.with_deprecated_mapping(:ssl_certificate_verification) do |ssl_certificate_verification|
+        if ssl_certificate_verification == true
+          "full"
+        else
+          "none"
+        end
+      end
+    end
+
+    params['ssl_enabled'] = @ssl_enabled unless @ssl_enabled.nil?
+    params['ssl_certificate_authorities'] = @ssl_certificate_authorities unless @ssl_certificate_authorities.nil?
+    params['ssl_keystore_path'] = @ssl_keystore_path unless @ssl_keystore_path.nil?
+    params['ssl_keystore_password'] = @ssl_keystore_password unless @ssl_keystore_password.nil?
+    params['ssl_truststore_path'] = @ssl_truststore_path unless @ssl_truststore_path.nil?
+    params['ssl_truststore_password'] = @ssl_truststore_password unless @ssl_truststore_password.nil?
+    params['ssl_verification_mode'] = @ssl_verification_mode unless @ssl_verification_mode.nil?
+  end
+
   # To be overidden by the -java version
   VALID_HTTP_ACTIONS = ["index", "delete", "create", "update"]
   def valid_actions

lib/logstash/outputs/elasticsearch.rb

@@ -107,38 +107,53 @@ def self.create_http_client(options)
     end
 
     def self.setup_ssl(logger, params)
-      params["ssl"] = true if params["hosts"].any? {|h| h.scheme == "https" }
-      return {} if params["ssl"].nil?
+      params["ssl_enabled"] = true if params["hosts"].any? {|h| h.scheme == "https" }
+      return {} if params["ssl_enabled"].nil?
 
-      return {:ssl => {:enabled => false}} if params["ssl"] == false
+      return {:ssl => {:enabled => false}} if params["ssl_enabled"] == false
 
-      cacert, truststore, truststore_password, keystore, keystore_password =
-        params.values_at('cacert', 'truststore', 'truststore_password', 'keystore', 'keystore_password')
+      ssl_certificate_authorities, ssl_truststore_path, ssl_certificate, ssl_keystore_path = params.values_at('ssl_certificate_authorities', 'ssl_truststore_path', 'ssl_certificate', 'ssl_keystore_path')
 
-      if cacert && truststore
-        raise(LogStash::ConfigurationError, "Use either \"cacert\" or \"truststore\" when configuring the CA certificate") if truststore
+      if ssl_certificate_authorities && ssl_truststore_path
+        raise LogStash::ConfigurationError, 'Use either "ssl_certificate_authorities/cacert" or "ssl_truststore_path/truststore" when configuring the CA certificate'
+      end
+
+      if ssl_certificate && ssl_keystore_path
+        raise LogStash::ConfigurationError, 'Use either "ssl_certificate" or "ssl_keystore_path/keystore" when configuring client certificates'
       end
 
       ssl_options = {:enabled => true}
 
-      if cacert
-        ssl_options[:ca_file] = cacert
-      elsif truststore
-        ssl_options[:truststore_password] = truststore_password.value if truststore_password
+      if ssl_certificate_authorities&.any?
+        raise LogStash::ConfigurationError, 'Multiple values on "ssl_certificate_authorities" are not supported by this plugin' if ssl_certificate_authorities.size > 1
+        ssl_options[:ca_file] = ssl_certificate_authorities.first
       end
 
-      ssl_options[:truststore] = truststore if truststore
-      if keystore
-        ssl_options[:keystore] = keystore
-        ssl_options[:keystore_password] = keystore_password.value if keystore_password
+      setup_ssl_store(ssl_options, 'truststore', params)
+      setup_ssl_store(ssl_options, 'keystore', params)
+
+      ssl_key = params["ssl_key"]
+      if ssl_certificate
+        raise LogStash::ConfigurationError, 'Using an "ssl_certificate" requires an "ssl_key"' unless ssl_key
+        ssl_options[:client_cert] = ssl_certificate
+        ssl_options[:client_key] = ssl_key
+      elsif !ssl_key.nil?
+        raise LogStash::ConfigurationError, 'An "ssl_certificate" is required when using an "ssl_key"'
       end
 
-      if !params["ssl_certificate_verification"]
-        logger.warn "You have enabled encryption but DISABLED certificate verification, " +
-                    "to make sure your data is secure remove `ssl_certificate_verification => false`"
-        ssl_options[:verify] = :disable # false accepts self-signed but still validates hostname
+      ssl_verification_mode = params["ssl_verification_mode"]
+      unless ssl_verification_mode.nil?
+        case ssl_verification_mode
+          when 'none'
+            logger.warn "You have enabled encryption but DISABLED certificate verification, " +
+                          "to make sure your data is secure set `ssl_verification_mode => full`"
+            ssl_options[:verify] = :disable
+          else
+            ssl_options[:verify] = :strict
+        end
       end
 
+      ssl_options[:cipher_suites] = params["ssl_cipher_suites"] if params.include?("ssl_cipher_suites")
       ssl_options[:trust_strategy] = params["ssl_trust_strategy"] if params.include?("ssl_trust_strategy")
 
       protocols = params['ssl_supported_protocols']
@@ -147,6 +162,16 @@ def self.setup_ssl(logger, params)
       { ssl: ssl_options }
     end
 
+    # @param kind is a string [truststore|keystore]
+    def self.setup_ssl_store(ssl_options, kind, params)
+      store_path = params["ssl_#{kind}_path"]
+      if store_path
+        ssl_options[kind.to_sym] = store_path
+        ssl_options["#{kind}_type".to_sym] = params["ssl_#{kind}_type"] if params.include?("ssl_#{kind}_type")
+        ssl_options["#{kind}_password".to_sym] = params["ssl_#{kind}_password"].value if params.include?("ssl_#{kind}_password")
+      end
+    end
+
     def self.setup_basic_auth(logger, params)
       user, password = params["user"], params["password"]
 

lib/logstash/outputs/elasticsearch/http_client_builder.rb

@@ -45,35 +45,79 @@ module APIConfigs
         # Enable SSL/TLS secured communication to Elasticsearch cluster. Leaving this unspecified will use whatever scheme
         # is specified in the URLs listed in 'hosts'. If no explicit protocol is specified plain HTTP will be used.
         # If SSL is explicitly disabled here the plugin will refuse to start if an HTTPS URL is given in 'hosts'
-        :ssl => { :validate => :boolean },
+        :ssl => { :validate => :boolean, :deprecated => "Set 'ssl_enabled' instead." },
+
+        # Enable SSL/TLS secured communication to Elasticsearch cluster. Leaving this unspecified will use whatever scheme
+        # is specified in the URLs listed in 'hosts'. If no explicit protocol is specified plain HTTP will be used.
+        # If SSL is explicitly disabled here the plugin will refuse to start if an HTTPS URL is given in 'hosts'
+        :ssl_enabled => { :validate => :boolean },
 
         # Option to validate the server's certificate. Disabling this severely compromises security.
         # For more information on disabling certificate verification please read
         # https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
-        :ssl_certificate_verification => { :validate => :boolean, :default => true },
+        :ssl_certificate_verification => { :validate => :boolean, :default => true, :deprecated => "Set 'ssl_verification_mode' instead." },
+
+        # Options to verify the server's certificate.
+        # "full": validates that the provided certificate has an issue date that’s within the not_before and not_after dates;
+        # chains to a trusted Certificate Authority (CA); has a hostname or IP address that matches the names within the certificate.
+        # "none": performs no certificate validation. Disabling this severely compromises security (https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf)
+        :ssl_verification_mode => { :validate => %w[full none], :default => 'full' },
 
         # The .cer or .pem file to validate the server's certificate
-        :cacert => { :validate => :path },
+        :cacert => { :validate => :path, :deprecated => "Set 'ssl_certificate_authorities' instead." },
+
+        # The .cer or .pem files to validate the server's certificate
+        :ssl_certificate_authorities => { :validate => :path, :list => true },
 
         # One or more hex-encoded SHA256 fingerprints to trust as Certificate Authorities
         :ca_trusted_fingerprint => LogStash::PluginMixins::CATrustedFingerprintSupport,
 
         # The JKS truststore to validate the server's certificate.
         # Use either `:truststore` or `:cacert`
-        :truststore => { :validate => :path },
+        :truststore => { :validate => :path, :deprecated => "Set 'ssl_truststore_path' instead." },
+
+        # The JKS truststore to validate the server's certificate.
+        # Use either `:ssl_truststore_path` or `:ssl_certificate_authorities`
+        :ssl_truststore_path => { :validate => :path },
+
+        # The format of the truststore file. It must be either jks or pkcs12
+        :ssl_truststore_type => { :validate => %w[pkcs12 jks] },
+
+        # Set the truststore password
+        :truststore_password => { :validate => :password, :deprecated => "Use 'ssl_truststore_password' instead." },
 
         # Set the truststore password
-        :truststore_password => { :validate => :password },
+        :ssl_truststore_password => { :validate => :password },
 
         # The keystore used to present a certificate to the server.
         # It can be either .jks or .p12
-        :keystore => { :validate => :path },
+        :keystore => { :validate => :path, :deprecated => "Set 'ssl_keystore_path' instead." },
+
+        # The keystore used to present a certificate to the server.
+        # It can be either .jks or .p12
+        :ssl_keystore_path => { :validate => :path },
+
+        # The format of the keystore file. It must be either jks or pkcs12
+        :ssl_keystore_type => { :validate => %w[pkcs12 jks] },
 
         # Set the keystore password
-        :keystore_password => { :validate => :password },
+        :keystore_password => { :validate => :password, :deprecated => "Set 'ssl_keystore_password' instead." },
+
+        # Set the keystore password
+        :ssl_keystore_password => { :validate => :password },
 
         :ssl_supported_protocols => { :validate => ['TLSv1.1', 'TLSv1.2', 'TLSv1.3'], :default => [], :list => true },
 
+        # OpenSSL-style X.509 certificate certificate to authenticate the client
+        :ssl_certificate => { :validate => :path },
+
+        # OpenSSL-style RSA private key to authenticate the client
+        :ssl_key => { :validate => :path },
+
+        # The list of cipher suites to use, listed by priorities.
+        # Supported cipher suites vary depending on which version of Java is used.
+        :ssl_cipher_suites => { :validate => :string, :list => true },
+
         # This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list.
         # Note: This will return ALL nodes with HTTP enabled (including master nodes!). If you use
         # this with master nodes, you probably want to disable HTTP on them by setting

lib/logstash/plugin_mixins/elasticsearch/api_configs.rb

@@ -28,8 +28,7 @@ def build_client(license_checker = nil)
 
       setup_hosts
 
-
-      params['ssl'] = effectively_ssl? unless params.include?('ssl')
+      params['ssl_enabled'] = effectively_ssl? unless params.include?('ssl_enabled')
 
       # inject the TrustStrategy from CATrustedFingerprintSupport
       if trust_strategy_for_ca_trusted_fingerprint
@@ -74,7 +73,7 @@ def setup_hosts
     end
 
     def effectively_ssl?
-      return @ssl unless @ssl.nil?
+      return @ssl_enabled unless @ssl_enabled.nil?
 
       hosts = Array(@hosts)
       return false if hosts.nil? || hosts.empty?

lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.13.1'
+  s.version         = '11.14.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -26,6 +26,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0'
   s.add_runtime_dependency 'logstash-mixin-deprecation_logger_support', '~>1.0'
   s.add_runtime_dependency 'logstash-mixin-ca_trusted_fingerprint_support', '~>1.0'
+  s.add_runtime_dependency 'logstash-mixin-normalize_config_support', '~>1.0'
 
   s.add_development_dependency 'logstash-codec-plain'
   s.add_development_dependency 'logstash-devutils'

logstash-output-elasticsearch.gemspec

@@ -289,8 +289,8 @@ def curl_and_get_json_response(url, method: :get, retrieve_err_payload: false);
         "hosts" => [ get_host_port ],
         "user" => user,
         "password" => password,
-        "ssl" => true,
-        "cacert" => cacert,
+        "ssl_enabled" => true,
+        "ssl_certificate_authorities" => cacert,
         "index" => index
       }
     end 
@@ -302,7 +302,7 @@ def curl_and_get_json_response(url, method: :get, retrieve_err_payload: false);
 
       context "when no keystore nor ca cert set and verification is disabled" do
         let(:config) do
-          super().tap { |config| config.delete('cacert') }.merge('ssl_certificate_verification' => false)
+          super().tap { |config| config.delete('ssl_certificate_authorities') }.merge('ssl_verification_mode' => 'none')
         end
 
         include_examples("an indexer", true)
@@ -311,9 +311,9 @@ def curl_and_get_json_response(url, method: :get, retrieve_err_payload: false);
       context "when keystore is set and verification is disabled" do
         let(:config) do
           super().merge(
-              'ssl_certificate_verification' => false,
-              'keystore' => 'spec/fixtures/test_certs/test.p12',
-              'keystore_password' => '1234567890'
+              'ssl_verification_mode' => 'none',
+              'ssl_keystore_path' => 'spec/fixtures/test_certs/test.p12',
+              'ssl_keystore_password' => '1234567890'
           )
         end
 
@@ -322,10 +322,10 @@ def curl_and_get_json_response(url, method: :get, retrieve_err_payload: false);
 
       context "when keystore has self-signed cert and verification is disabled" do
         let(:config) do
-          super().tap { |config| config.delete('cacert') }.merge(
-              'ssl_certificate_verification' => false,
-              'keystore' => 'spec/fixtures/test_certs/test_self_signed.p12',
-              'keystore_password' => '1234567890'
+          super().tap { |config| config.delete('ssl_certificate_authorities') }.merge(
+              'ssl_verification_mode' => 'none',
+              'ssl_keystore_path' => 'spec/fixtures/test_certs/test_self_signed.p12',
+              'ssl_keystore_password' => '1234567890'
           )
         end
 
@@ -349,30 +349,30 @@ def curl_and_get_json_response(url, method: :get, retrieve_err_payload: false);
         let(:config) do
           {
               "hosts" => ["https://#{CGI.escape(user)}:#{CGI.escape(password)}@elasticsearch:9200"],
-              "ssl" => true,
-              "cacert" => "spec/fixtures/test_certs/test.crt",
+              "ssl_enabled" => true,
+              "ssl_certificate_authorities" => "spec/fixtures/test_certs/test.crt",
               "index" => index
           }
         end
 
         include_examples("an indexer", true)
       end
 
-      context "without providing `cacert`" do
+      context "without providing `ssl_certificate_authorities`" do
         let(:config) do
           super().tap do |c|
-            c.delete("cacert")
+            c.delete("ssl_certificate_authorities")
           end
         end
 
         it_behaves_like("PKIX path failure")
       end
 
       if Gem::Version.new(LOGSTASH_VERSION) >= Gem::Version.new("8.3.0")
-        context "with `ca_trusted_fingerprint` instead of `cacert`" do
+        context "with `ca_trusted_fingerprint` instead of `ssl_certificate_authorities`" do
           let(:config) do
             super().tap do |c|
-              c.delete("cacert")
+              c.delete("ssl_certificate_authorities")
               c.update("ca_trusted_fingerprint" => ca_trusted_fingerprint)
             end
           end