Merge branch 'master' into java-dependency

Author: tylerjl

CHANGELOG.md

@@ -3,15 +3,18 @@
 ### Summary
 
 #### Features
+* Added 'file_rolling_type' parameter to allow selecting file logging rotation type between "dailyRollingFile" or "rollingFile". Also added 'daily_rolling_date_pattern', 'rolling_file_max_backup_index' and 'rolling_file_max_file_size' for file rolling customization.
 
 #### Bugfixes
 * Permissions on the Elasticsearch plugin directory have been fixed to permit world read rights.
 * The service systemd unit now `Wants=` a network target to fix bootup parallelization problems.
 * Recursively create the logdir for elasticsearch when creating multiple instances
 * Files and directories with root ownership now specify UID/GID 0 instead to improve compatability with *BSDs.
+* Elasticsearch Debian init file changed to avoid throwing errors when DATA_DIR, WORK_DIR and/or LOG_DIR were an empty variable.
 
 #### Changes
 * The `api_ca_file` and `api_ca_path` parameters have been added to support custom CA bundles for API access.
+* Numerics in elasticsearch.yml will always be properly unquoted.
 * puppetlabs/java is now listed as a dependency in metadata.json to avoid unexpected installation problems.
 
 #### Testing changes

CONTRIBUTORS

@@ -28,3 +28,4 @@ Philip Wigg (philipwigg)
 Ian Bissett (bisscuitt)
 Fabian M. Krack (onibox)
 Rhommel Lamas (rhoml)
+Jose Luis Ledesma (sp-joseluis-ledesma)

manifests/init.pp

@@ -298,6 +298,27 @@
 #   Value type is string
 #   Default value: undef
 #
+# [*file_rolling_type*]
+#   Configuration for the file appender rotation. It can be 'dailyRollingFile'
+#   or 'rollingFile'. The first rotates by name, and the second one by size.
+#   Value type is string
+#   Default value: dailyRollingFile
+#
+# [*daily_rolling_date_pattern*]
+#   File pattern for the file appender log when file_rolling_type is 'dailyRollingFile'
+#   Value type is string
+#   Default value: "'.'yyyy-MM-dd"
+#
+# [*rolling_file_max_backup_index*]
+#   Max number of logs to store whern file_rolling_type is 'rollingFile'
+#   Value type is integer
+#   Default value: 1
+#
+# [*rolling_file_max_file_size*]
+#   Max log file size when file_rolling_type is 'rollingFile'
+#   Value type is string
+#   Default value: 10MB
+#
 # The default values for the parameters are set in elasticsearch::params. Have
 # a look at the corresponding <tt>params.pp</tt> manifest file if you need more
 # technical information about them.
@@ -323,66 +344,70 @@
 # * Richard Pijnenburg <mailto:[email protected]>
 #
 class elasticsearch(
-  $ensure                  = $elasticsearch::params::ensure,
-  $status                  = $elasticsearch::params::status,
-  $restart_on_change       = $elasticsearch::params::restart_on_change,
-  $restart_config_change   = $elasticsearch::restart_on_change,
-  $restart_package_change  = $elasticsearch::restart_on_change,
-  $restart_plugin_change   = $elasticsearch::restart_on_change,
-  $autoupgrade             = $elasticsearch::params::autoupgrade,
-  $version                 = false,
-  $package_provider        = 'package',
-  $package_url             = undef,
-  $package_dir             = $elasticsearch::params::package_dir,
-  $package_name            = $elasticsearch::params::package,
-  $package_pin             = true,
-  $purge_package_dir       = $elasticsearch::params::purge_package_dir,
-  $package_dl_timeout      = $elasticsearch::params::package_dl_timeout,
-  $proxy_url               = undef,
-  $elasticsearch_user      = $elasticsearch::params::elasticsearch_user,
-  $elasticsearch_group     = $elasticsearch::params::elasticsearch_group,
-  $configdir               = $elasticsearch::params::configdir,
-  $purge_configdir         = $elasticsearch::params::purge_configdir,
-  $service_provider        = 'init',
-  $init_defaults           = undef,
-  $init_defaults_file      = undef,
-  $init_template           = "${module_name}/etc/init.d/${elasticsearch::params::init_template}",
-  $config                  = undef,
-  $config_hiera_merge      = false,
-  $datadir                 = $elasticsearch::params::datadir,
-  $logdir                  = $elasticsearch::params::logdir,
-  $plugindir               = $elasticsearch::params::plugindir,
-  $java_install            = false,
-  $java_package            = undef,
-  $manage_repo             = false,
-  $repo_version            = undef,
-  $repo_priority           = undef,
-  $repo_key_id             = '46095ACC8548582C1A2699A9D27D666CD88E42B4',
-  $repo_key_source         = 'http://packages.elastic.co/GPG-KEY-elasticsearch',
-  $repo_proxy              = undef,
-  $logging_file            = undef,
-  $logging_config          = undef,
-  $logging_template        = undef,
-  $default_logging_level   = $elasticsearch::params::default_logging_level,
-  $repo_stage              = false,
-  $instances               = undef,
-  $instances_hiera_merge   = false,
-  $plugins                 = undef,
-  $plugins_hiera_merge     = false,
-  $use_ssl                 = undef,
-  $validate_ssl            = undef,
-  $ssl_user                = undef,
-  $ssl_password            = undef,
-  $api_protocol            = 'http',
-  $api_host                = 'localhost',
-  $api_port                = 9200,
-  $api_timeout             = 10,
-  $api_basic_auth_username = undef,
-  $api_basic_auth_password = undef,
-  $api_ca_file             = undef,
-  $api_ca_path             = undef,
-  $validate_tls            = true,
-  $system_key              = undef,
+  $ensure                         = $elasticsearch::params::ensure,
+  $status                         = $elasticsearch::params::status,
+  $restart_on_change              = $elasticsearch::params::restart_on_change,
+  $restart_config_change          = $elasticsearch::restart_on_change,
+  $restart_package_change         = $elasticsearch::restart_on_change,
+  $restart_plugin_change          = $elasticsearch::restart_on_change,
+  $autoupgrade                    = $elasticsearch::params::autoupgrade,
+  $version                        = false,
+  $package_provider               = 'package',
+  $package_url                    = undef,
+  $package_dir                    = $elasticsearch::params::package_dir,
+  $package_name                   = $elasticsearch::params::package,
+  $package_pin                    = true,
+  $purge_package_dir              = $elasticsearch::params::purge_package_dir,
+  $package_dl_timeout             = $elasticsearch::params::package_dl_timeout,
+  $proxy_url                      = undef,
+  $elasticsearch_user             = $elasticsearch::params::elasticsearch_user,
+  $elasticsearch_group            = $elasticsearch::params::elasticsearch_group,
+  $configdir                      = $elasticsearch::params::configdir,
+  $purge_configdir                = $elasticsearch::params::purge_configdir,
+  $service_provider               = 'init',
+  $init_defaults                  = undef,
+  $init_defaults_file             = undef,
+  $init_template                  = "${module_name}/etc/init.d/${elasticsearch::params::init_template}",
+  $config                         = undef,
+  $config_hiera_merge             = false,
+  $datadir                        = $elasticsearch::params::datadir,
+  $logdir                         = $elasticsearch::params::logdir,
+  $plugindir                      = $elasticsearch::params::plugindir,
+  $java_install                   = false,
+  $java_package                   = undef,
+  $manage_repo                    = false,
+  $repo_version                   = undef,
+  $repo_priority                  = undef,
+  $repo_key_id                    = '46095ACC8548582C1A2699A9D27D666CD88E42B4',
+  $repo_key_source                = 'http://packages.elastic.co/GPG-KEY-elasticsearch',
+  $repo_proxy                     = undef,
+  $logging_file                   = undef,
+  $logging_config                 = undef,
+  $logging_template               = undef,
+  $default_logging_level          = $elasticsearch::params::default_logging_level,
+  $repo_stage                     = false,
+  $instances                      = undef,
+  $instances_hiera_merge          = false,
+  $plugins                        = undef,
+  $plugins_hiera_merge            = false,
+  $use_ssl                        = undef,
+  $validate_ssl                   = undef,
+  $ssl_user                       = undef,
+  $ssl_password                   = undef,
+  $api_protocol                   = 'http',
+  $api_host                       = 'localhost',
+  $api_port                       = 9200,
+  $api_timeout                    = 10,
+  $api_basic_auth_username        = undef,
+  $api_basic_auth_password        = undef,
+  $api_ca_file                    = undef,
+  $api_ca_path                    = undef,
+  $validate_tls                   = true,
+  $system_key                     = undef,
+  $file_rolling_type              = $elasticsearch::params::file_rolling_type,
+  $daily_rolling_date_pattern     = $elasticsearch::params::daily_rolling_date_pattern,
+  $rolling_file_max_backup_index  = $elasticsearch::params::rolling_file_max_backup_index,
+  $rolling_file_max_file_size     = $elasticsearch::params::rolling_file_max_file_size,
 ) inherits elasticsearch::params {
 
   anchor {'elasticsearch::begin': }
@@ -403,6 +428,15 @@
     fail("\"${status}\" is not a valid status parameter value")
   }
 
+  if ! ($file_rolling_type in [ 'dailyRollingFile', 'rollingFile']) {
+    file("\"${file_rolling_type}\" is not a valid type")
+  }
+
+  validate_integer($rolling_file_max_backup_index)
+  validate_string($daily_rolling_date_pattern)
+  validate_string($rolling_file_max_file_size)
+
+
   # restart on change
   validate_bool(
     $restart_on_change,

manifests/instance.pp

@@ -111,33 +111,58 @@
 #   Value type is string
 #   Default value: undef
 #
+# [*file_rolling_type*]
+#   Configuration for the file appender rotation. It can be 'dailyRollingFile'
+#   or 'rollingFile'. The first rotates by name, and the second one by size.
+#   Value type is string
+#   Default value: dailyRollingFile
+#
+# [*daily_rolling_date_pattern*]
+#   File pattern for the file appender log when file_rolling_type is 'dailyRollingFile'
+#   Value type is string
+#   Default value: "'.'yyyy-MM-dd"
+#
+# [*rolling_file_max_backup_index*]
+#   Max number of logs to store whern file_rolling_type is 'rollingFile'
+#   Value type is integer
+#   Default value: 1
+#
+# [*rolling_file_max_file_size*]
+#   Max log file size when file_rolling_type is 'rollingFile'
+#   Value type is string
+#   Default value: 10MB
+#
 # === Authors
 #
 # * Tyler Langlois <mailto:[email protected]>
 # * Richard Pijnenburg <mailto:[email protected]>
 #
 define elasticsearch::instance(
-  $ensure             = $elasticsearch::ensure,
-  $status             = $elasticsearch::status,
-  $config             = undef,
-  $configdir          = undef,
-  $datadir            = undef,
-  $logdir             = undef,
-  $logging_file       = undef,
-  $logging_config     = undef,
-  $logging_template   = undef,
-  $logging_level      = $elasticsearch::default_logging_level,
-  $service_flags      = undef,
-  $init_defaults      = undef,
-  $init_defaults_file = undef,
-  $init_template      = $elasticsearch::init_template,
-  $ssl                = false,
-  $ca_certificate     = undef,
-  $certificate        = undef,
-  $private_key        = undef,
-  $keystore_password  = undef,
-  $keystore_path      = undef,
-  $system_key         = $elasticsearch::system_key,
+  $ensure                        = $elasticsearch::ensure,
+  $status                        = $elasticsearch::status,
+  $config                        = undef,
+  $configdir                     = undef,
+  $datadir                       = undef,
+  $logdir                        = undef,
+  $logging_file                  = undef,
+  $logging_config                = undef,
+  $logging_template              = undef,
+  $logging_level                 = $elasticsearch::default_logging_level,
+  $service_flags                 = undef,
+  $init_defaults                 = undef,
+  $init_defaults_file            = undef,
+  $init_template                 = $elasticsearch::init_template,
+  $ssl                           = false,
+  $ca_certificate                = undef,
+  $certificate                   = undef,
+  $private_key                   = undef,
+  $keystore_password             = undef,
+  $keystore_path                 = undef,
+  $system_key                    = $elasticsearch::system_key,
+  $file_rolling_type             = $elasticsearch::file_rolling_type,
+  $daily_rolling_date_pattern    = $elasticsearch::daily_rolling_date_pattern,
+  $rolling_file_max_backup_index = $elasticsearch::rolling_file_max_backup_index,
+  $rolling_file_max_file_size    = $elasticsearch::rolling_file_max_file_size,
 ) {
 
   require elasticsearch::params

manifests/params.pp

@@ -60,6 +60,14 @@
     'index.indexing.slowlog' => 'TRACE, index_indexing_slow_log_file',
   }
 
+  $file_rolling_type = 'dailyRollingFile'
+
+  $daily_rolling_date_pattern = '"\'.\'yyyy-MM-dd"'
+
+  $rolling_file_max_backup_index = 1
+
+  $rolling_file_max_file_size ='10MB'
+
   #### Internal module values
 
   # User and Group for the files and user to run the service as.

spec/classes/000_elasticsearch_init_spec.rb

@@ -397,6 +397,31 @@
         it { should contain_file('/var/run/elasticsearch').with(:owner => 'myesuser') } if facts[:osfamily] == 'RedHat'
       end
 
+      context "rollingFile appender" do
+
+        let (:params) {
+          default_params.merge({
+            :file_rolling_type => 'rollingFile',
+            :rolling_file_max_backup_index => 10,
+            :rolling_file_max_file_size => '100MB',
+            :instances => { 'test' => {} }
+          })
+        }
+
+        it { should contain_file('/etc/elasticsearch/test/logging.yml').with_content(/type: rollingFile/, /maxBackupIndex: 10/, /maxBackupIndex: 10/, /maxFileSize: 100MB/) }
+      end
+
+      context "Default file appender" do
+
+        let (:params) {
+          default_params.merge({
+            :instances => { 'test' => {} }
+          })
+        }
+
+        it { should contain_file('/etc/elasticsearch/test/logging.yml').with_content(/type: dailyRollingFile/, /datePattern: "'.'yyyy-MM-dd"/) }
+      end
+
     end
 
   end

spec/templates/001_elasticsearch.yml.erb_spec.rb

@@ -74,4 +74,16 @@ def unindent
       }.config))
   end
 
+  it 'should not quote numeric values' do
+    harness.set(
+      '@data', {
+        'some.setting' => '10'
+      }
+    )
+
+    expect( YAML.load(harness.run) ).to eq( YAML.load(%q{
+      some.setting: 10
+    }.config))
+  end
+
 end

templates/etc/elasticsearch/elasticsearch.yml.erb

@@ -1,14 +1,19 @@
 ### MANAGED BY PUPPET ###
 <%-
   $LOAD_PATH.unshift(File.join(File.dirname(__FILE__),"..","..","..","lib"))
+  require 'puppet_x/elastic/deep_to_i'
   require 'puppet_x/elastic/hash'
 
   @yml_string = ''
 
   if [email protected]?
 
     # Sort Hash and transform it into yaml
-    @yml_string += @data.extend(Puppet_X::Elastic::SortedHash).to_yaml
+    @yml_string += Puppet_X::Elastic::deep_to_i(
+      @data
+    ).extend(
+      Puppet_X::Elastic::SortedHash
+    ).to_yaml
 
     # Puppet < 4 uses ZAML, which has some deviations from Puppet 4 YAML
     # implementation

templates/etc/elasticsearch/logging.yml.erb

@@ -28,9 +28,14 @@ appender:
       conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
 
   file:
-    type: dailyRollingFile
+    type: <%= @file_rolling_type %>
     file: ${path.logs}/${cluster.name}.log
-    datePattern: "'.'yyyy-MM-dd"
+    <%- if @file_rolling_type == 'dailyRollingFile' -%>
+    datePattern: <%= @daily_rolling_date_pattern %>
+    <%- elsif @file_rolling_type == 'rollingFile' -%>
+    maxBackupIndex: <%= @rolling_file_max_backup_index %>
+    maxFileSize: <%= @rolling_file_max_file_size %>
+    <%- end -%>
     layout:
       type: pattern
       conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"

templates/etc/init.d/elasticsearch.Debian.erb

@@ -149,7 +149,9 @@ case "$1" in
 	fi
 
 	# Prepare environment
-	mkdir -p "$LOG_DIR" "$DATA_DIR" "$WORK_DIR" && chown "$ES_USER":"$ES_GROUP" "$LOG_DIR" "$DATA_DIR" "$WORK_DIR"
+	for DIR in "$DATA_DIR" "$LOG_DIR" "$WORK_DIR"; do
+		[ ! -z "$DIR" ] && mkdir -p "$DIR" && chown "$ES_USER":"$ES_GROUP" "$DIR"
+	done
 	touch "$PID_FILE" && chown "$ES_USER":"$ES_GROUP" "$PID_FILE"
 
 	if [ -n "$MAX_OPEN_FILES" ]; then