diff --git a/tests/logstash_7.6.2.conf b/tests/logstash_7.6.2.conf index 9a13c9d..92e1c00 100644 --- a/tests/logstash_7.6.2.conf +++ b/tests/logstash_7.6.2.conf @@ -2,9 +2,9 @@ input { stdin { } # logstash stops when input is closed } -filter { +filter +{ grok { - add_field => { "received_at" => "%{@timestamp}" "received_from" => "%{@source_host}" @@ -15,9 +15,36 @@ filter { patterns_dir => [ "/tmp/logpatterns-groktest" ] match => { - # RSYSLOGCUSTOM always last (and no PREFIX)! - "message" => ["%{RSYSLOGCUSTOMHEADER}%{PIXIU_ALARM}", "%{RSYSLOGCUSTOMHEADER}%{PIXIU_BASH}", "%{RSYSLOGCUSTOMHEADER}%{PIXIU_NGINX}", "%{RSYSLOGCUSTOMHEADER}%{PIXIU_TOMCAT}", "%{RSYSLOGPREFIX}%{PIXIU_APACHE}", "%{RSYSLOGPREFIX}%{SU_MSG}", "%{RSYSLOGPREFIX}%{SUDO_MSG}", "%{RSYSLOGPREFIX}%{REFRAME_MSG}", "%{RSYSLOGPREFIX}%{BASH_MSG}", "%{RSYSLOGPREFIX}%{FAIL2BAN_MSG}", "%{RSYSLOGPREFIX}%{SINGULARITY_MSG}", "%{RSYSLOGPREFIX}%{DHCPD_MSG}", "%{RSYSLOGPREFIX}%{SSH_MSG}", "%{RSYSLOGPREFIX}%{MODULECMD_MSG}", "%{RSYSLOGPREFIX}%{LMOD_MSG}", "%{RSYSLOGPREFIX}%{NFS_MSG}", "%{RSYSLOGPREFIX}%{CEPH_MSG}", "%{RSYSLOGPREFIX}%{OPENNEBULA_MSG}", "%{RSYSLOGPREFIX}%{JUBE_MSG}", "%{RSYSLOGPREFIX}%{SHOREWALL_MSG}", "%{RSYSLOGPREFIX}%{KEYVALUE_MSG}", "%{RSYSLOGPREFIX}%{QUATTOR_MSG}", "%{RSYSLOGPREFIX}%{SNOOPY_MSG}", "%{RSYSLOGPREFIX}%{APACHE_MSG}", "%{RSYSLOGCUSTOM}"] + "message" => [ + "%{RSYSLOGCUSTOMHEADER}%{PIXIU_ALARM}", + "%{RSYSLOGCUSTOMHEADER}%{PIXIU_BASH}", + "%{RSYSLOGCUSTOMHEADER}%{PIXIU_NGINX}", + "%{RSYSLOGCUSTOMHEADER}%{PIXIU_TOMCAT}", + "%{RSYSLOGPREFIX}%{PIXIU_APACHE}", + "%{RSYSLOGPREFIX}%{SU_MSG}", + "%{RSYSLOGPREFIX}%{SUDO_MSG}", + "%{RSYSLOGPREFIX}%{REFRAME_MSG}", + "%{RSYSLOGPREFIX}%{BASH_MSG}", + "%{RSYSLOGPREFIX}%{FAIL2BAN_MSG}", + "%{RSYSLOGPREFIX}%{SINGULARITY_MSG}", + "%{RSYSLOGPREFIX}%{DHCPD_MSG}", + "%{RSYSLOGPREFIX}%{SSH_MSG}", + "%{RSYSLOGPREFIX}%{MODULECMD_MSG}", + "%{RSYSLOGPREFIX}%{LMOD_MSG}", + "%{RSYSLOGPREFIX}%{NFS_MSG}", + "%{RSYSLOGPREFIX}%{CEPH_MSG}", + "%{RSYSLOGPREFIX}%{OPENNEBULA_MSG}", + "%{RSYSLOGPREFIX}%{JUBE_MSG}", + "%{RSYSLOGPREFIX}%{SHOREWALL_MSG}", + "%{RSYSLOGPREFIX}%{KEYVALUE_MSG}", + "%{RSYSLOGPREFIX}%{QUATTOR_MSG}", + "%{RSYSLOGPREFIX}%{SNOOPY_MSG}", + "%{RSYSLOGPREFIX}%{APACHE_MSG}", + # RSYSLOGCUSTOM always last (and no PREFIX)! + "%{RSYSLOGCUSTOM}" + ] } + id => "main_grok" } kv { @@ -32,28 +59,26 @@ filter { mutate { replace => { "@source_host" => "%{syslog_hostname}" - "@message" => "%{syslog_message}" } } } - if "_grokparsefailure" not in [tags] { - mutate { - # in the future remove the field 'message' too - remove_field => [ "syslog_hostname", "syslog_message", "syslog_timestamp", "KEY_EQ_VALUEDATA", "int" ] - convert => { "success" => "boolean" } - - # we need MB converted to MiB for bytes2human - gsub => [ "used_space", "([0-9]+[kKmMgGtTpP])[bB]", "\1ib", - "quota_space_limit", "([0-9]+[kKmMgGtTpP])[bB]", "\1ib" ] - } + if ([syslog_message]) { + mutate { + replace => { + "@message" => "%{syslog_message}" + } + } } - if [jube_id] { - if ("_grokparsefailure" not in [tags]) { - mutate { - convert => { "success" => "boolean" } - } + if ("_grokparsefailure" not in [tags]) { + mutate { + remove_field => [ "syslog_hostname", "syslog_message", "syslog_timestamp", "KEY_EQ_VALUEDATA", "int" ] + convert => { "success" => "boolean" } + + # we need MB converted to MiB for bytes2human + gsub => [ "used_space", "([0-9]+[kKmMgGtTpP])[bB]", "\1ib", + "quota_space_limit", "([0-9]+[kKmMgGtTpP])[bB]", "\1ib" ] } }