Merge pull request #17 from wpoely86/pixiu

Add patterns for pixiu (AB#14630)

Author: lexming

files/pixiu

@@ -0,0 +1,7 @@
+PIXIU_BASH \s*%{IPORHOST:syslog_hostname} \[%{PROG:program}\]: \[%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND} %{USERNAME} %{GREEDYDATA:command}\] return code=\[%{INT:return_code:int}\], execute (success|failed) by \[%{USERNAME:username}\(uid=%{INT:uid:int}\)\] from \[%{DATA:terminal} \(%{IPORHOST:source_ip}\)\]
+
+PIXIU_APACHE \[%{HTTPDATE:timestamp}\] %{IPORHOST:clientip} %{HTTPDUSER:auth} %{WORD:verb} %{NOTSPACE:request} %{GREEDYDATA:other}
+
+PIXIU_NGINX \s*%{IPORHOST:syslog_hostname} %{SYSLOGPROG} %{IPORHOST:clientip} - %{HTTPDUSER:auth}\s+\[%{HTTPDATE:timestamp}\] \"%{WORD:verb} %{NOTSPACE:request} HTTP/%{NUMBER:httpversion}\" %{NUMBER:response:int} %{GREEDYDATA:other}
+
+PIXIU_TOMCAT \s*%{IPORHOST:syslog_hostname} %{SYSLOGPROG} \[%{HTTPDATE:timestamp}\]\^%{IPORHOST:clientip}\^%{HTTPDUSER:auth}\^%{WORD:verb} %{NOTSPACE:request} HTTP/%{NUMBER:httpversion}\^%{GREEDYDATA:other}

tests/data/pixiu

@@ -0,0 +1,52 @@
+data = [
+    {
+        # LogFormat "%t %a %u %m %U %{User-Agent}i %{Referer}i %{X-Forwarded-For}i %{Content-Type}i %{Accept-Language}i %{Accept}i %{Accept-Encoding}i %>s %b %T" syslog
+        "raw": "<85>Oct 17 12:48:49 C4STO01-Node2 apache[3984033]: [17/Oct/2022:12:48:49 +0200] 127.0.0.1 - GET /api/v2/aa/current_session curl/7.69.1 - - - - */* - 200 201 0",
+        "expected": {
+            "@source_host": "C4STO01-Node2",
+            "program": "apache",
+            "clientip": "127.0.0.1",
+            "verb": "GET",
+            "request": "/api/v2/aa/current_session",
+        },
+    },
+    {
+#    log_format  main  '$remote_addr - $log_remote_user  [$time_local] "$log_request" $status "$request_time" '
+#                      '$body_bytes_sent "$log_http_referer" "$log_http_user_agent" "$log_http_x_forwarded_for" '
+#                      '$upstream_addr $upstream_status "$upstream_response_time" '
+#                      '"$log_x_auth_token" "$log_x_csrf_token" "$log_cookie_x_auth_token" "$x_real_ip" '
+#                      '"$log_content_type" "$log_http_accept_language" "$log_http_accept" "$log_http_accept_encoding" ';
+        "raw": '<174>Oct 17 12:50:42 HKSTO03-Node1 nginx 127.0.0.1 - -  [17/Oct/2022:12:50:42 +0200] "GET /dsware/service/cluster/storagepool/query*** HTTP/1.1" 200 "0.062" 924 "-" "Apache-HttpClient/5.1 (Java/1.8.0_322)" "-" 127.0.0.1:9527 200 "0.064" "********" "********" "********" "127.0.0.1" "application/json;charset=UTF-8" "-" "-" "gzip, x-gzip, deflate"',
+        "expected": {
+            "@source_host": "HKSTO03-Node1",
+            "program": "nginx",
+            "clientip": "127.0.0.1",
+            "verb": "GET",
+            "request": "/dsware/service/cluster/storagepool/query***",
+            "response": 200,
+        },
+    },
+    {
+        "raw": "<174>Oct 17 13:05:07 C4STO01-Node2 [/bin/bash]: [2022-10-17 13:05:07 root cat httpd.conf] return code=[0], execute success by [root(uid=0)] from [pts/0 (172.18.124.57)]",
+        "expected": {
+            "@source_host": "C4STO01-Node2",
+            "program": "/bin/bash",
+            "username": "root",
+            "command": "cat httpd.conf",
+            "return_code": 0,
+            "uid": 0,
+            "source_ip": "172.18.124.57",
+            "terminal": "pts/0",
+        },
+    },
+    {
+        "raw": "<166>Oct 19 09:08:25 C4STO01-Node2 tomcat [19/Oct/2022:09:08:17 +0200]^127.0.0.1^-^GET /dsware/service/noAuth/managerStatus HTTP/1.1^curl/7.69.1^-^-^application/json^-^*/*^-^200^36^0.002^",
+        "expected": {
+            "@source_host": "C4STO01-Node2",
+            "program": "tomcat",
+            "clientip": "127.0.0.1",
+            "verb": "GET",
+            "request": "/dsware/service/noAuth/managerStatus",
+        },
+    },
+]

tests/data/su

@@ -68,4 +68,14 @@ data = [
         'su_user': 'vsc43020',
     }
 },
+{
+    "raw": "<37>Oct 10 11:52:35 C4STO01-Node2 su[3984121]: (to storage) root on none",
+    "expected": {
+        'program': 'su',
+        '@source_host': 'C4STO01-Node2',
+        'su_user': 'root',
+        'su_tty': 'none',
+        'su_to': 'storage',
+    }
+},
 ]

tests/data/sudo

@@ -33,4 +33,16 @@ data = [
             "sudo_command": "/usr/bin/ls",
         },
     },
+    {
+        "raw": "<85>Oct 10 11:52:35 C4STO01-Node2 sudo[3984032]:  ha_omm : TTY=unknown ; PWD=/opt/dfv/oam/oam-u/ha/ha/module/hacom/bin ; USER=root ; COMMAND=../../../module/harm/plugin/script/harm_mgr.sh floatIP.sh status active key1=value1; 0 ha2",
+        "expected": {
+            "@source_host": "C4STO01-Node2",
+            "program": "sudo",
+            "sudo_pwd": "/opt/dfv/oam/oam-u/ha/ha/module/hacom/bin",
+            "sudo_runas": "root",
+            "sudo_tty": "unknown",
+            "sudo_user": "ha_omm",
+            "sudo_command": "../../../module/harm/plugin/script/harm_mgr.sh floatIP.sh status active key1=value1; 0 ha2",
+        },
+    },
 ]

tests/logstash_7.6.2.conf

@@ -16,7 +16,7 @@ filter {
 
         match => {
             # RSYSLOGCUSTOM always last (and no PREFIX)!
-            "message" => ["%{RSYSLOGPREFIX}%{SU_MSG}", "%{RSYSLOGPREFIX}%{SUDO_MSG}", "%{RSYSLOGPREFIX}%{REFRAME_MSG}", "%{RSYSLOGPREFIX}%{BASH_MSG}", "%{RSYSLOGPREFIX}%{FAIL2BAN_MSG}", "%{RSYSLOGPREFIX}%{SINGULARITY_MSG}", "%{RSYSLOGPREFIX}%{DHCPD_MSG}", "%{RSYSLOGPREFIX}%{SSH_MSG}", "%{RSYSLOGPREFIX}%{MODULECMD_MSG}", "%{RSYSLOGPREFIX}%{LMOD_MSG}", "%{RSYSLOGPREFIX}%{NFS_MSG}", "%{RSYSLOGPREFIX}%{CEPH_MSG}", "%{RSYSLOGPREFIX}%{OPENNEBULA_MSG}", "%{RSYSLOGPREFIX}%{JUBE_MSG}", "%{RSYSLOGPREFIX}%{SHOREWALL_MSG}", "%{RSYSLOGPREFIX}%{KEYVALUE_MSG}", "%{RSYSLOGPREFIX}%{QUATTOR_MSG}", "%{RSYSLOGPREFIX}%{SNOOPY_MSG}", "%{RSYSLOGCUSTOM}"]
+            "message" => ["%{RSYSLOGCUSTOMHEADER}%{PIXIU_BASH}", "%{RSYSLOGCUSTOMHEADER}%{PIXIU_NGINX}", "%{RSYSLOGCUSTOMHEADER}%{PIXIU_TOMCAT}", "%{RSYSLOGPREFIX}%{PIXIU_APACHE}", "%{RSYSLOGPREFIX}%{SU_MSG}", "%{RSYSLOGPREFIX}%{SUDO_MSG}", "%{RSYSLOGPREFIX}%{REFRAME_MSG}", "%{RSYSLOGPREFIX}%{BASH_MSG}", "%{RSYSLOGPREFIX}%{FAIL2BAN_MSG}", "%{RSYSLOGPREFIX}%{SINGULARITY_MSG}", "%{RSYSLOGPREFIX}%{DHCPD_MSG}", "%{RSYSLOGPREFIX}%{SSH_MSG}", "%{RSYSLOGPREFIX}%{MODULECMD_MSG}", "%{RSYSLOGPREFIX}%{LMOD_MSG}", "%{RSYSLOGPREFIX}%{NFS_MSG}", "%{RSYSLOGPREFIX}%{CEPH_MSG}", "%{RSYSLOGPREFIX}%{OPENNEBULA_MSG}", "%{RSYSLOGPREFIX}%{JUBE_MSG}", "%{RSYSLOGPREFIX}%{SHOREWALL_MSG}", "%{RSYSLOGPREFIX}%{KEYVALUE_MSG}", "%{RSYSLOGPREFIX}%{QUATTOR_MSG}", "%{RSYSLOGPREFIX}%{SNOOPY_MSG}", "%{RSYSLOGCUSTOM}"]
         }
     }
 
@@ -25,7 +25,7 @@ filter {
     }
 
     date {
-        match => [ "syslog_timestamp", "yyyy-MM-dd'T'HH:mm:ss.SSSSSSZZ", "yyyy-MM-dd'T'HH:mm:ssZZ", "yyyy-MM-dd HH:mm:ss.SSSSSS" ]
+        match => [ "syslog_timestamp", "yyyy-MM-dd'T'HH:mm:ss.SSSSSSZZ", "yyyy-MM-dd'T'HH:mm:ssZZ", "yyyy-MM-dd HH:mm:ss.SSSSSS", "MMM dd HH:mm:ss" ]
     }
 
     if ("exclude_tags" not in [tags]) {