VMWare_ESXI: Addition of more fields and parser enhancement

SEKOIA-IO · Dec 20, 2024 · a6923e5 · a6923e5
1 parent cf11456
commit a6923e5
Show file tree

Hide file tree

Showing 31 changed files with 504 additions and 21 deletions.
diff --git a/VMWare/vmware-esxi/_meta/fields.yml b/VMWare/vmware-esxi/_meta/fields.yml
@@ -1,3 +1,18 @@
+vmware.esxi.log.category:
+  description: Log category
+  name: vmware.esxi.log.category
+  type: keyword
+
+vmware.esxi.log.type:
+  description: Infos about the event type of the log
+  name: vmware.esxi.log.type
+  type: keyword
+
+vmware.esxi.operation.id:
+  description: Operation ID
+  name: vmware.esxi.operation.id
+  type: keyword
+
 wmware.esxi.event.serial_number:
   description: Serial number
   name: wmware.esxi.event.serial_number

diff --git a/VMWare/vmware-esxi/ingest/parser.yml b/VMWare/vmware-esxi/ingest/parser.yml
@@ -10,7 +10,7 @@ pipeline:
       properties:
         input_field: "original.message"
         output_field: result
-        pattern: "%{EVENT_LOG_SSH_AUTH}|%{PROCESS_LOG_SSH_AUTH}|%{PROCESS_LOG_SSH_PASSWORD}|%{AUTH_CONNECTION_EVENT}|%{AUTH_CONNECTION_DISCONNECTED}|%{AUTH_CONNECTION_RECEIVE_DISCONNECTED}|%{AUTH_SESSION_EVENT}|%{PROCESS_LOG_AUTH_USER}|%{PROCESS_LOG_CHANGE_USER}|%{AUTH_PUBLIC_KEY}|%{NETWORK_SOURCE}|%{NETWORK_HOSTNAME}|%{PROCESS_COMMAND}|%{PROCESS_COMMAND_USER_WITH_PID}|%{PROCESS_COMMAND_USER_WITHOUT_PID}|%{PROCESS_LOG_LAUNCHING}|%{VARIOUS_FILE_ACTION}|%{VARIOUS_FILE_ACTION_FILENAME_ONLY}|%{VARIOUS_KEY_REASON}|%{VARIOUS_SERIAL_NUMBER}|%{VARIOUS_DEVICE_OUTCOME_REASON}|%{VARIOUS_REASON_1}|%{VARIOUS_REASON_2}|%{DEFAULT_EVENT_LOG}|%{DEFAULT_PROCESS_LOG}|%{DEFAULT_PROCESS_LOG_WITH_ADDITIONAL_INFO}|%{SSHD_INFO}"
+        pattern: "%{EVENT_LOG_SSH_AUTH}|%{PROCESS_LOG_SSH_AUTH}|%{PROCESS_LOG_SSH_PASSWORD}|%{AUTH_CONNECTION_EVENT}|%{AUTH_CONNECTION_DISCONNECTED}|%{AUTH_CONNECTION_RECEIVE_DISCONNECTED}|%{AUTH_SESSION_EVENT}|%{PROCESS_LOG_AUTH_USER}|%{PROCESS_LOG_CHANGE_USER}|%{AUTH_PUBLIC_KEY}|%{NETWORK_SOURCE}|%{NETWORK_HOSTNAME}|%{PROCESS_COMMAND}|%{PROCESS_COMMAND_USER_WITH_PID}|%{PROCESS_COMMAND_USER_WITHOUT_PID}|%{PROCESS_LOG_LAUNCHING}|%{VARIOUS_FILE_ACTION}|%{VARIOUS_FILE_ACTION_FILENAME_ONLY}|%{VARIOUS_KEY_REASON}|%{VARIOUS_SERIAL_NUMBER}|%{VARIOUS_DEVICE_OUTCOME_REASON}|%{VARIOUS_REASON_1}|%{VARIOUS_REASON_2}|%{DEFAULT_EVENT_LOG}|%{DEFAULT_PROCESS_LOG_WITH_ADDITIONAL_INFO}|%{DEFAULT_PROCESS_LOG}|%{SSHD_INFO}"
         custom_patterns:
           ## Freestyle patterns to be used for messages that contains connection event information
           ## To identify auth event we can use `auth_action` field or `auth_category` described below
@@ -31,7 +31,7 @@ pipeline:
           NETWORK_SOURCE: '^(?:\s*)(?P<network_action>Connection|connection) from (?:%{SSHD_CLIENT_ADDRESS:ip_address}) port %{NUMBER:port}%{DATA}?'
 
           ## Freestyle patterns to be used for messages that contains network hostname information
-          NETWORK_HOSTNAME: '^(?:\s*)%{WORD} on %{DATA:network_category} to (?:%{SSHD_CLIENT_ADDRESS:ip_address}) port %{NUMBER:port}%{DATA}?'
+          NETWORK_HOSTNAME: '^(?:\s*)%{WORD:log_lvl} on %{DATA:network_category} to (?:%{SSHD_CLIENT_ADDRESS:ip_address}) port %{NUMBER:port}%{DATA}?'
 
           ## Freestyle patterns to be used for messages that contains command and user
           PROCESS_COMMAND: '^(?:\s*)(?P<process_category>Executing) %{GREEDYDATA:command}?'
@@ -71,7 +71,7 @@ pipeline:
 
           PROCESS_HEADER: '^(%{WORD}: )?(?:\s*)(?:%{WORD:log_lvl}) %{PROCESS_WITH_PID}' # e.g. `error hostd[2099655]` or `hostd: error hostd[2099655]`
           PROCESS_WITH_PID: '%{DATA:process}\[%{NUMBER:pid}\]' # e.g. `hostd[2099655]`
-          PROCESS_ORIGINATOR: '\[%{WORD}\@%{NUMBER} %{DATA}\]' # e.g. `[Originator@6876 sub=Vimsvc.ha-eventmgr]`
+          PROCESS_ORIGINATOR: '\[Originator\@%{NUMBER}( sub=%{NOTSPACE:sub})?( opID=%{NOTSPACE:op_id})?( user=%{USER:username}(\:%{NOTSPACE:user_domain})?)?%{DATA}\]' # e.g. `[Originator@6876 sub=Vimsvc.ha-eventmgr]`
           PROCESS_HEADER_WITH_ORIGINATOR: "%{PROCESS_HEADER} %{PROCESS_ORIGINATOR}" # e.g. `error hostd[2099655] [Originator@6876 sub=Vimsvc.ha-eventmgr]`
 
           ## This is log pattern to work with ssh process. Based on `DEFAULT_PROCESS_LOG_WITH_ADDITIONAL_INFO`
@@ -137,10 +137,7 @@ pipeline:
           ### info hostd[2099655] [Originator@6876 sub=Vimsvc.ha-eventmgr] <LOG_MSG>
           DEFAULT_PROCESS_LOG: "%{PROCESS_HEADER_WITH_ORIGINATOR} %{GREEDYDATA:message}?"
 
-          ## Difference between this one and previous is that meta info can contain various info like `user=<username>`, `Req <Req id>`, `opID=<operation id>` and so on
-          ## Example of log:
-          ### info hostd[2099655] [Originator@6876 sub=Vimsvc.ha-eventmgr opID=233816e0 user=vpxuser Req 06028] <LOG_MSG>
-          DEFAULT_PROCESS_LOG_WITH_ADDITIONAL_INFO: "%{PROCESS_HEADER_WITH_ORIGINATOR} %{GREEDYDATA:message}?"
+          DEFAULT_PROCESS_LOG_WITH_ADDITIONAL_INFO: '%{PROCESS_HEADER_WITH_ORIGINATOR} \[%{NOTSPACE:log_category}\] %{GREEDYDATA:message}?'
 
           # More SSHD patterns: https://github.com/SEKOIA-IO/intake-formats/blob/main/OpenSSH/openssh/ingest/parser.yml
           SSHD_CLIENT_ADDRESS: "%{IP:ip_address}|%{HOSTNAME:source_domain}"
@@ -187,6 +184,7 @@ stages:
       # Fields by definition that we get from messages
       - set:
           user.name: "{{parsed_event.result.username}}"
+          user.domain: "{{parsed_event.result.user_domain}}"
           source.ip: "{{parsed_event.result.ip_address}}"
           source.port: "{{parsed_event.result.port}}"
           source.user.name: "{{parsed_event.result.source_username}}"
@@ -195,6 +193,13 @@ stages:
           wmware.esxi.key.id: "{{parsed_event.result.key_id | trim}}"
           wmware.esxi.event.serial_number: "{{parsed_event.result.serial_number}}"
           host.name: "{{parsed_event.result.server_name}}"
+          process.name: "{{parsed_event.result.process}}"
+          vmware.esxi.operation.id: "{{parsed_event.result.op_id}}"
+          vmware.esxi.log.category: "{{parsed_event.result.auth_category or parsed_event.result.log_category}}"
+
+      - set:
+          vmware.esxi.log.type: "{{parsed_event.result.log_lvl or parsed_event.result.network_category | lower }}"
+        filter: "{{parsed_event.result.log_lvl != None or parsed_event.result.network_category != None}}"
 
       # It is necessary to postprocess the command line because it might contain `'` at the beginning and/or at the end
       - set:
@@ -206,7 +211,7 @@ stages:
         filter: >
           {{
             parsed_event.result.pid != None and
-             parsed_event.result.process_category != None
+            parsed_event.result.process_category != None
           }}
 
   set_event_reason:
@@ -217,9 +222,9 @@ stages:
           event.reason: "{{parsed_event.result.reason}}"
         filter: >
           {{
-             parsed_event.result.key_id != None or
-             parsed_event.result.serial_number != None or
-             parsed_event.result.outcome != None
+            parsed_event.result.key_id != None or
+            parsed_event.result.serial_number != None or
+            parsed_event.result.outcome != None
           }}
 
       - set:
@@ -301,15 +306,15 @@ stages:
         filter: >
           {{ 
             parsed_event.result.auth_category  == None and
-             parsed_event.result.auth_action == None and
-             parsed_event.result.network_category == None and
-             parsed_event.result.network_action == None and
-             parsed_event.result.configuration_category == None and
-             parsed_event.result.configuration_action == None and
-             parsed_event.result.file_action == None and
-             parsed_event.result.file_category == None and
-             parsed_event.result.session_category == None and
-             parsed_event.result.session_action == None
+            parsed_event.result.auth_action == None and
+            parsed_event.result.network_category == None and
+            parsed_event.result.network_action == None and
+            parsed_event.result.configuration_category == None and
+            parsed_event.result.configuration_action == None and
+            parsed_event.result.file_action == None and
+            parsed_event.result.file_category == None and
+            parsed_event.result.session_category == None and
+            parsed_event.result.session_action == None
           }}
 
       # SSHD events

diff --git a/VMWare/vmware-esxi/tests/AUTH_SESSION_change_user.json b/VMWare/vmware-esxi/tests/AUTH_SESSION_change_user.json
@@ -16,6 +16,9 @@
       "product": "ESXi",
       "vendor": "VMware"
     },
+    "process": {
+      "name": "hostd"
+    },
     "related": {
       "ip": [
         "127.0.0.1"
@@ -34,6 +37,16 @@
     },
     "user": {
       "name": "pyvmomi"
+    },
+    "vmware": {
+      "esxi": {
+        "log": {
+          "type": "info"
+        },
+        "operation": {
+          "id": "esxcli-76-ecbf"
+        }
+      }
     }
   }
 }
diff --git a/VMWare/vmware-esxi/tests/AUTH_SESSION_change_user_with_user_agent.json b/VMWare/vmware-esxi/tests/AUTH_SESSION_change_user_with_user_agent.json
@@ -16,6 +16,9 @@
       "product": "ESXi",
       "vendor": "VMware"
     },
+    "process": {
+      "name": "hostd"
+    },
     "related": {
       "ip": [
         "127.0.0.1"
@@ -44,6 +47,16 @@
       "os": {
         "name": "Other"
       }
+    },
+    "vmware": {
+      "esxi": {
+        "log": {
+          "type": "info"
+        },
+        "operation": {
+          "id": "esxcli-76-ecbf"
+        }
+      }
     }
   }
 }
diff --git a/VMWare/vmware-esxi/tests/AUTH_SESSION_user.json b/VMWare/vmware-esxi/tests/AUTH_SESSION_user.json
@@ -16,13 +16,27 @@
       "product": "ESXi",
       "vendor": "VMware"
     },
+    "process": {
+      "name": "hostd"
+    },
     "related": {
       "user": [
         "root"
       ]
     },
     "user": {
       "name": "root"
+    },
+    "vmware": {
+      "esxi": {
+        "log": {
+          "category": "Auth",
+          "type": "info"
+        },
+        "operation": {
+          "id": "esxcli-a5-20ae"
+        }
+      }
     }
   }
 }
diff --git a/VMWare/vmware-esxi/tests/AUTH_SSH_PASSWORD.json b/VMWare/vmware-esxi/tests/AUTH_SSH_PASSWORD.json
@@ -16,6 +16,9 @@
       "product": "ESXi",
       "vendor": "VMware"
     },
+    "process": {
+      "name": "hostd"
+    },
     "related": {
       "ip": [
         "127.0.0.1"
@@ -30,6 +33,16 @@
     },
     "user": {
       "name": "root"
+    },
+    "vmware": {
+      "esxi": {
+        "log": {
+          "type": "info"
+        },
+        "operation": {
+          "id": "esxcli-76-ecbf"
+        }
+      }
     }
   }
 }
diff --git a/VMWare/vmware-esxi/tests/AUTH_SSH_event_closed.json b/VMWare/vmware-esxi/tests/AUTH_SSH_event_closed.json
@@ -16,6 +16,9 @@
       "product": "ESXi",
       "vendor": "VMware"
     },
+    "process": {
+      "name": "vob.user.ssh.session.closed"
+    },
     "related": {
       "ip": [
         "1.2.3.4"

diff --git a/VMWare/vmware-esxi/tests/AUTH_SSH_event_opened.json b/VMWare/vmware-esxi/tests/AUTH_SSH_event_opened.json
@@ -16,6 +16,9 @@
       "product": "ESXi",
       "vendor": "VMware"
     },
+    "process": {
+      "name": "vob.user.ssh.session.opened"
+    },
     "related": {
       "ip": [
         "1.2.3.4"

diff --git a/VMWare/vmware-esxi/tests/AUTH_SSH_login_failed.json b/VMWare/vmware-esxi/tests/AUTH_SSH_login_failed.json
@@ -17,6 +17,9 @@
       "product": "ESXi",
       "vendor": "VMware"
     },
+    "process": {
+      "name": "vob.user.ssh.session.failed"
+    },
     "related": {
       "ip": [
         "1.2.3.4"

diff --git a/VMWare/vmware-esxi/tests/AUTH_SSH_process_closed.json b/VMWare/vmware-esxi/tests/AUTH_SSH_process_closed.json
@@ -16,6 +16,9 @@
       "product": "ESXi",
       "vendor": "VMware"
     },
+    "process": {
+      "name": "hostd"
+    },
     "related": {
       "ip": [
         "1.2.3.4"
@@ -30,6 +33,13 @@
     },
     "user": {
       "name": "root"
+    },
+    "vmware": {
+      "esxi": {
+        "log": {
+          "type": "error"
+        }
+      }
     }
   }
 }
diff --git a/VMWare/vmware-esxi/tests/AUTH_SSH_process_opened.json b/VMWare/vmware-esxi/tests/AUTH_SSH_process_opened.json
@@ -16,6 +16,9 @@
       "product": "ESXi",
       "vendor": "VMware"
     },
+    "process": {
+      "name": "hostd"
+    },
     "related": {
       "ip": [
         "1.2.3.4"
@@ -30,6 +33,13 @@
     },
     "user": {
       "name": "root"
+    },
+    "vmware": {
+      "esxi": {
+        "log": {
+          "type": "info"
+        }
+      }
     }
   }
 }
diff --git a/VMWare/vmware-esxi/tests/NETWORK_HOSTNAME.json b/VMWare/vmware-esxi/tests/NETWORK_HOSTNAME.json
@@ -25,6 +25,13 @@
       "address": "255.255.255.255",
       "ip": "255.255.255.255",
       "port": 67
+    },
+    "vmware": {
+      "esxi": {
+        "log": {
+          "type": "DHCPDISCOVER"
+        }
+      }
     }
   }
 }
diff --git a/VMWare/vmware-esxi/tests/PROCESS_BEGIN.json b/VMWare/vmware-esxi/tests/PROCESS_BEGIN.json
@@ -0,0 +1,41 @@
+{
+  "input": {
+    "message": "info vpxa[2100523] [Originator@6876 sub=vpxLro opID=PollQuickStatsLoop-37dc5435-21] [VpxLRO] -- BEGIN lro-437646 -- vpxa -- vpxapi.VpxaService.fetchQuickStats -- 12345678-abcd-ef90-1234-abcdef123456",
+    "sekoiaio": {
+      "intake": {
+        "dialect": "VMware ESXi",
+        "dialect_uuid": "2b13307b-7439-4973-900a-2b58303cac90"
+      }
+    }
+  },
+  "expected": {
+    "message": "info vpxa[2100523] [Originator@6876 sub=vpxLro opID=PollQuickStatsLoop-37dc5435-21] [VpxLRO] -- BEGIN lro-437646 -- vpxa -- vpxapi.VpxaService.fetchQuickStats -- 12345678-abcd-ef90-1234-abcdef123456",
+    "event": {
+      "category": [
+        "process"
+      ],
+      "reason": "-- BEGIN lro-437646 -- vpxa -- vpxapi.VpxaService.fetchQuickStats -- 12345678-abcd-ef90-1234-abcdef123456",
+      "type": [
+        "info"
+      ]
+    },
+    "observer": {
+      "product": "ESXi",
+      "vendor": "VMware"
+    },
+    "process": {
+      "name": "vpxa"
+    },
+    "vmware": {
+      "esxi": {
+        "log": {
+          "category": "VpxLRO",
+          "type": "info"
+        },
+        "operation": {
+          "id": "PollQuickStatsLoop-37dc5435-21"
+        }
+      }
+    }
+  }
+}