Merge branch 'main' into fix/fortigate_timestamp_issue

SEKOIA-IO · Dec 11, 2024 · dd192ea · dd192ea
2 parents 66aa02b + 27fe310
commit dd192ea
Show file tree

Hide file tree

Showing 16 changed files with 430 additions and 16 deletions.
diff --git a/HAProxy/haproxy/CHANGELOG.md b/HAProxy/haproxy/CHANGELOG.md
@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## 2024-12.04 - 1.0.1
+
+- Add support for aktci at the end of the log
+
 ## 2024-03.04 - 1.0.0
 
 ### Added

diff --git a/HAProxy/haproxy/ingest/parser.yml b/HAProxy/haproxy/ingest/parser.yml
@@ -14,7 +14,7 @@ pipeline:
             ([0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})([0-9])
           HAPROXYURL: "(%{URIPROTO:url_scheme}://)?(?:%{USER:url_username}(?::[^@]*)?@)?(?:%{URIHOST:url_domain})?(?:%{URIPATHPARAM:url_path})"
           TLS_PROTOCOL: "TLS"
-          HAPROXYHTTPBASE: '%{IP:source_ip}:%{INT:source_port} \[%{HAPROXYDATE}\] %{NOTSPACE} %{NOTSPACE}/%{NOTSPACE} %{INT}/%{INT}/%{INT}/%{INT}/%{NOTSPACE} %{INT:http_response_status_code} %{NOTSPACE:http_response_bytes} %{DATA:http_request_cookie} %{DATA:http_response_cookie} %{NOTSPACE} %{INT}/%{INT}/%{INT}/%{INT}/%{NOTSPACE} %{INT}/%{INT} (\{%{DATA:captured_request_headers}\})?( )?(\{%{DATA:captured_response_headers}\})?( )?"(<BADREQ>|(%{WORD:http_request_method} (?:%{HAPROXYURL:url_original})?( HTTP/%{NUMBER:http_version})?))?"( %{TLS_PROTOCOL:tls_protocol}v%{NUMBER:tls_version})?'
+          HAPROXYHTTPBASE: '%{IP:source_ip}:%{INT:source_port} \[%{HAPROXYDATE}\] %{NOTSPACE} %{NOTSPACE}/%{NOTSPACE} %{INT}/%{INT}/%{INT}/%{INT}/%{NOTSPACE} %{INT:http_response_status_code} %{NOTSPACE:http_response_bytes} %{DATA:http_request_cookie} %{DATA:http_response_cookie} %{NOTSPACE} %{INT}/%{INT}/%{INT}/%{INT}/%{NOTSPACE} %{INT}/%{INT} (\{%{DATA:captured_request_headers}\})?( )?(\{%{DATA:captured_response_headers}\})?( )?"(<BADREQ>|(%{WORD:http_request_method} (?:%{HAPROXYURL:url_original})?( HTTP/%{NUMBER:http_version})?))?"( %{TLS_PROTOCOL:tls_protocol}v%{NUMBER:tls_version})?%{GREEDYDATA}'
 
   - name: json
     filter: "{{grok.message.json_msg | length > 0}}"

diff --git a/HAProxy/haproxy/tests/access4.json b/HAProxy/haproxy/tests/access4.json
@@ -0,0 +1,45 @@
+{
+  "input": {
+    "message": "90.83.225.109:54761 [10/Apr/2024:15:41:58.284] frontend_https~ backend_lb/LB100 1796/0/0/28/1824 200 1060 - - --VN 296/296/33/6/0 0/0 {saas.ms.example.com} \"GET /path/get/resource HTTP/1.1\" TLSv1.2 aktci:\"46.193.65.202\"\n",
+    "sekoiaio": {
+      "intake": {
+        "dialect": "HAProxy",
+        "dialect_uuid": "ff1873e7-8757-4b1a-b0ca-b33f9b27f3d9"
+      }
+    }
+  },
+  "expected": {
+    "message": "90.83.225.109:54761 [10/Apr/2024:15:41:58.284] frontend_https~ backend_lb/LB100 1796/0/0/28/1824 200 1060 - - --VN 296/296/33/6/0 0/0 {saas.ms.example.com} \"GET /path/get/resource HTTP/1.1\" TLSv1.2 aktci:\"46.193.65.202\"\n",
+    "event": {
+      "kind": "access"
+    },
+    "http": {
+      "request": {
+        "method": "GET"
+      },
+      "response": {
+        "bytes": 1060,
+        "status_code": 200
+      },
+      "version": "1.1"
+    },
+    "related": {
+      "ip": [
+        "90.83.225.109"
+      ]
+    },
+    "source": {
+      "address": "90.83.225.109",
+      "ip": "90.83.225.109",
+      "port": 54761
+    },
+    "tls": {
+      "version": "1.2",
+      "version_protocol": "TLS"
+    },
+    "url": {
+      "original": "/path/get/resource",
+      "path": "/path/get/resource"
+    }
+  }
+}
diff --git a/HarfangLab/harfanglab/_meta/fields.yml b/HarfangLab/harfanglab/_meta/fields.yml
@@ -953,6 +953,11 @@ action.properties.param9:
   name: action.properties.param9
   type: keyword
 
+harfanglab.agent_ids:
+  description: ''
+  name: harfanglab.agent_ids
+  type: keyword
+
 harfanglab.aggregation_key:
   description: The key to the events aggregation
   name: harfanglab.aggregation_key

diff --git a/HarfangLab/harfanglab/ingest/parser.yml b/HarfangLab/harfanglab/ingest/parser.yml
@@ -142,6 +142,10 @@ stages:
           organization.id: "{{json_event.message.tenant}}"
           url.original: "{{json_event.message.details_url_request.url}}"
 
+      - set:
+          harfanglab.agent_ids: "{{json_event.message.agents | map(attribute='agent_id') | list}}"
+        filter: "{{json_event.message.agents | length > 0}}"
+
   network_info:
     actions:
       - set:

diff --git a/HarfangLab/harfanglab/tests/threat_critical.json b/HarfangLab/harfanglab/tests/threat_critical.json
@@ -13,6 +13,9 @@
       "name": "harfanglab"
     },
     "harfanglab": {
+      "agent_ids": [
+        "af5e2f63-becd-4660-ade8-30d04c0dd044"
+      ],
       "count": {
         "rules": 1,
         "users_impacted": 0

diff --git a/HarfangLab/harfanglab/tests/threat_log.json b/HarfangLab/harfanglab/tests/threat_log.json
@@ -13,6 +13,10 @@
       "name": "harfanglab"
     },
     "harfanglab": {
+      "agent_ids": [
+        "215fe295-905f-4a8d-8347-e9d438d4e415",
+        "999ba0c7-96b8-4c57-bf0e-63b24813c873"
+      ],
       "count": {
         "rules": 4,
         "users_impacted": 3

diff --git a/Microsoft/microsoft-365-defender/_meta/manifest.yml b/Microsoft/microsoft-365-defender/_meta/manifest.yml
@@ -1,11 +1,11 @@
 uuid: 05e6f36d-cee0-4f06-b575-9e43af779f9f
-name: Microsoft 365 Defender
+name: Microsoft Defender XDR / Microsoft 365 Defender
 slug: microsoft-365-defender
 automation_connector_uuid: 57f8f587-18ee-434b-a4ed-b5459f5b0fef
 automation_module_uuid: 525eecc0-9eee-484d-92bd-039117cf4dac
 
 description: >-
-  Microsoft 365 Defender is a entreprise defense suite that coordinates detection, prevention, investigation, and response across endpoints, identities, email, and cloud applications.
+  Microsoft Defender XDR is a entreprise defense suite that coordinates detection, prevention, investigation, and response across endpoints, identities, email, and cloud applications.
 
   Microsoft Defender for Endpoint is an enterprise endpoint security platform designed to help enterprise networks prevent, detect, investigate, and respond to advanced threats.
 

diff --git a/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml b/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml
@@ -626,7 +626,7 @@ pipeline:
           AUTHENTICATION_WEB: "User %{USERNAME:user} logged in via %{DATA} from %{IP:src} using %{DATA:proto}"
           REASON1: 'User-ID server monitor %{HOSTNAME:hostname}\(%{WORD:vsys}\) %{GREEDYDATA:message}'
           REASON2: "ldap cfg %{WORD:config_name} connected to server %{IP:destination_ip}:%{INT:port}, initiated by: %{IP:source_ip}"
-          REASON3: "When authenticating user %{WORD:user} from %{IP:source_ip}, a less secure authentication method %{WORD:auth_method} is used. Please migrate to %{WORD:recommended_methods1} or %{DATA:recommended_methods2}. Authentication Profile %{WORD:auth_profile}, vsys %{WORD:vsys}, Server Profile %{WORD:server_profile}, Server Address %{IP:destination_ip}"
+          REASON3: "When authenticating user '?%{WORD:user}'? from '?%{IP:source_ip}'?, a less secure authentication method %{WORD:auth_method} is used. Please migrate to %{WORD:recommended_methods1} or %{DATA:recommended_methods2}. Authentication Profile '?%{WORD:auth_profile}'?, vsys '?%{WORD:vsys}'?, Server Profile '?%{WORD:server_profile}'?, Server Address '?%{IP:destination_ip}'?"
           REASON4: "failed authentication for user %{WORD:user}. Reason: %{GREEDYDATA:reason} auth profile %{WORD:auth_profile}, vsys %{WORD:vsys}, server profile %{WORD:server_profile}, server address %{IP:destination_ip}, auth protocol %{WORD:auth_protocol}, From: %{IP:source_ip}"
           REASON5: 'authenticated for user %{WORD:user}\.   auth profile %{WORD:auth_profile}, vsys %{WORD:vsys}, server profile %{DATA:server_profile}, server address %{IP:destination_ip}, auth protocol %{WORD:auth_protocol}, admin role %{WORD:admin_role}, From: %{IP:source_ip}\.'
     filter: '{{parsed_event.message.get("EventDescription") != None}}'

diff --git a/Palo Alto Networks/paloalto-ngfw/tests/test_system_event_13.json b/Palo Alto Networks/paloalto-ngfw/tests/test_system_event_13.json
@@ -0,0 +1,74 @@
+{
+  "input": {
+    "message": "1,2024/11/26 22:10:01,02410100000000,SYSTEM,auth,2555,2024/11/26 22:10:01,,auth-success,FWPA,0,0,general,informational,\"When authenticating user 'test000555' from '1.2.5.5', a less secure authentication method PAP is used. Please migrate to PEAP or EAP-TTLS. Authentication Profile 'FWPA', vsys 'shared', Server Profile 'RADIUS_RSA', Server Address '1.7.4.2'\",738970652229900000000,0x0,0,0,0,0,,FWPAN00,0,0,2024-11-26T22:10:01.627+01:00",
+    "sekoiaio": {
+      "intake": {
+        "dialect": "Palo Alto NGFW",
+        "dialect_uuid": "903ec1b8-f206-4ba5-8563-db21da09cafd"
+      }
+    }
+  },
+  "expected": {
+    "message": "1,2024/11/26 22:10:01,02410100000000,SYSTEM,auth,2555,2024/11/26 22:10:01,,auth-success,FWPA,0,0,general,informational,\"When authenticating user 'test000555' from '1.2.5.5', a less secure authentication method PAP is used. Please migrate to PEAP or EAP-TTLS. Authentication Profile 'FWPA', vsys 'shared', Server Profile 'RADIUS_RSA', Server Address '1.7.4.2'\",738970652229900000000,0x0,0,0,0,0,,FWPAN00,0,0,2024-11-26T22:10:01.627+01:00",
+    "event": {
+      "category": [
+        "authentication"
+      ],
+      "dataset": "system",
+      "reason": "When authenticating user 'test000555' from '1.2.5.5', a less secure authentication method PAP is used. Please migrate to PEAP or EAP-TTLS. Authentication Profile 'FWPA', vsys 'shared', Server Profile 'RADIUS_RSA', Server Address '1.7.4.2'",
+      "type": [
+        "start"
+      ]
+    },
+    "@timestamp": "2024-11-26T21:10:01.627000Z",
+    "action": {
+      "name": "auth-success",
+      "type": "auth"
+    },
+    "destination": {
+      "address": "1.7.4.2",
+      "ip": "1.7.4.2"
+    },
+    "log": {
+      "hostname": "FWPAN00",
+      "level": "informational",
+      "logger": "system"
+    },
+    "observer": {
+      "name": "FWPAN00",
+      "product": "PAN-OS",
+      "serial_number": "02410100000000"
+    },
+    "paloalto": {
+      "DGHierarchyLevel1": "0",
+      "DGHierarchyLevel2": "0",
+      "DGHierarchyLevel3": "0",
+      "DGHierarchyLevel4": "0",
+      "EventID": "auth-success",
+      "Threat_ContentType": "auth",
+      "authetification": {
+        "profile": "FWPA"
+      },
+      "server": {
+        "profile": "RADIUS_RSA"
+      },
+      "vsys": "shared"
+    },
+    "related": {
+      "ip": [
+        "1.2.5.5",
+        "1.7.4.2"
+      ],
+      "user": [
+        "test000555"
+      ]
+    },
+    "source": {
+      "address": "1.2.5.5",
+      "ip": "1.2.5.5"
+    },
+    "user": {
+      "name": "test000555"
+    }
+  }
+}
diff --git a/Pradeo/pradeo-mtd/ingest/parser.yml b/Pradeo/pradeo-mtd/ingest/parser.yml
@@ -176,16 +176,16 @@ stages:
           pradeo.device.mdmId: "{{json_event.message.content.deviceApplicationCompliance.deviceApplication.device.emmDeviceInfo.externalId}}"
           pradeo.device.emm: "{{json_event.message.content.deviceApplicationCompliance.deviceApplication.device.emmDeviceInfo.emm}}"
           pradeo.compliance.matchedResponseRules: "{{json_event.message.content.deviceApplication.compliance.matchedResponseRules}}"
-          pradeo.application.id: "{{json_event.message.content.deviceApplicationCompliance.application.id}}"
-          pradeo.application.package: "{{json_event.message.content.deviceApplicationCompliance.application.package.package}}"
-          pradeo.application.system: "{{json_event.message.content.deviceApplicationCompliance.application.package.system}}"
-          pradeo.application.version: "{{json_event.message.content.deviceApplicationCompliance.application.version}}"
-          pradeo.application.versionCode: "{{json_event.message.content.deviceApplicationCompliance.application.versionCode}}"
-          pradeo.application.name: "{{json_event.message.content.deviceApplicationCompliance.application.name}}"
-          pradeo.application.md5: "{{json_event.message.content.deviceApplicationCompliance.application.md5}}"
-          pradeo.application.sha1: "{{json_event.message.content.deviceApplicationCompliance.application.sha1}}"
-          pradeo.application.sha256: "{{json_event.message.content.deviceApplicationCompliance.application.sha256}}"
-          pradeo.detection.status: "{{json_event.message.content.deviceApplicationCompliance.status}}"
+          pradeo.application.id: "{{json_event.message.content.deviceApplicationCompliance.deviceApplication.application.id}}"
+          pradeo.application.package: "{{json_event.message.content.deviceApplicationCompliance.deviceApplication.application.package.package}}"
+          pradeo.application.system: "{{json_event.message.content.deviceApplicationCompliance.deviceApplication.application.package.system}}"
+          pradeo.application.version: "{{json_event.message.content.deviceApplicationCompliance.deviceApplication.application.version}}"
+          pradeo.application.versionCode: "{{json_event.message.content.deviceApplicationCompliance.deviceApplication.application.versionCode}}"
+          pradeo.application.name: "{{json_event.message.content.deviceApplicationCompliance.deviceApplication.application.name}}"
+          pradeo.application.md5: "{{json_event.message.content.deviceApplicationCompliance.deviceApplication.application.md5}}"
+          pradeo.application.sha1: "{{json_event.message.content.deviceApplicationCompliance.deviceApplication.application.sha1}}"
+          pradeo.application.sha256: "{{json_event.message.content.deviceApplicationCompliance.deviceApplication.application.sha256}}"
+          pradeo.detection.status: "{{json_event.message.content.deviceApplicationCompliance.deviceApplication.status}}"
       - filter: '{{json_event.message.type == "DeviceComplianceUpdated"}}'
         set:
           event.category: ["process"]