Merge branch 'main' into lv/ocsf_add_support_of_1_3

SEKOIA-IO · Nov 14, 2024 · 83c2782 · 83c2782
2 parents 8bea0af + d4259d9
commit 83c2782
Show file tree

Hide file tree

Showing 23 changed files with 516 additions and 5 deletions.
diff --git a/CrowdStrike/crowdstrike-telemetry/_meta/fields.yml b/CrowdStrike/crowdstrike-telemetry/_meta/fields.yml
@@ -1,3 +1,8 @@
+crowdstrike.base_filename:
+  description: Base Filename
+  name: crowdstrike.base_filename
+  type: keyword
+
 crowdstrike.customer_id:
   description: Customer ID (cid)
   name: crowdstrike.customer_id

diff --git a/CrowdStrike/crowdstrike-telemetry/ingest/parser.yml b/CrowdStrike/crowdstrike-telemetry/ingest/parser.yml
@@ -53,6 +53,9 @@ stages:
           "host.domain": "{{parsed_event.message.MachineDomain}}"
           "host.mac": "{{parsed_event.message.MAC}}"
 
+      - set:
+          crowdstrike.base_filename: "{{parsed_event.message.ContextBaseFileName}}"
+
   set_registry_fields:
     actions:
       - set:

diff --git a/CrowdStrike/crowdstrike-telemetry/tests/telemetry_event_26.json b/CrowdStrike/crowdstrike-telemetry/tests/telemetry_event_26.json
@@ -18,6 +18,7 @@
       "id": "111111111111111"
     },
     "crowdstrike": {
+      "base_filename": "svchost.exe",
       "customer_id": "222222222222222222222"
     },
     "file": {

diff --git a/HarfangLab/harfanglab/ingest/parser.yml b/HarfangLab/harfanglab/ingest/parser.yml
@@ -277,7 +277,6 @@ stages:
           event.type: ["info"]
           event.provider: "{{json_event.message.source_name}}"
           event.code: "{{json_event.message.event_id|string}}"
-          source.ip: "{{json_event.message.event_data.IpAddress}}"
           action.id: "{{json_event.message.event_id}}"
           user.name: "{{json_event.message.event_data.SubjectUserName}}"
           user.domain: "{{json_event.message.event_data.SubjectDomainName}}"
@@ -475,6 +474,10 @@ stages:
           action.properties.TaskContentNew_Command: "{{parse_task_info.message.Task.Actions.Exec.Command}}"
           action.properties.TaskContentNew_Args: "{{parse_task_info.message.Task.Actions.Exec.Arguments}}"
 
+      - set:
+          source.ip: "{{json_event.message.event_data.IpAddress}}"
+        filter: "{{json_event.message.event_data.IpAddress | is_ipaddress}}"
+
   authentication_info:
     actions:
       - set:

diff --git a/HarfangLab/harfanglab/tests/authentication_2.json b/HarfangLab/harfanglab/tests/authentication_2.json
@@ -0,0 +1,103 @@
+{
+  "input": {
+    "message": "{\"event_data\": {\"RestrictedAdminMode\": \"-\", \"SubjectUserName\": \"-\", \"SubjectUserSid\": \"S-1-0-0\", \"TargetOutboundUserName\": \"-\", \"ElevatedToken\": \"%%1843\", \"VirtualAccount\": \"%%1843\", \"ProcessId\": \"0x0\", \"AuthenticationPackageName\": \"NTLM\", \"LogonProcessName\": \"NtLmSsp\", \"IpPort\": \"-\", \"WorkstationName\": \"WORKSTATION_NAME\", \"LogonGuid\": \"{00000000-0000-0000-0000-000000000000}\", \"IpAddress\": \"-\", \"TargetLinkedLogonId\": \"0x0\", \"SubjectDomainName\": \"-\", \"TargetOutboundDomainName\": \"-\", \"ImpersonationLevel\": \"%%1833\", \"SubjectLogonId\": \"0x0\", \"TargetLogonId\": \"0x6accabcc3\", \"LogonType\": \"3\", \"TargetUserSid\": \"S-1-5-21-11111111111-111111111111-11111111-111\", \"LmPackageName\": \"NTLM V2\", \"TargetUserName\": \"johndoe\", \"TransmittedServices\": \"-\", \"TargetDomainName\": \"EXAMPLE\", \"ProcessName\": \"-\", \"KeyLength\": \"128\"}, \"groups\": [], \"type\": \"wineventlog\", \"computer_name\": \"example.local\", \"destination\": \"syslog\", \"record_number\": 177355019, \"@Version\": \"1\", \"log_name\": \"Security\", \"@event_create_date\": \"2024-11-05T11:10:19.543Z\", \"level\": \"log_always\", \"timestamp\": \"2024-11-05T11:10:20.274688148Z\", \"process_id\": 704, \"user_data\": {}, \"log_type\": \"eventlog\", \"keywords\": [\"AuditSuccess\", \"ReservedKeyword63\"], \"user\": {\"domain\": \"\", \"identifier\": \"\", \"name\": \"\", \"type\": \"unknown\"}, \"tenant\": \"11111111111111111111\", \"thread_id\": 9168, \"agent\": {\"dnsdomainname\": \"example.local\", \"osproducttype\": \"Windows Server 2022 Datacenter\", \"domain\": null, \"osversion\": \"10.0.20348\", \"ostype\": \"windows\", \"distroid\": null, \"domainname\": \"EXAMPLE\", \"additional_info\": {}, \"version\": \"4.1.6\", \"hostname\": \"EXAMPLE\", \"agentid\": \"555555555-9999-9999-9999-3e333333cccc\"}, \"event_id\": 4624, \"provider_guid\": \"555555555-9999-9999-9999-3e333333cccc\", \"source_name\": \"Microsoft-Windows-Security-Auditing\"}"
+  },
+  "expected": {
+    "message": "{\"event_data\": {\"RestrictedAdminMode\": \"-\", \"SubjectUserName\": \"-\", \"SubjectUserSid\": \"S-1-0-0\", \"TargetOutboundUserName\": \"-\", \"ElevatedToken\": \"%%1843\", \"VirtualAccount\": \"%%1843\", \"ProcessId\": \"0x0\", \"AuthenticationPackageName\": \"NTLM\", \"LogonProcessName\": \"NtLmSsp\", \"IpPort\": \"-\", \"WorkstationName\": \"WORKSTATION_NAME\", \"LogonGuid\": \"{00000000-0000-0000-0000-000000000000}\", \"IpAddress\": \"-\", \"TargetLinkedLogonId\": \"0x0\", \"SubjectDomainName\": \"-\", \"TargetOutboundDomainName\": \"-\", \"ImpersonationLevel\": \"%%1833\", \"SubjectLogonId\": \"0x0\", \"TargetLogonId\": \"0x6accabcc3\", \"LogonType\": \"3\", \"TargetUserSid\": \"S-1-5-21-11111111111-111111111111-11111111-111\", \"LmPackageName\": \"NTLM V2\", \"TargetUserName\": \"johndoe\", \"TransmittedServices\": \"-\", \"TargetDomainName\": \"EXAMPLE\", \"ProcessName\": \"-\", \"KeyLength\": \"128\"}, \"groups\": [], \"type\": \"wineventlog\", \"computer_name\": \"example.local\", \"destination\": \"syslog\", \"record_number\": 177355019, \"@Version\": \"1\", \"log_name\": \"Security\", \"@event_create_date\": \"2024-11-05T11:10:19.543Z\", \"level\": \"log_always\", \"timestamp\": \"2024-11-05T11:10:20.274688148Z\", \"process_id\": 704, \"user_data\": {}, \"log_type\": \"eventlog\", \"keywords\": [\"AuditSuccess\", \"ReservedKeyword63\"], \"user\": {\"domain\": \"\", \"identifier\": \"\", \"name\": \"\", \"type\": \"unknown\"}, \"tenant\": \"11111111111111111111\", \"thread_id\": 9168, \"agent\": {\"dnsdomainname\": \"example.local\", \"osproducttype\": \"Windows Server 2022 Datacenter\", \"domain\": null, \"osversion\": \"10.0.20348\", \"ostype\": \"windows\", \"distroid\": null, \"domainname\": \"EXAMPLE\", \"additional_info\": {}, \"version\": \"4.1.6\", \"hostname\": \"EXAMPLE\", \"agentid\": \"555555555-9999-9999-9999-3e333333cccc\"}, \"event_id\": 4624, \"provider_guid\": \"555555555-9999-9999-9999-3e333333cccc\", \"source_name\": \"Microsoft-Windows-Security-Auditing\"}",
+    "event": {
+      "action": "authentication_network",
+      "category": [
+        "authentication"
+      ],
+      "code": "4624",
+      "dataset": "eventlog",
+      "provider": "Microsoft-Windows-Security-Auditing",
+      "type": [
+        "info",
+        "start"
+      ]
+    },
+    "@timestamp": "2024-11-05T11:10:19.543000Z",
+    "action": {
+      "id": 4624,
+      "outcome": "success",
+      "properties": {
+        "AuthenticationPackageName": "NTLM",
+        "ElevatedToken": "%%1843",
+        "ImpersonationLevel": "%%1833",
+        "KeyLength": "128",
+        "LmPackageName": "NTLM V2",
+        "LogonGuid": "{00000000-0000-0000-0000-000000000000}",
+        "LogonProcessName": "NtLmSsp",
+        "LogonType": "3",
+        "ProcessId": "0x0",
+        "SubjectLogonId": "0x0",
+        "SubjectUserSid": "S-1-0-0",
+        "TargetDomainName": "EXAMPLE",
+        "TargetLinkedLogonId": "0x0",
+        "TargetLogonId": "0x6accabcc3",
+        "TargetUserName": "johndoe",
+        "TargetUserSid": "S-1-5-21-11111111111-111111111111-11111111-111",
+        "VirtualAccount": "%%1843",
+        "WorkstationName": "WORKSTATION_NAME"
+      }
+    },
+    "agent": {
+      "id": "555555555-9999-9999-9999-3e333333cccc",
+      "name": "harfanglab"
+    },
+    "harfanglab": {
+      "groups": []
+    },
+    "host": {
+      "domain": "EXAMPLE",
+      "hostname": "EXAMPLE",
+      "name": "EXAMPLE",
+      "os": {
+        "full": "Windows Server 2022 Datacenter",
+        "version": "10.0.20348"
+      }
+    },
+    "log": {
+      "hostname": "EXAMPLE"
+    },
+    "organization": {
+      "id": "11111111111111111111"
+    },
+    "related": {
+      "hosts": [
+        "EXAMPLE"
+      ]
+    },
+    "sekoiaio": {
+      "authentication": {
+        "process": {
+          "name": "NtLmSsp"
+        }
+      },
+      "client": {
+        "name": "WORKSTATION_NAME",
+        "os": {
+          "type": "windows"
+        }
+      },
+      "server": {
+        "name": "EXAMPLE",
+        "os": {
+          "type": "windows"
+        }
+      }
+    },
+    "server": {
+      "domain": "EXAMPLE"
+    },
+    "user": {
+      "id": "S-1-0-0",
+      "target": {
+        "domain": "EXAMPLE",
+        "id": "S-1-5-21-11111111111-111111111111-11111111-111",
+        "name": "johndoe"
+      }
+    }
+  }
+}
diff --git a/Microsoft/microsoft-365-defender/_meta/fields.yml b/Microsoft/microsoft-365-defender/_meta/fields.yml
@@ -777,6 +777,11 @@ microsoft.defender.observer.interface.type:
   name: microsoft.defender.observer.interface.type
   type: keyword
 
+microsoft.defender.operation.properties:
+  description: Additional properties of the operation
+  name: microsoft.defender.operation.properties
+  type: object
+
 microsoft.defender.report.id:
   description: Unique identifier for the event
   name: microsoft.defender.report.id

diff --git a/Microsoft/microsoft-365-defender/ingest/parser.yml b/Microsoft/microsoft-365-defender/ingest/parser.yml
@@ -259,6 +259,21 @@ stages:
       - set:
           network.protocol: "{{json_event.message.properties.RequestProtocol or json_event.message.properties.Protocol}}"
         filter: '{{json_event.message.properties.get("RequestProtocol") != None or (json_event.message.properties.get("Protocol") != None and json_event.message.properties.Protocol != "Negotiate")}}'
+      - set:
+          microsoft.defender.operation.properties: >
+            {
+              {%- for property in json_event.message.properties.RawEventData.OperationProperties -%}
+                {%- if property.Value != null -%}
+                  {%- if property.Value | from_json == None -%}
+                    "{{property.Name}}":"{{property.Value}}",
+                  {%- else -%}
+                    "{{property.Name}}": {{property.Value | from_json}},
+                  {%- endif -%}
+                {%- endif -%}
+              {%- endfor -%}
+            }
+        filter: '{{json_event.message.properties.RawEventData.get("OperationProperties") != None}}'
+
   set_alert_evidence_fields:
     actions:
       - set:

diff --git a/Microsoft/microsoft-365-defender/tests/test_cloud_app3.json b/Microsoft/microsoft-365-defender/tests/test_cloud_app3.json
@@ -84,6 +84,12 @@
           ],
           "type": "Run"
         },
+        "operation": {
+          "properties": {
+            "IsThrottled": "False",
+            "MailAccessType": "Bind"
+          }
+        },
         "report": {
           "id": "98261974_20893_f747c19c-0664-45c8-aac9-8f16e7714de1"
         }

diff --git a/OCSF/ocsf/ingest/parser.yml b/OCSF/ocsf/ingest/parser.yml
@@ -568,9 +568,11 @@ stages:
           destination.domain: "{{ parse_event.message.dst_endpoint.hostname }}"
         filter: "{{ parse_event.message.dst_endpoint.get('hostname') != None }}"
       - set:
-          destination.ip: "{{ parse_event.message.dst_endpoint.ip }}"
           destination.mac: "{{ parse_event.message.dst_endpoint.mac }}"
           destination.port: "{{ parse_event.message.dst_endpoint.port }}"
+      - set:
+          destination.ip: "{{ parse_event.message.dst_endpoint.ip }}"
+        filter: "{{ parse_event.message.dst_endpoint.ip | is_ipaddress }}"
       - set:
           network.application: "{{ parse_event.message.dst_endpoint.svc_name }}"
         filter: "{{ parse_event.message.dst_endpoint.get('svc_name') != None }}"

diff --git a/OCSF/ocsf/tests/test_network_activity_7.json b/OCSF/ocsf/tests/test_network_activity_7.json
@@ -0,0 +1,42 @@
+{
+  "input": {
+    "message": "{\"metadata\":{\"product\":{\"version\":\"5\",\"name\":\"Amazon VPC\",\"feature\":{\"name\":\"Flowlogs\"},\"vendor_name\":\"AWS\"},\"profiles\":[\"cloud\",\"security_control\",\"datetime\"],\"version\":\"1.1.0\"},\"cloud\":{\"account\":{\"uid\":\"111111111111\"},\"region\":\"eu-west-3\",\"zone\":\"euw3-az1\",\"provider\":\"AWS\"},\"src_endpoint\":{\"port\":null,\"svc_name\":\"-\",\"ip\":\"-\",\"intermediate_ips\":null,\"interface_uid\":\"eni-11111111111111111\",\"vpc_uid\":\"vpc-11111111111111111\",\"instance_uid\":\"-\",\"subnet_uid\":\"subnet-11111111111111111\"},\"dst_endpoint\":{\"port\":null,\"svc_name\":\"-\",\"ip\":\"-\",\"intermediate_ips\":null,\"interface_uid\":null,\"vpc_uid\":null,\"instance_uid\":null,\"subnet_uid\":null},\"connection_info\":{\"protocol_num\":null,\"tcp_flags\":null,\"protocol_ver\":\"-\",\"boundary_id\":99,\"boundary\":null,\"direction_id\":99,\"direction\":\"-\"},\"traffic\":null,\"time\":1731529427000,\"time_dt\":1731529427000,\"start_time_dt\":1731529427000,\"end_time_dt\":1731529458000,\"status_code\":\"NODATA\",\"severity_id\":1,\"severity\":\"Informational\",\"class_name\":\"Network Activity\",\"class_uid\":4001,\"category_name\":\"Network Activity\",\"category_uid\":4,\"activity_name\":\"Unknown\",\"activity_id\":0,\"action\":\"-\",\"action_id\":99,\"disposition\":\"-\",\"type_uid\":400100,\"type_name\":\"Network Activity: Unknown\",\"accountid\":null,\"region\":null,\"asl_version\":null,\"unmapped\":[[\"sublocation_id\",\"-\"],[\"sublocation_type\",\"-\"]],\"observables\":null}\n",
+    "sekoiaio": {
+      "intake": {
+        "dialect": "OCSF [BETA]",
+        "dialect_uuid": "a9c959ac-78ec-47a4-924e-8156a77cebf5"
+      }
+    }
+  },
+  "expected": {
+    "message": "{\"metadata\":{\"product\":{\"version\":\"5\",\"name\":\"Amazon VPC\",\"feature\":{\"name\":\"Flowlogs\"},\"vendor_name\":\"AWS\"},\"profiles\":[\"cloud\",\"security_control\",\"datetime\"],\"version\":\"1.1.0\"},\"cloud\":{\"account\":{\"uid\":\"111111111111\"},\"region\":\"eu-west-3\",\"zone\":\"euw3-az1\",\"provider\":\"AWS\"},\"src_endpoint\":{\"port\":null,\"svc_name\":\"-\",\"ip\":\"-\",\"intermediate_ips\":null,\"interface_uid\":\"eni-11111111111111111\",\"vpc_uid\":\"vpc-11111111111111111\",\"instance_uid\":\"-\",\"subnet_uid\":\"subnet-11111111111111111\"},\"dst_endpoint\":{\"port\":null,\"svc_name\":\"-\",\"ip\":\"-\",\"intermediate_ips\":null,\"interface_uid\":null,\"vpc_uid\":null,\"instance_uid\":null,\"subnet_uid\":null},\"connection_info\":{\"protocol_num\":null,\"tcp_flags\":null,\"protocol_ver\":\"-\",\"boundary_id\":99,\"boundary\":null,\"direction_id\":99,\"direction\":\"-\"},\"traffic\":null,\"time\":1731529427000,\"time_dt\":1731529427000,\"start_time_dt\":1731529427000,\"end_time_dt\":1731529458000,\"status_code\":\"NODATA\",\"severity_id\":1,\"severity\":\"Informational\",\"class_name\":\"Network Activity\",\"class_uid\":4001,\"category_name\":\"Network Activity\",\"category_uid\":4,\"activity_name\":\"Unknown\",\"activity_id\":0,\"action\":\"-\",\"action_id\":99,\"disposition\":\"-\",\"type_uid\":400100,\"type_name\":\"Network Activity: Unknown\",\"accountid\":null,\"region\":null,\"asl_version\":null,\"unmapped\":[[\"sublocation_id\",\"-\"],[\"sublocation_type\",\"-\"]],\"observables\":null}\n",
+    "event": {
+      "action": "unknown",
+      "category": [
+        "network"
+      ],
+      "end": "2024-11-13T20:24:18Z",
+      "kind": "event",
+      "severity": 1,
+      "start": "2024-11-13T20:23:47Z",
+      "type": [
+        "info"
+      ]
+    },
+    "@timestamp": "2024-11-13T20:23:47Z",
+    "cloud": {
+      "account": {
+        "id": "111111111111"
+      },
+      "availability_zone": "euw3-az1",
+      "provider": "AWS",
+      "region": "eu-west-3"
+    },
+    "ocsf": {
+      "activity_id": 0,
+      "activity_name": "Unknown",
+      "class_name": "Network Activity",
+      "class_uid": 4001
+    }
+  }
+}
diff --git a/Office 365/o365/_meta/fields.yml b/Office 365/o365/_meta/fields.yml
@@ -408,6 +408,11 @@ office365.logon_error:
   name: office365.logon_error
   type: keyword
 
+office365.operation.properties:
+  description: A list of objects describing the operation
+  name: office365.operation.properties
+  type: object
+
 office365.record_type:
   description: The type of the operation
   name: office365.record_type

diff --git a/Office 365/o365/ingest/parser.yml b/Office 365/o365/ingest/parser.yml
@@ -118,6 +118,21 @@ stages:
           office365.audit.object_id: "{{json_event.message.ObjectId}}"
           office365.virus_info: "{{json_event.message.VirusInfo}}"
           office365.virus_vendor: "{{json_event.message.VirusVendor}}"
+      - set:
+          office365.operation.properties: >
+            {
+              {%- for property in json_event.message.OperationProperties -%}
+                {%- if property.Value != null -%}
+                  {%- if property.Value | from_json == None -%}
+                    "{{property.Name}}":"{{property.Value}}",
+                  {%- else -%}
+                    "{{property.Name}}": {{property.Value | from_json}},
+                  {%- endif -%}
+                {%- endif -%}
+              {%- endfor -%}
+            }
+        filter: '{{json_event.message.get("OperationProperties") != None}}'
+
       - translate:
         dictionary:
           0: "Regular"
@@ -164,6 +179,10 @@ stages:
         filter: '{{json_event.message.get("Parameters") != None}}'
       - set:
           office365.context.aad_session_id: "{{json_event.message.SessionId}}"
+          office365.context.client.id: "{{json_event.message.ClientAppId}}"
+      - set:
+          office365.context.client.id: "{{json_event.message.AppId}}"
+        filter: '{{json_event.message.get("ClientAppId") == ""}}'
 
   parse_exchange_item:
     actions:
@@ -189,6 +208,7 @@ stages:
           user.id: "{{json_event.message.LogonUserSid}}"
           office365.exchange.mailbox_guid: "{{json_event.message.MailboxGuid}}"
           office365.context.aad_session_id: "{{json_event.message.SessionId}}"
+          office365.context.client.id: "{{json_event.message.ClientAppId}}"
       - set:
           email.subject: "{{json_event.message.Item.Subject}}"
           email.message_id: "{{json_event.message.Item.InternetMessageId[1:-1]}}"
@@ -237,6 +257,7 @@ stages:
             ]
       - set:
           office365.context.aad_session_id: "{{json_event.message.SessionId}}"
+          office365.context.client.id: "{{json_event.message.ClientAppId}}"
   parse_share_point:
     actions:
       - set:
@@ -254,6 +275,7 @@ stages:
     actions:
       - set:
           office365.context.aad_session_id: "{{json_event.message.SessionId}}"
+          office365.context.client.id: "{{json_event.message.ClientAppId}}"
 
   parse_network_traffic:
     actions:

diff --git a/Office 365/o365/tests/clientipadress.json b/Office 365/o365/tests/clientipadress.json
@@ -23,6 +23,17 @@
       "target": "user"
     },
     "office365": {
+      "context": {
+        "client": {
+          "id": "clientappidxxxx-xxx-xxx-xxxx"
+        }
+      },
+      "operation": {
+        "properties": {
+          "IsThrottled": "False",
+          "MailAccessType": "Bind"
+        }
+      },
       "record_type": 50,
       "result_status": "Succeeded",
       "user_type": {

diff --git a/Office 365/o365/tests/exchange_item_aggregated.json b/Office 365/o365/tests/exchange_item_aggregated.json
@@ -20,6 +20,12 @@
       "context": {
         "aad_session_id": "dcdad6b2-f279-48c6-9ed8-3df0ffde4ece"
       },
+      "operation": {
+        "properties": {
+          "IsThrottled": "False",
+          "MailAccessType": "Bind"
+        }
+      },
       "record_type": 50,
       "result_status": "Succeeded",
       "user_type": {

diff --git a/Office 365/o365/tests/exchange_item_update.json b/Office 365/o365/tests/exchange_item_update.json
@@ -29,6 +29,11 @@
       "subject": "HI"
     },
     "office365": {
+      "context": {
+        "client": {
+          "id": "037fd006-a72b-49ae-4bb0-08dba30c8729"
+        }
+      },
       "exchange": {
         "mailbox_guid": "8208550a-4001-439d-a9f6-e95d76767507"
       },

diff --git a/Office 365/o365/tests/inbox_rule.json b/Office 365/o365/tests/inbox_rule.json
@@ -21,7 +21,10 @@
         "object_id": "EURPR07A010.PROD.OUTLOOK.COM/Microsoft Exchange Hosted Organizations/example.onmicrosoft.com/bc1b1df3-f861-4aec-bf7c-40ce5b5566c1\\RULE_NAME"
       },
       "context": {
-        "aad_session_id": "984c0958-0631-4b90-b116-15094fc36847"
+        "aad_session_id": "984c0958-0631-4b90-b116-15094fc36847",
+        "client": {
+          "id": "00000002-0000-0ff1-ce00-000000000000"
+        }
       },
       "exchange_admin": {
         "parameters": [