Merge branch 'main' into lv/ocsf_add_support_of_1_3

SEKOIA-IO · Dec 13, 2024 · 00c6ff6 · 00c6ff6
2 parents 43aec55 + 807b5c0
commit 00c6ff6
Show file tree

Hide file tree

Showing 151 changed files with 2,694 additions and 93 deletions.
diff --git a/AWS/aws-cloudtrail/ingest/parser.yml b/AWS/aws-cloudtrail/ingest/parser.yml
@@ -14,6 +14,7 @@ pipeline:
     external:
       name: grok.match
       properties:
+        raise_errors: false
         input_field: json_event.message.sourceIPAddress
         output_field: source
         pattern: "(%{IP:ip}|%{HOSTNAME:domain})"

diff --git a/AWS/aws-guardduty/ingest/parser.yml b/AWS/aws-guardduty/ingest/parser.yml
@@ -13,6 +13,7 @@ pipeline:
     external:
       name: grok.match
       properties:
+        raise_errors: false
         input_field: json_event.message.type
         output_field: finding
         pattern: "%{DATA:threat_purpose}:%{DATA:affected_resource_type}/%{WORD:threat_family_name}(.%{DATA:detection_mecanism})?(!%{DATA:artifact})?"

diff --git a/Azure/azure-network-watcher/ingest/parser.yml b/Azure/azure-network-watcher/ingest/parser.yml
@@ -9,6 +9,7 @@ pipeline:
     external:
       name: grok.match
       properties:
+        raise_errors: false
         input_field: "{{json_event.message.get('flow.0')}}"
         output_field: result
         pattern: "%{NUMBER:timestamp},%{IP:source_ip},%{IP:destination_ip},%{NUMBER:source_port},%{NUMBER:destination_port},%{PROTOCOL:protocol},%{TRAFFICFLOW:traffic_flow},%{TRAFFICDECISION:traffic_decision}(|,(%{FLOWSTATE:flow_state}|),(%{INT:source_packets}|),(%{INT:source_bytes}|),(%{INT:destination_packets}|),(%{INT:destination_bytes}|))"

diff --git a/Azure/azure-windows/ingest/parser.yml b/Azure/azure-windows/ingest/parser.yml
@@ -24,6 +24,7 @@ pipeline:
     external:
       name: grok.match
       properties:
+        raise_errors: false
         input_field: "{{parse_windows_event.message.EventData.SubjectUserName or parse_windows_event.message.EventData.User}}"
         output_field: result
         pattern: "(%{USER_WITH_DOMAIN}|%{GREEDYDATA:user_name})"
@@ -36,6 +37,7 @@ pipeline:
     external:
       name: kv.parse-kv
       properties:
+        raise_errors: false
         input_field: "{{parse_windows_event.message.EventData.Hashes | lower}}"
         output_field: result
         value_sep: "="

diff --git a/Beats/winlogbeat/ingest/parser.yml b/Beats/winlogbeat/ingest/parser.yml
@@ -10,6 +10,7 @@ pipeline:
     external:
       name: kv.parse-kv
       properties:
+        raise_errors: false
         input_field: "{{json.event.winlog.event_data.Hashes}}"
         output_field: hash
         value_sep: "="

diff --git a/CatoNetworks/cato-sase/ingest/parser.yml b/CatoNetworks/cato-sase/ingest/parser.yml
@@ -19,6 +19,7 @@ pipeline:
     external:
       name: grok.match
       properties:
+        raise_errors: false
         input_field: "{{json_event.output.mitre_attack_tactics}}"
         output_field: message
         pattern: '%{DATA:tactic_name_1} \(%{DATA:tactic_id_1}\)\, %{DATA:tactic_name_2} \(%{DATA:tactic_id_2}\)'
@@ -28,6 +29,7 @@ pipeline:
     external:
       name: grok.match
       properties:
+        raise_errors: false
         input_field: "{{json_event.output.mitre_attack_techniques}}"
         output_field: message
         pattern: '%{DATA:technique_name_1} \(%{DATA:technique_id_1}\)\, %{DATA:technique_name_2} \(%{DATA:technique_id_2}\)'

diff --git a/Cisco/cisco-esa/_meta/fields.yml b/Cisco/cisco-esa/_meta/fields.yml
@@ -121,6 +121,11 @@ cisco.esa.url:
   name: cisco.esa.url
   type: keyword
 
+cisco.esa.url_domain:
+  description: ''
+  name: cisco.esa.url_domain
+  type: keyword
+
 email.attachments:
   description: A list of objects describing the attachment files sent along with an
     email message

diff --git a/Cisco/cisco-esa/ingest/parser.yml b/Cisco/cisco-esa/ingest/parser.yml
@@ -32,6 +32,7 @@ pipeline:
     external:
       name: grok.match
       properties:
+        raise_errors: false
         input_field: "{{parsed_event.message.ExternalMsgID}}"
         output_field: message
         pattern: "<%{MESSAGE_ID}>|%{MESSAGE_ID}"
@@ -42,6 +43,7 @@ pipeline:
     external:
       name: grok.match
       properties:
+        raise_errors: false
         input_field: "{{parsed_event.message.duser}}"
         output_field: message
         pattern: "%{GREEDYDATA:duser_name}@%{GREEDYDATA:duser_domain}"
@@ -50,6 +52,7 @@ pipeline:
     external:
       name: grok.match
       properties:
+        raise_errors: false
         input_field: "{{parsed_event.message.suser}}"
         output_field: message
         pattern: "%{GREEDYDATA:suser_name}@%{GREEDYDATA:suser_domain}"
@@ -67,6 +70,7 @@ pipeline:
     external:
       name: dict.parse
       properties:
+        output_field: message
         input_field: >
           {{ parsed_event.message.ESAURLDetails }}
 
@@ -209,11 +213,6 @@ stages:
                 {% endif %}
               {% endif %}
             {%- endfor %}]
-          cisco.esa.url: >-
-            [{% for url, details in dict(json_event_url_details.message).items() %}
-              "{% if details.get('ExpandedUrl') is not none %}{{ details.ExpandedUrl }}{% else %}{{ url }}{% endif %}"
-              {% if not loop.last %},{% endif %}
-            {% endfor %}]
           url.domain: "{{parsed_event.message.EAURLDetails}}"
           cisco.esa.delivery.connection_id: "{{parsed_event.message.ESADCID}}"
           cisco.esa.injection.connection_id: "{{parsed_event.message.ESAICID}}"
@@ -232,6 +231,19 @@ stages:
           cisco.esa.helo.ip: "{{parsed_event.message.ESAHeloIP}}"
         filter: "{{parsed_event.message.ESAHeloIP | is_ipaddress}}"
 
+      - set:
+          cisco.esa.url_domain: >-
+            [{% for url, details in json_event_url_details.message.items() %}
+              {% if details.get('ExpandedUrl') is not none %}"{{url.replace('https://','').replace('http://','').split('/')[0]}}", "{{ details.ExpandedUrl.replace('https://','').replace('http://','').split('/')[0] }}"{% else %}"{{ url.replace('https://','').replace('http://','').split('/')[0] }}"{% endif %}
+              {% if not loop.last %},{% endif %}
+            {% endfor %}]
+          cisco.esa.url: >-
+            [{% for url, details in json_event_url_details.message.items() %}
+              {% if details.get('ExpandedUrl') is not none %}"{{url}}", "{{ details.ExpandedUrl }}"{% else %}"{{ url }}"{% endif %}
+              {% if not loop.last %},{% endif %}
+            {% endfor %}]
+        filter: "{{json_event_url_details.message | length > 0}}"
+
       - set:
           cisco.esa.helo.domain: "{{parsed_event.message.ESAHeloDomain}}"
           cisco.esa.sender_group: "{{parsed_event.message.ESASenderGroup}}"

diff --git a/Cisco/cisco-esa/tests/test_attachments_details.json b/Cisco/cisco-esa/tests/test_attachments_details.json
@@ -58,6 +58,10 @@
         "url": [
           "http://schemas.microsoft.com/office/2004/12/omml",
           "http://www.w3.org/TR/REC-html40"
+        ],
+        "url_domain": [
+          "schemas.microsoft.com",
+          "www.w3.org"
         ]
       }
     },

diff --git a/Cisco/cisco-esa/tests/test_ingest_log2.json b/Cisco/cisco-esa/tests/test_ingest_log2.json
@@ -61,6 +61,10 @@
         "url": [
           "http://mandrill.appc.cisco.com/track/open.php?u=30372747&id=d57275a6c9df40418a90fd977e3bf506",
           "https://bce-demo.appc.cisco.com/sensors/a7b04388-0f6e-11e9-8def-0242ac110002"
+        ],
+        "url_domain": [
+          "bce-demo.appc.cisco.com",
+          "mandrill.appc.cisco.com"
         ]
       }
     },

diff --git a/Cisco/cisco-esa/tests/test_ingest_log5.json b/Cisco/cisco-esa/tests/test_ingest_log5.json
@@ -55,6 +55,13 @@
         "url": [
           "https://facebook.com/u/john.doe",
           "https://tiktok.com",
+          "https://tinyurl.es/tbdra",
+          "www.twitter.com"
+        ],
+        "url_domain": [
+          "facebook.com",
+          "tiktok.com",
+          "tinyurl.es",
           "www.twitter.com"
         ]
       }

diff --git a/Cisco/cisco-esa/tests/test_ingest_log7.json b/Cisco/cisco-esa/tests/test_ingest_log7.json
@@ -54,8 +54,7 @@
           "domain": {
             "age": "30 days (or greater)"
           }
-        },
-        "url": []
+        }
       }
     },
     "email": {

diff --git a/Cisco/cisco-ios/ingest/parser.yml b/Cisco/cisco-ios/ingest/parser.yml
@@ -14,6 +14,7 @@ pipeline:
     external:
       name: grok.match
       properties:
+        raise_errors: false
         input_field: parsed_event.message.description
         pattern: "%{LINEPROTO}|%{LINK}"
         custom_patterns:
@@ -24,6 +25,7 @@ pipeline:
     external:
       name: grok.match
       properties:
+        raise_errors: false
         input_field: parsed_event.message.description
         pattern: "%{SEC_LOGIN_SUCCESS}|%{SYS_LOGIN_FAILURE}|%{SYS_LOGOUT}|%{SYS_TTY_EXPIRE_TIMER}"
         custom_patterns:
@@ -34,6 +36,7 @@ pipeline:
     filter: '{{parsed_event.message.facility in ["SEC_LOGIN", "SYS"]}}'
   - name: parsed_description
     external:
+      raise_errors: false
       name: grok.match
       properties:
         input_field: parsed_event.message.description

diff --git a/Cisco/cisco-nx-os/ingest/parser.yml b/Cisco/cisco-nx-os/ingest/parser.yml
@@ -14,6 +14,7 @@ pipeline:
     external:
       name: grok.match
       properties:
+        raise_errors: false
         input_field: parsed_event.message.description
         pattern: "%{ETHPORT_IF_DOWN}|%{ETHPORT_IF_UP}|%{ETHPORT_IF}|%{ETHPORT_CONTROL}|%{ETHPORT_LAN}|%{ETHPORT_TRANSCEIVER}|%{ETHPORT_CHANNEL}"
         custom_patterns:
@@ -30,6 +31,7 @@ pipeline:
     external:
       name: grok.match
       properties:
+        raise_errors: false
         input_field: parsed_event.message.description
         pattern: "%{PAM_MESSAGE}|%{FILE_OPEN_FAILURE}"
         custom_patterns:
@@ -42,6 +44,7 @@ pipeline:
     external:
       name: grok.match
       properties:
+        raise_errors: false
         input_field: parsed_event.message.description
         pattern: "%{VSHD_CONFIG}|%{VSHD_CMD_EXEC}"
         custom_patterns:
@@ -53,6 +56,7 @@ pipeline:
     external:
       name: grok.match
       properties:
+        raise_errors: false
         input_field: parsed_event.message.description
         pattern: "%{ARP_DUP}"
         custom_patterns:

diff --git a/Citrix/citrix-adc/ingest/parser.yml b/Citrix/citrix-adc/ingest/parser.yml
@@ -90,7 +90,7 @@ pipeline:
           CIPHER_SUITE: '"?"?[\w\-\.]+"?"?'
 
   - name: set_audit_log_fields
-    filter: '{{not original.message.startswith("CEF")}}'
+    filter: '{{not original.message.startswith("CEF") and parse_audit_header.message.type not in ["AAATM"]}}'
 
   - name: set_connection_log_fields
     filter: "{{ parse_audit_header.message.type == 'TCP' }}"
@@ -105,25 +105,29 @@ pipeline:
     filter: "{{ parse_audit_header.message.type == 'SSLLOG' }}"
 
   - name: set_other_log_fields
-    filter: "{{ parse_audit_header.message.type not in ['SSLVPN', 'SSLLOG', 'TCP'] }}"
+    filter: "{{ parse_audit_header.message.type not in ['SSLVPN', 'SSLLOG', 'TCP', 'AAATM'] }}"
 
 stages:
   set_cef_header_fields:
     actions:
       - set:
           event.kind: "alert"
           event.dataset: "alert"
+
       - set:
           observer.vendor: "{{parsed_event.message.DeviceVendor}}"
           observer.product: "{{parsed_event.message.DeviceProduct}}"
           observer.version: "{{parsed_event.message.DeviceVersion}}"
+
       - set:
           source.ip: "{{parsed_event.message.src}}"
           source.port: "{{parsed_event.message.spt}}"
+
       - set:
           event.reason: "{{parsed_event.message.msg}}"
           event.action: "{{parsed_event.message.act}}"
           event.category: ["network"]
+
       - set:
           url.original: "{{parsed_event.message.request}}"
       - set:

diff --git a/Citrix/citrix-adc/tests/test_aaatm.json b/Citrix/citrix-adc/tests/test_aaatm.json
@@ -4,20 +4,12 @@
   },
   "expected": {
     "message": "09/29/2023:07:40:56 GMT ADC 0-PPE-1 : default AAATM Message 1111111111 0 :  \"AAA JSON-PARSE: ns_aaa_json_parser_StartElementHandler: NAME_VAL state, multi valued attribute start 'ConnectionId' seen\"",
-    "event": {
-      "category": [
-        "network"
-      ],
-      "code": "Message",
-      "dataset": "audit_aaatm",
-      "reason": "\"AAA JSON-PARSE: ns_aaa_json_parser_StartElementHandler: NAME_VAL state, multi valued attribute start 'ConnectionId' seen\"",
-      "type": [
-        "connection"
-      ]
-    },
-    "@timestamp": "2023-09-29T07:40:56Z",
-    "observer": {
-      "name": "ADC"
+    "sekoiaio": {
+      "intake": {
+        "parsing_warnings": [
+          "No fields extracted from original event"
+        ]
+      }
     }
   }
 }
diff --git a/CybeReason/malop-json/_meta/manifest.yml b/CybeReason/malop-json/_meta/manifest.yml
@@ -1,7 +1,7 @@
 uuid: 9f89b634-0531-437b-b060-a9d9f2d270db
 name: Cybereason EDR
 slug: cybereason-malop-json
-automation_connector_uuid: ff092b32-68dc-11ee-8c99-0242ac120002
+automation_connector_uuid: 8128d255-22df-4f4c-96af-ca6c1123f4cf
 automation_module_uuid: b96361fb-a01b-4ae7-8927-9622b9ea0acf
 
 description: >-