From deb5582a3d65c27b3b823aeeeb05880a0b975364 Mon Sep 17 00:00:00 2001 From: lvoloshyn-sekoia Date: Mon, 16 Dec 2024 14:15:26 +0200 Subject: [PATCH] Retarus - parse sender properly --- .../retarus_email_security/ingest/parser.yml | 26 +++++- .../retarus_email_security/tests/event.json | 8 +- .../tests/event_header_extra.json | 86 +++++++++++++++++++ .../tests/event_inbound.json | 7 +- .../tests/event_no_sender.json | 12 +-- .../tests/event_outbound.json | 7 +- .../tests/example1.json | 9 +- .../tests/threat_cx0.json | 8 +- .../tests/threat_multiscan.json | 8 +- .../tests/threat_pzd.json | 8 +- .../tests/threat_sandboxing.json | 8 +- 11 files changed, 137 insertions(+), 50 deletions(-) create mode 100644 Retarus/retarus_email_security/tests/event_header_extra.json diff --git a/Retarus/retarus_email_security/ingest/parser.yml b/Retarus/retarus_email_security/ingest/parser.yml index f55fa8ea7..f9e851d6f 100644 --- a/Retarus/retarus_email_security/ingest/parser.yml +++ b/Retarus/retarus_email_security/ingest/parser.yml @@ -3,6 +3,7 @@ pipeline: - name: json_event external: name: json.parse-json + - name: parse_sender external: name: grok.match @@ -12,6 +13,20 @@ pipeline: output_field: sender pattern: "^%{GREEDYDATA:username}@%{GREEDYDATA:domain}$" filter: "{{json_event.message.sender != null and json_event.message.sender != ''}}" + + - name: parse_header_from + external: + name: grok.match + properties: + raise_errors: false + input_field: json_event.message.metaData.header.from + output_field: message + pattern: ".*??.*" + custom_patterns: + EMAILADDRESSPART: '[a-zA-Z0-9_\.+-=:]+' + EMAILADDRESS: "%{EMAILADDRESSPART:local}@%{EMAILADDRESSPART:remote}" + filter: "{{json_event.message.get('metaData', {}).get('header', {}).get('from') != None}}" + - name: parse_recipient external: name: grok.match @@ -21,9 +36,12 @@ pipeline: output_field: recipient pattern: "^%{GREEDYDATA:username}@%{GREEDYDATA:domain}$" filter: "{{json_event.message.recipient != null}}" + - name: field_extraction + - name: hash_extraction filter: "{{final.retarus.class == 'THREAT'}}" + stages: field_extraction: actions: @@ -37,6 +55,8 @@ stages: observer.version: "{{json_event.message.version}}" observer.hostname: "{{json_event.message.host}}" + "@timestamp": "{{json_event.message.ts | to_rfc3339}}" + organization.id: "{{json_event.message.customer}}" source.domain: "{{parse_sender.sender.domain}}" destination.domain: "{{parse_recipient.recipient.domain}}" @@ -45,8 +65,10 @@ stages: "retarus.spf.record": "{{json_event.message.metaData.authentication.spf.dns}}" "retarus.dkim.result": "{{json_event.message.metaData.authentication.dkim.details}}" - email.from.address: ["{{json_event.message.metaData.header.from}}"] - email.sender.address: ["{{json_event.message.recipient}}"] + email.from.address: > + {%- if parse_header_from.message.email | length > 0 -%}["{{parse_header_from.message.email}}"]{%- endif -%} + + email.to.address: ["{{json_event.message.recipient}}"] email.subject: "{{json_event.message.metaData.header.subject}}" retarus.timestamp: "{{json_event.message.ts}}" diff --git a/Retarus/retarus_email_security/tests/event.json b/Retarus/retarus_email_security/tests/event.json index 80ed49be6..374262564 100644 --- a/Retarus/retarus_email_security/tests/event.json +++ b/Retarus/retarus_email_security/tests/event.json @@ -13,6 +13,7 @@ "info" ] }, + "@timestamp": "2021-05-18T14:50:30Z", "action": { "name": "EVENT", "outcome": "success", @@ -25,12 +26,7 @@ "top_level_domain": "com" }, "email": { - "from": { - "address": [ - "null" - ] - }, - "sender": { + "to": { "address": [ "recepient@mail.com" ] diff --git a/Retarus/retarus_email_security/tests/event_header_extra.json b/Retarus/retarus_email_security/tests/event_header_extra.json new file mode 100644 index 000000000..7afe91cc5 --- /dev/null +++ b/Retarus/retarus_email_security/tests/event_header_extra.json @@ -0,0 +1,86 @@ +{ + "input": { + "message": "{\"version\": \"1.0\", \"host\": \"events.retarus.com\", \"customer\": \"CUSTOMER1\", \"class\": \"EVENT\", \"type\": \"MTA\", \"direction\": \"INBOUND\", \"status\": \"ACCEPTED\", \"ts\": \"2024-12-09 14:08:23 +0100\", \"sourceIp\": \"1.2.3.4\", \"recipient\": \"recipient@example.com\", \"sender\": \"sender@amazonses.com\", \"mimeId\": \"\", \"rmxId\": \"20210711-145842-xxxxxx-xxxxxx-0@mailin27\", \"metaData\": {\"transportEncryption\": {\"requested\": true, \"established\": true, \"protocol\": \"TLSv1.3\", \"cipherSuite\": \"TLS_AES_256_GCM_SHA384 (256/256 bits)\"}, \"contentEncryption\": false, \"authentication\": {\"spf\": {\"status\": \"pass\", \"details\": \"spf=pass smtp.helo=a48-110.smtp-out.amazonses.com smtp.mailfrom=amazonses.com\", \"dns\": \"v=spf1 ip4:5.6.7.8/22 ip4:9.10.11.12/22 -all\"}, \"dkim\": {\"status\": \"pass\", \"details\": \"dkim=pass reason=\\\"good signature\\\" header.d=amazonses.com header.i=@amazonses.com\", \"selector\": \"ug7nbtf4gccmlpwj322ax3p6ow6yfsug\", \"domain\": \"amazonses.com\"}}, \"header\": {\"subject\": \"Exchange Notification: Password Expiration Notice\", \"from\": \"\\\"example.com - System Manager- Auth(am9obi5kb2VAZXhhbXBsZTMuY29t)example.com\\\" \"}}, \"subtype\": \"INCOMING\"}" + }, + "expected": { + "message": "{\"version\": \"1.0\", \"host\": \"events.retarus.com\", \"customer\": \"CUSTOMER1\", \"class\": \"EVENT\", \"type\": \"MTA\", \"direction\": \"INBOUND\", \"status\": \"ACCEPTED\", \"ts\": \"2024-12-09 14:08:23 +0100\", \"sourceIp\": \"1.2.3.4\", \"recipient\": \"recipient@example.com\", \"sender\": \"sender@amazonses.com\", \"mimeId\": \"\", \"rmxId\": \"20210711-145842-xxxxxx-xxxxxx-0@mailin27\", \"metaData\": {\"transportEncryption\": {\"requested\": true, \"established\": true, \"protocol\": \"TLSv1.3\", \"cipherSuite\": \"TLS_AES_256_GCM_SHA384 (256/256 bits)\"}, \"contentEncryption\": false, \"authentication\": {\"spf\": {\"status\": \"pass\", \"details\": \"spf=pass smtp.helo=a48-110.smtp-out.amazonses.com smtp.mailfrom=amazonses.com\", \"dns\": \"v=spf1 ip4:5.6.7.8/22 ip4:9.10.11.12/22 -all\"}, \"dkim\": {\"status\": \"pass\", \"details\": \"dkim=pass reason=\\\"good signature\\\" header.d=amazonses.com header.i=@amazonses.com\", \"selector\": \"ug7nbtf4gccmlpwj322ax3p6ow6yfsug\", \"domain\": \"amazonses.com\"}}, \"header\": {\"subject\": \"Exchange Notification: Password Expiration Notice\", \"from\": \"\\\"example.com - System Manager- Auth(am9obi5kb2VAZXhhbXBsZTMuY29t)example.com\\\" \"}}, \"subtype\": \"INCOMING\"}", + "event": { + "category": [ + "email" + ], + "outcome": "success", + "type": [ + "info" + ] + }, + "@timestamp": "2024-12-09T13:08:23Z", + "action": { + "name": "EVENT", + "outcome": "success", + "outcome_reason": "ACCEPTED" + }, + "destination": { + "address": "example.com", + "domain": "example.com", + "registered_domain": "example.com", + "top_level_domain": "com" + }, + "email": { + "from": { + "address": [ + "john.doe@example3.com" + ] + }, + "subject": "Exchange Notification: Password Expiration Notice", + "to": { + "address": [ + "recipient@example.com" + ] + } + }, + "observer": { + "hostname": "events.retarus.com", + "product": "Email Security", + "vendor": "Retarus", + "version": "1.0" + }, + "organization": { + "id": "CUSTOMER1" + }, + "related": { + "hosts": [ + "amazonses.com", + "events.retarus.com", + "example.com" + ], + "ip": [ + "1.2.3.4" + ] + }, + "retarus": { + "class": "EVENT", + "dkim": { + "result": "dkim=pass reason=\"good signature\" header.d=amazonses.com header.i=@amazonses.com" + }, + "email_direction": "INBOUND", + "message_id": "20210711-145842-xxxxxx-xxxxxx-0@mailin27", + "mime_message_id": "", + "recipient": "recipient@example.com", + "sender": "sender@amazonses.com", + "spf": { + "record": "v=spf1 ip4:5.6.7.8/22 ip4:9.10.11.12/22 -all", + "status": "pass" + }, + "status": "ACCEPTED", + "timestamp": "2024-12-09 14:08:23 +0100", + "type": "MTA" + }, + "source": { + "address": "amazonses.com", + "domain": "amazonses.com", + "ip": "1.2.3.4", + "registered_domain": "amazonses.com", + "top_level_domain": "com" + } + } +} \ No newline at end of file diff --git a/Retarus/retarus_email_security/tests/event_inbound.json b/Retarus/retarus_email_security/tests/event_inbound.json index e0a9bc506..72abba398 100644 --- a/Retarus/retarus_email_security/tests/event_inbound.json +++ b/Retarus/retarus_email_security/tests/event_inbound.json @@ -13,6 +13,7 @@ "info" ] }, + "@timestamp": "2021-07-11T12:58:43Z", "action": { "name": "EVENT", "outcome": "success", @@ -30,12 +31,12 @@ "sender@example.com" ] }, - "sender": { + "subject": "This is a test mail", + "to": { "address": [ "xxxxxxx@retarus.de" ] - }, - "subject": "This is a test mail" + } }, "observer": { "hostname": "events.retarus.com", diff --git a/Retarus/retarus_email_security/tests/event_no_sender.json b/Retarus/retarus_email_security/tests/event_no_sender.json index 0e9f81f5c..6a0feb6a1 100644 --- a/Retarus/retarus_email_security/tests/event_no_sender.json +++ b/Retarus/retarus_email_security/tests/event_no_sender.json @@ -13,6 +13,7 @@ "info" ] }, + "@timestamp": "2022-09-12T14:30:58Z", "action": { "name": "EVENT", "outcome": "success", @@ -25,17 +26,12 @@ "top_level_domain": "org" }, "email": { - "from": { - "address": [ - "MAILER-DAEMON (Mail Delivery System)" - ] - }, - "sender": { + "subject": "Undelivered Mail Returned to Sender", + "to": { "address": [ "user@example.org" ] - }, - "subject": "Undelivered Mail Returned to Sender" + } }, "observer": { "hostname": "events.retarus.com", diff --git a/Retarus/retarus_email_security/tests/event_outbound.json b/Retarus/retarus_email_security/tests/event_outbound.json index 6d5a51385..3f42c392a 100644 --- a/Retarus/retarus_email_security/tests/event_outbound.json +++ b/Retarus/retarus_email_security/tests/event_outbound.json @@ -13,6 +13,7 @@ "info" ] }, + "@timestamp": "2021-07-11T12:58:43Z", "action": { "name": "EVENT", "outcome": "success", @@ -30,12 +31,12 @@ "sender@example.com" ] }, - "sender": { + "subject": "This is a test mail", + "to": { "address": [ "xxxxxxx@retarus.de" ] - }, - "subject": "This is a test mail" + } }, "observer": { "hostname": "events.retarus.com", diff --git a/Retarus/retarus_email_security/tests/example1.json b/Retarus/retarus_email_security/tests/example1.json index e92a981f4..deab85b2c 100644 --- a/Retarus/retarus_email_security/tests/example1.json +++ b/Retarus/retarus_email_security/tests/example1.json @@ -12,6 +12,7 @@ "info" ] }, + "@timestamp": "2021-10-01T07:00:00Z", "destination": { "address": "recipientdomain.fr", "domain": "recipientdomain.fr", @@ -21,15 +22,15 @@ "email": { "from": { "address": [ - "sender " + "sender@senderdomain.fr" ] }, - "sender": { + "subject": "This is a subject", + "to": { "address": [ "recipient@recipientdomain.fr" ] - }, - "subject": "This is a subject" + } }, "observer": { "hostname": "host.fr", diff --git a/Retarus/retarus_email_security/tests/threat_cx0.json b/Retarus/retarus_email_security/tests/threat_cx0.json index 9744b0f8b..d3ca47e32 100644 --- a/Retarus/retarus_email_security/tests/threat_cx0.json +++ b/Retarus/retarus_email_security/tests/threat_cx0.json @@ -14,6 +14,7 @@ "info" ] }, + "@timestamp": "2018-10-16T12:58:18Z", "action": { "name": "THREAT", "outcome": "failure" @@ -25,12 +26,7 @@ "top_level_domain": "de" }, "email": { - "from": { - "address": [ - "null" - ] - }, - "sender": { + "to": { "address": [ "xxxxxxx@retarus.de" ] diff --git a/Retarus/retarus_email_security/tests/threat_multiscan.json b/Retarus/retarus_email_security/tests/threat_multiscan.json index 237c24dc3..21d61494d 100644 --- a/Retarus/retarus_email_security/tests/threat_multiscan.json +++ b/Retarus/retarus_email_security/tests/threat_multiscan.json @@ -14,6 +14,7 @@ "info" ] }, + "@timestamp": "2018-10-16T12:58:43Z", "action": { "name": "THREAT", "outcome": "failure" @@ -25,12 +26,7 @@ "top_level_domain": "de" }, "email": { - "from": { - "address": [ - "null" - ] - }, - "sender": { + "to": { "address": [ "xxxxxxx@retarus.de" ] diff --git a/Retarus/retarus_email_security/tests/threat_pzd.json b/Retarus/retarus_email_security/tests/threat_pzd.json index cbc32eb46..cede3bd9c 100644 --- a/Retarus/retarus_email_security/tests/threat_pzd.json +++ b/Retarus/retarus_email_security/tests/threat_pzd.json @@ -14,6 +14,7 @@ "info" ] }, + "@timestamp": "2018-10-16T12:58:56Z", "action": { "name": "THREAT", "outcome": "failure" @@ -25,12 +26,7 @@ "top_level_domain": "de" }, "email": { - "from": { - "address": [ - "null" - ] - }, - "sender": { + "to": { "address": [ "xxxxxxx@retarus.de" ] diff --git a/Retarus/retarus_email_security/tests/threat_sandboxing.json b/Retarus/retarus_email_security/tests/threat_sandboxing.json index c267493db..80d335573 100644 --- a/Retarus/retarus_email_security/tests/threat_sandboxing.json +++ b/Retarus/retarus_email_security/tests/threat_sandboxing.json @@ -14,6 +14,7 @@ "info" ] }, + "@timestamp": "2018-10-16T13:03:43Z", "action": { "name": "THREAT", "outcome": "failure" @@ -25,12 +26,7 @@ "top_level_domain": "de" }, "email": { - "from": { - "address": [ - "null" - ] - }, - "sender": { + "to": { "address": [ "xxxxxxx@retarus.de" ]