Retarus - parse sender and timestamp properly

Retarus/retarus_email_security/_meta/smart-descriptions.json

@@ -1,23 +1,40 @@
 [
   {
-    "value": "{retarus.sender} sent an e-mail to {retarus.recipient} with status: {retarus.status} (Message-ID: {retarus.message_id})",
+    "value": "Retarus Email Security analyzed a malware sent by {retarus.sender} to {retarus.recipient} with status: {retarus.status} (detection engine: {retarus.type}, Message-ID: {retarus.message_id})",
     "conditions": [
       {
         "field": "retarus.class",
-        "value": "EVENT"
+        "value": "THREAT"
+      }
+    ]
+  },
+  {
+    "value": "{email.from.address} sent an e-mail to {email.to.address} with status: {retarus.status} (Message-ID: {retarus.message_id})",
+    "conditions": [
+      {
+        "field": "email.from.address"
       },
       {
-        "field": "retarus.type",
-        "value": "MTA"
+        "field": "email.to.address"
       }
     ]
   },
   {
-    "value": "Retarus Email Security analyzed a malware sent by {retarus.sender} to {retarus.recipient} with status: {retarus.status} (detection engine: {retarus.type}, Message-ID: {retarus.message_id})",
+    "value": "An e-mail was sent from {retarus.sender} to {retarus.recipient} with status: {retarus.status} (Message-ID: {retarus.message_id})",
     "conditions": [
       {
-        "field": "retarus.class",
-        "value": "THREAT"
+        "field": "retarus.sender"
+      },
+      {
+        "field": "retarus.recipient"
+      }
+    ]
+  },
+  {
+    "value": "An e-mail was sent to {retarus.recipient} with status: {retarus.status} (Message-ID: {retarus.message_id})",
+    "conditions": [
+      {
+        "field": "retarus.recipient"
       }
     ]
   }

Retarus/retarus_email_security/ingest/parser.yml

@@ -3,6 +3,7 @@ pipeline:
   - name: json_event
     external:
       name: json.parse-json
+
   - name: parse_sender
     external:
       name: grok.match
@@ -12,6 +13,20 @@ pipeline:
         output_field: sender
         pattern: "^%{GREEDYDATA:username}@%{GREEDYDATA:domain}$"
     filter: "{{json_event.message.sender != null and json_event.message.sender != ''}}"
+
+  - name: parse_header_from
+    external:
+      name: grok.match
+      properties:
+        raise_errors: false
+        input_field: json_event.message.metaData.header.from
+        output_field: message
+        pattern: ".*?<?%{EMAILADDRESS:email}>?.*"
+        custom_patterns:
+          EMAILADDRESSPART: '[a-zA-Z0-9_\.+-=:]+'
+          EMAILADDRESS: "%{EMAILADDRESSPART:local}@%{EMAILADDRESSPART:remote}"
+    filter: "{{json_event.message.get('metaData', {}).get('header', {}).get('from') != None}}"
+
   - name: parse_recipient
     external:
       name: grok.match
@@ -21,9 +36,12 @@ pipeline:
         output_field: recipient
         pattern: "^%{GREEDYDATA:username}@%{GREEDYDATA:domain}$"
     filter: "{{json_event.message.recipient != null}}"
+
   - name: field_extraction
+
   - name: hash_extraction
     filter: "{{final.retarus.class == 'THREAT'}}"
+
 stages:
   field_extraction:
     actions:
@@ -37,6 +55,8 @@ stages:
           observer.version: "{{json_event.message.version}}"
           observer.hostname: "{{json_event.message.host}}"
 
+          "@timestamp": "{{json_event.message.ts | to_rfc3339}}"
+
           organization.id: "{{json_event.message.customer}}"
           source.domain: "{{parse_sender.sender.domain}}"
           destination.domain: "{{parse_recipient.recipient.domain}}"
@@ -45,8 +65,10 @@ stages:
           "retarus.spf.record": "{{json_event.message.metaData.authentication.spf.dns}}"
           "retarus.dkim.result": "{{json_event.message.metaData.authentication.dkim.details}}"
 
-          email.from.address: ["{{json_event.message.metaData.header.from}}"]
-          email.sender.address: ["{{json_event.message.recipient}}"]
+          email.from.address: >
+            {%- if parse_header_from.message.email | length > 0 -%}["{{parse_header_from.message.email}}"]{%- endif -%}
+
+          email.to.address: ["{{json_event.message.recipient}}"]
           email.subject: "{{json_event.message.metaData.header.subject}}"
 
           retarus.timestamp: "{{json_event.message.ts}}"

Retarus/retarus_email_security/tests/event.json

@@ -13,6 +13,7 @@
         "info"
       ]
     },
+    "@timestamp": "2021-05-18T14:50:30Z",
     "action": {
       "name": "EVENT",
       "outcome": "success",
@@ -25,12 +26,7 @@
       "top_level_domain": "com"
     },
     "email": {
-      "from": {
-        "address": [
-          "null"
-        ]
-      },
-      "sender": {
+      "to": {
         "address": [
           "[email protected]"
         ]

Retarus/retarus_email_security/tests/event_header_extra.json

@@ -0,0 +1,86 @@
+{
+  "input": {
+    "message": "{\"version\": \"1.0\", \"host\": \"events.retarus.com\", \"customer\": \"CUSTOMER1\", \"class\": \"EVENT\", \"type\": \"MTA\", \"direction\": \"INBOUND\", \"status\": \"ACCEPTED\", \"ts\": \"2024-12-09 14:08:23 +0100\", \"sourceIp\": \"1.2.3.4\", \"recipient\": \"[email protected]\", \"sender\": \"[email protected]\", \"mimeId\": \"<[email protected]>\", \"rmxId\": \"20210711-145842-xxxxxx-xxxxxx-0@mailin27\", \"metaData\": {\"transportEncryption\": {\"requested\": true, \"established\": true, \"protocol\": \"TLSv1.3\", \"cipherSuite\": \"TLS_AES_256_GCM_SHA384 (256/256 bits)\"}, \"contentEncryption\": false, \"authentication\": {\"spf\": {\"status\": \"pass\", \"details\": \"spf=pass smtp.helo=a48-110.smtp-out.amazonses.com smtp.mailfrom=amazonses.com\", \"dns\": \"v=spf1 ip4:5.6.7.8/22 ip4:9.10.11.12/22 -all\"}, \"dkim\": {\"status\": \"pass\", \"details\": \"dkim=pass reason=\\\"good signature\\\" header.d=amazonses.com [email protected]\", \"selector\": \"ug7nbtf4gccmlpwj322ax3p6ow6yfsug\", \"domain\": \"amazonses.com\"}}, \"header\": {\"subject\": \"Exchange Notification: Password Expiration Notice\", \"from\": \"\\\"example.com - System Manager- Auth(am9obi5kb2VAZXhhbXBsZTMuY29t)example.com\\\" <[email protected]>\"}}, \"subtype\": \"INCOMING\"}"
+  },
+  "expected": {
+    "message": "{\"version\": \"1.0\", \"host\": \"events.retarus.com\", \"customer\": \"CUSTOMER1\", \"class\": \"EVENT\", \"type\": \"MTA\", \"direction\": \"INBOUND\", \"status\": \"ACCEPTED\", \"ts\": \"2024-12-09 14:08:23 +0100\", \"sourceIp\": \"1.2.3.4\", \"recipient\": \"[email protected]\", \"sender\": \"[email protected]\", \"mimeId\": \"<[email protected]>\", \"rmxId\": \"20210711-145842-xxxxxx-xxxxxx-0@mailin27\", \"metaData\": {\"transportEncryption\": {\"requested\": true, \"established\": true, \"protocol\": \"TLSv1.3\", \"cipherSuite\": \"TLS_AES_256_GCM_SHA384 (256/256 bits)\"}, \"contentEncryption\": false, \"authentication\": {\"spf\": {\"status\": \"pass\", \"details\": \"spf=pass smtp.helo=a48-110.smtp-out.amazonses.com smtp.mailfrom=amazonses.com\", \"dns\": \"v=spf1 ip4:5.6.7.8/22 ip4:9.10.11.12/22 -all\"}, \"dkim\": {\"status\": \"pass\", \"details\": \"dkim=pass reason=\\\"good signature\\\" header.d=amazonses.com [email protected]\", \"selector\": \"ug7nbtf4gccmlpwj322ax3p6ow6yfsug\", \"domain\": \"amazonses.com\"}}, \"header\": {\"subject\": \"Exchange Notification: Password Expiration Notice\", \"from\": \"\\\"example.com - System Manager- Auth(am9obi5kb2VAZXhhbXBsZTMuY29t)example.com\\\" <[email protected]>\"}}, \"subtype\": \"INCOMING\"}",
+    "event": {
+      "category": [
+        "email"
+      ],
+      "outcome": "success",
+      "type": [
+        "info"
+      ]
+    },
+    "@timestamp": "2024-12-09T13:08:23Z",
+    "action": {
+      "name": "EVENT",
+      "outcome": "success",
+      "outcome_reason": "ACCEPTED"
+    },
+    "destination": {
+      "address": "example.com",
+      "domain": "example.com",
+      "registered_domain": "example.com",
+      "top_level_domain": "com"
+    },
+    "email": {
+      "from": {
+        "address": [
+          "[email protected]"
+        ]
+      },
+      "subject": "Exchange Notification: Password Expiration Notice",
+      "to": {
+        "address": [
+          "[email protected]"
+        ]
+      }
+    },
+    "observer": {
+      "hostname": "events.retarus.com",
+      "product": "Email Security",
+      "vendor": "Retarus",
+      "version": "1.0"
+    },
+    "organization": {
+      "id": "CUSTOMER1"
+    },
+    "related": {
+      "hosts": [
+        "amazonses.com",
+        "events.retarus.com",
+        "example.com"
+      ],
+      "ip": [
+        "1.2.3.4"
+      ]
+    },
+    "retarus": {
+      "class": "EVENT",
+      "dkim": {
+        "result": "dkim=pass reason=\"good signature\" header.d=amazonses.com [email protected]"
+      },
+      "email_direction": "INBOUND",
+      "message_id": "20210711-145842-xxxxxx-xxxxxx-0@mailin27",
+      "mime_message_id": "<[email protected]>",
+      "recipient": "[email protected]",
+      "sender": "[email protected]",
+      "spf": {
+        "record": "v=spf1 ip4:5.6.7.8/22 ip4:9.10.11.12/22 -all",
+        "status": "pass"
+      },
+      "status": "ACCEPTED",
+      "timestamp": "2024-12-09 14:08:23 +0100",
+      "type": "MTA"
+    },
+    "source": {
+      "address": "amazonses.com",
+      "domain": "amazonses.com",
+      "ip": "1.2.3.4",
+      "registered_domain": "amazonses.com",
+      "top_level_domain": "com"
+    }
+  }
+}

Retarus/retarus_email_security/tests/event_inbound.json

@@ -13,6 +13,7 @@
         "info"
       ]
     },
+    "@timestamp": "2021-07-11T12:58:43Z",
     "action": {
       "name": "EVENT",
       "outcome": "success",
@@ -30,12 +31,12 @@
           "[email protected]"
         ]
       },
-      "sender": {
+      "subject": "This is a test mail",
+      "to": {
         "address": [
           "[email protected]"
         ]
-      },
-      "subject": "This is a test mail"
+      }
     },
     "observer": {
       "hostname": "events.retarus.com",

Retarus/retarus_email_security/tests/event_no_sender.json

@@ -13,6 +13,7 @@
         "info"
       ]
     },
+    "@timestamp": "2022-09-12T14:30:58Z",
     "action": {
       "name": "EVENT",
       "outcome": "success",
@@ -25,17 +26,12 @@
       "top_level_domain": "org"
     },
     "email": {
-      "from": {
-        "address": [
-          "MAILER-DAEMON (Mail Delivery System)"
-        ]
-      },
-      "sender": {
+      "subject": "Undelivered Mail Returned to Sender",
+      "to": {
         "address": [
           "[email protected]"
         ]
-      },
-      "subject": "Undelivered Mail Returned to Sender"
+      }
     },
     "observer": {
       "hostname": "events.retarus.com",

Retarus/retarus_email_security/tests/event_outbound.json

@@ -13,6 +13,7 @@
         "info"
       ]
     },
+    "@timestamp": "2021-07-11T12:58:43Z",
     "action": {
       "name": "EVENT",
       "outcome": "success",
@@ -30,12 +31,12 @@
           "[email protected]"
         ]
       },
-      "sender": {
+      "subject": "This is a test mail",
+      "to": {
         "address": [
           "[email protected]"
         ]
-      },
-      "subject": "This is a test mail"
+      }
     },
     "observer": {
       "hostname": "events.retarus.com",

Retarus/retarus_email_security/tests/example1.json

@@ -12,6 +12,7 @@
         "info"
       ]
     },
+    "@timestamp": "2021-10-01T07:00:00Z",
     "destination": {
       "address": "recipientdomain.fr",
       "domain": "recipientdomain.fr",
@@ -21,15 +22,15 @@
     "email": {
       "from": {
         "address": [
-          "sender <sender@senderdomain.fr>"
+          "[email protected]"
         ]
       },
-      "sender": {
+      "subject": "This is a subject",
+      "to": {
         "address": [
           "[email protected]"
         ]
-      },
-      "subject": "This is a subject"
+      }
     },
     "observer": {
       "hostname": "host.fr",

Retarus/retarus_email_security/tests/threat_cx0.json

@@ -14,6 +14,7 @@
         "info"
       ]
     },
+    "@timestamp": "2018-10-16T12:58:18Z",
     "action": {
       "name": "THREAT",
       "outcome": "failure"
@@ -25,12 +26,7 @@
       "top_level_domain": "de"
     },
     "email": {
-      "from": {
-        "address": [
-          "null"
-        ]
-      },
-      "sender": {
+      "to": {
         "address": [
           "[email protected]"
         ]

Retarus/retarus_email_security/tests/threat_multiscan.json

@@ -14,6 +14,7 @@
         "info"
       ]
     },
+    "@timestamp": "2018-10-16T12:58:43Z",
     "action": {
       "name": "THREAT",
       "outcome": "failure"
@@ -25,12 +26,7 @@
       "top_level_domain": "de"
     },
     "email": {
-      "from": {
-        "address": [
-          "null"
-        ]
-      },
-      "sender": {
+      "to": {
         "address": [
           "[email protected]"
         ]

Retarus/retarus_email_security/tests/threat_pzd.json

@@ -14,6 +14,7 @@
         "info"
       ]
     },
+    "@timestamp": "2018-10-16T12:58:56Z",
     "action": {
       "name": "THREAT",
       "outcome": "failure"
@@ -25,12 +26,7 @@
       "top_level_domain": "de"
     },
     "email": {
-      "from": {
-        "address": [
-          "null"
-        ]
-      },
-      "sender": {
+      "to": {
         "address": [
           "[email protected]"
         ]