From 78a93392d672511ff0b2fcfe0c421bb0eeb43414 Mon Sep 17 00:00:00 2001 From: vg-svitla Date: Fri, 29 Nov 2024 12:44:41 +0200 Subject: [PATCH 1/7] Fix: Infoblox DDI format improvement --- Infoblox/ddi/_meta/fields.yml | 35 +++++++ Infoblox/ddi/ingest/parser.yml | 120 +++++++++++++++++++++-- Infoblox/ddi/tests/query_log_dhcp_1.json | 32 ++++++ Infoblox/ddi/tests/query_log_dhcp_2.json | 32 ++++++ Infoblox/ddi/tests/query_log_dhcp_3.json | 31 ++++++ Infoblox/ddi/tests/query_log_dhcp_4.json | 38 +++++++ Infoblox/ddi/tests/query_log_dhcp_5.json | 33 +++++++ Infoblox/ddi/tests/query_log_dhcp_6.json | 32 ++++++ Infoblox/ddi/tests/query_log_dhcp_7.json | 32 ++++++ Infoblox/ddi/tests/query_log_dhcp_8.json | 27 +++++ Infoblox/ddi/tests/query_log_dns_1.json | 36 +++++++ Infoblox/ddi/tests/query_log_dns_2.json | 33 +++++++ Infoblox/ddi/tests/query_log_dns_3.json | 33 +++++++ Infoblox/ddi/tests/query_log_dns_4.json | 34 +++++++ Infoblox/ddi/tests/query_log_dns_5.json | 34 +++++++ Infoblox/ddi/tests/query_log_dns_6.json | 34 +++++++ Infoblox/ddi/tests/query_log_dns_7.json | 33 +++++++ Infoblox/ddi/tests/query_log_dns_8.json | 33 +++++++ Infoblox/ddi/tests/query_log_dns_9.json | 35 +++++++ 19 files changed, 710 insertions(+), 7 deletions(-) create mode 100644 Infoblox/ddi/tests/query_log_dhcp_1.json create mode 100644 Infoblox/ddi/tests/query_log_dhcp_2.json create mode 100644 Infoblox/ddi/tests/query_log_dhcp_3.json create mode 100644 Infoblox/ddi/tests/query_log_dhcp_4.json create mode 100644 Infoblox/ddi/tests/query_log_dhcp_5.json create mode 100644 Infoblox/ddi/tests/query_log_dhcp_6.json create mode 100644 Infoblox/ddi/tests/query_log_dhcp_7.json create mode 100644 Infoblox/ddi/tests/query_log_dhcp_8.json create mode 100644 Infoblox/ddi/tests/query_log_dns_1.json create mode 100644 Infoblox/ddi/tests/query_log_dns_2.json create mode 100644 Infoblox/ddi/tests/query_log_dns_3.json create mode 100644 Infoblox/ddi/tests/query_log_dns_4.json create mode 100644 Infoblox/ddi/tests/query_log_dns_5.json create mode 100644 Infoblox/ddi/tests/query_log_dns_6.json create mode 100644 Infoblox/ddi/tests/query_log_dns_7.json create mode 100644 Infoblox/ddi/tests/query_log_dns_8.json create mode 100644 Infoblox/ddi/tests/query_log_dns_9.json diff --git a/Infoblox/ddi/_meta/fields.yml b/Infoblox/ddi/_meta/fields.yml index 064d69713..69f6818da 100644 --- a/Infoblox/ddi/_meta/fields.yml +++ b/Infoblox/ddi/_meta/fields.yml @@ -2,3 +2,38 @@ infoblox.ddi.category: description: The logging category of this event. name: infoblox.ddi.category type: keyword + +infoblox.dhcp.circuit_id: + description: The circuit ID. + name: infoblox.dhcp.circuit_id + type: keyword + +infoblox.dhcp.interface_ip: + description: The IP address of the interface. + name: infoblox.dhcp.interface_ip + type: ip + +infoblox.dhcp.lease_message: + description: The lease message. + name: infoblox.dhcp.lease_message + type: keyword + +infoblox.dhcp.lease_time: + description: The lease time. + name: infoblox.dhcp.lease_time + type: keyword + +infoblox.dhcp.request_message: + description: The request message. + name: infoblox.dhcp.request_message + type: keyword + +infoblox.dhcp.router_ip: + description: The IP address of the router. + name: infoblox.dhcp.router_ip + type: ip + +infoblox.dhcp.trans_id: + description: The transaction ID. + name: infoblox.dhcp.trans_id + type: keyword diff --git a/Infoblox/ddi/ingest/parser.yml b/Infoblox/ddi/ingest/parser.yml index 25264fc22..2fb950af6 100644 --- a/Infoblox/ddi/ingest/parser.yml +++ b/Infoblox/ddi/ingest/parser.yml @@ -5,7 +5,7 @@ pipeline: name: grok.match properties: output_field: message - pattern: "%{CLIENT}" + pattern: "%{DNS_FORMERR}|%{DNS_OTHER}|%{DNS_0}|%{DNS_1}|%{DNS_2}|%{DNS_3}|%{DNS_4}|%{DNS_5}|%{DNS_6}|%{DNS_7}|%{DNS_8}|%{DNS_9}|%{DNS_10}|%{DNS_11}|%{DNS_12}|%{DNS_13}" custom_patterns: QUERY_FLAGS: "%{QUERY_FLAGS_RD:flags_rd}%{QUERY_FLAGS_EDNS:flags_edns}?%{QUERY_FLAGS_TCP:flags_tcp}?%{QUERY_FLAGS_DNSSEC:flags_dnssec}?%{QUERY_FLAGS_CD:flags_cd}?%{QUERY_FLAGS_DNS_SERVER_COOKIE}?%{QUERY_FLAGS_DNS_SERVER_COOKIE_WITHOUT_VALID_SERVER}?" QUERY_FLAGS_RD: '[\+\-]' @@ -16,22 +16,128 @@ pipeline: QUERY_FLAGS_CD: "C" QUERY_FLAGS_DNS_SERVER_COOKIE: "V" QUERY_FLAGS_DNS_SERVER_COOKIE_WITHOUT_VALID_SERVER: "K" - CLIENT: '(%{WORD:category}: )?client ?(%{DATA}) %{IP:src}#%{INT:spt} (%{DATA}): query: %{IPORHOST:dns_question_name} %{WORD:dns_question_class} %{WORD:dns_question_type} %{QUERY_FLAGS} \(%{IP}\)' + CLIENT: "client (?:%{DATA} )?%{IP:client_ip}#%{NUMBER:client_port}:?" + VIEW: "view %{DATA:infoblox_nios_log_view}: " + + # Next patterns are inspired by + # https://github.com/elastic/integrations/blob/main/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_dns.yml + DNS_1: "zone %{DATA:dns_question_name}/%{DATA:dns_question_class}: notify from %{IP:client_ip}#%{NUMBER:client_port}:? %{GREEDYDATA:infoblox_nios_log_dns_message}" + DNS_2: "transfer of '%{DATA:dns_question_name}/%{DATA:dns_question_class}' from %{IP:client_ip}#%{NUMBER:client_port}:? %{GREEDYDATA:infoblox_nios_log_dns_message}" + DNS_3: "validating %{DATA:dns_question_name}/%{WORD:dns_question_type}: %{GREEDYDATA:infoblox_nios_log_dns_message}" + DNS_4: "(%{NOTSPACE:infoblox_nios_log_dns_category}:)?\\s*%{CLIENT} updating zone '%{DATA:dns_question_name}/%{DATA:dns_question_class}': %{GREEDYDATA:infoblox_nios_log_dns_message}" + DNS_5: "(%{NOTSPACE:infoblox_nios_log_dns_category}:)?\\s*%{CLIENT} \\(%{DATA}\\): %{VIEW}?query failed %{GREEDYDATA:infoblox_nios_log_dns_message}" + DNS_6: "(%{NOTSPACE:infoblox_nios_log_dns_category}:)?\\s*%{CLIENT} \\(%{DATA:infoblox_nios_log_dns_before_query}\\): rewriting query name %{DATA} to '%{DATA:infoblox_nios_log_dns_after_query}', type %{DATA:dns_question_type}" + DNS_7: "(%{NOTSPACE:infoblox_nios_log_dns_category}:)?\\s*%{CLIENT} \\(%{DATA}\\): %{VIEW}?query: %{DATA:dns_question_name} %{DATA:dns_question_class} %{WORD:dns_question_type} %{DATA:infoblox_nios_log_dns_header_flags} \\(%{IP:server_ip}\\)" + DNS_8: "(%{NOTSPACE:infoblox_nios_log_dns_category}:)?\\s*%{CLIENT} %{DATA:network_transport}: %{VIEW}?query: %{DATA:dns_question_name} %{DATA:dns_question_class} %{WORD:dns_question_type} response: %{DATA:dns_response_code} %{DATA:infoblox_nios_log_dns_header_flags}" + DNS_9: "(%{NOTSPACE:infoblox_nios_log_dns_category}:)?\\s*%{CLIENT} \\(%{DATA}\\): transfer of '%{DATA:dns_question_name}/%{DATA:dns_question_class}': %{GREEDYDATA:infoblox_nios_log_dns_message}" + DNS_10: "(%{NOTSPACE:infoblox_nios_log_dns_category}:)?\\s*CEF:0\\|Infoblox\\|NIOS\\|%{GREEDYDATA:infoblox_nios_log_dns_version}\\|RPZ-%{DATA:dns_answers_type}\\|%{DATA:infoblox_nios_log_dns_answers_policy}\\|\\d+\\|app=DNS dst=%{IP:server_ip} src=%{IP:client_ip} spt=%{NUMBER:client_port} view=%{DATA:infoblox_nios_log_dns_view_name} qtype=%{WORD:dns_question_type} msg=%{GREEDYDATA:infoblox_nios_log_dns_message}" + DNS_11: "(%{NOTSPACE:infoblox_nios_log_dns_category}:)?\\s*%{GREEDYDATA:_tmp_timestamp} %{CLIENT} %{DATA:network_transport}: %{VIEW}?query: %{DATA:dns_question_name} %{DATA:dns_question_class} %{WORD:dns_question_type} response: %{DATA:dns_response_code} %{DATA:infoblox_nios_log_dns_header_flags} %{GREEDYDATA:repeat_message}" + DNS_12: "(%{NOTSPACE:infoblox_nios_log_dns_category}:)?\\s*%{GREEDYDATA:_tmp_timestamp} %{CLIENT} %{DATA:network_transport}: %{VIEW}?query: %{DATA:dns_question_name} %{DATA:dns_question_class} %{WORD:dns_question_type} response: %{DATA:dns_response_code} %{DATA:infoblox_nios_log_dns_header_flags}" + DNS_13: "(%{NOTSPACE:infoblox_nios_log_dns_category}:)?\\s*%{CLIENT} %{GREEDYDATA:infoblox_nios_log_dns_message}" + + # Original pattern + DNS_0: '(%{WORD:infoblox_nios_log_dns_category}: )?client ?(%{DATA}) %{IP:client_ip}#%{INT:client_port} (%{DATA}): query: %{DATA:dns_question_name} %{WORD:dns_question_class} %{WORD:dns_question_type} %{QUERY_FLAGS} \(%{IP}\)' + + # Other patterns + + ## For DNS message like: + ## FORMERR resolving 'test.testing.io/AAAA/IN': 192.168.1.136#53 + DNS_FORMERR: "%{WORD:event_action} resolving '%{DATA:dns_question_name}/%{DATA:dns_question_type}/%{DATA:dns_question_class}': %{IP:client_ip}#%{NUMBER:client_port}" + + ## For other message like: + ## r-l-e:192.168.1.113,Fixed,P76984,c4:d0:e3:b4:08:4d,1732119022,1732291822,,$ + DNS_OTHER: "r-l-e:%{IP:client_ip},%{DATA:infoblox_nios_log_dns_category},%{DATA:infoblox_nios_log_dns_client_hostname},%{MAC:client_mac},%{NUMBER:infoblox_nios_log_dns_lease_start},%{NUMBER:infoblox_nios_log_dns_lease_end},%{GREEDYDATA:infoblox_nios_log_dns_message}" + + - name: parse_event + filter: "{{'REQUEST DHCP' in original.message or 'DHCPREQUEST' in original.message}}" + external: + name: grok.match + properties: + output_field: message + pattern: "%{DHCP_1}|%{DHCP_2}|%{DHCP_3}|%{DHCP_4}|%{DHCP_5}|%{DHCP_6}|%{DHCP_7}|%{DHCP_8}|%{DHCP_9}|%{DHCP_10}|%{DHCP_11}|%{DHCP_12}|%{DHCP_OTHER}" + custom_patterns: + DHCP_1: '%{WORD:event_action} for %{IP:client_ip} \(%{IP:infoblox_nios_log_dhcp_router_ip}\) from %{MAC:client_mac} \(%{DATA:infoblox_nios_log_dhcp_client_hostname}\) via (%{IP:infoblox_nios_log_dhcp_interface_ip}|%{WORD:observer_ingress_interface_name}) TransID %{DATA:infoblox_nios_log_dhcp_trans_id} uid %{DATA:infoblox_nios_log_dhcp_uid} \(%{GREEDYDATA:infoblox_nios_log_dhcp_lease_message}\)' + DHCP_2: '%{WORD:event_action} for %{IP:client_ip} from %{MAC:client_mac} \(%{DATA:infoblox_nios_log_dhcp_client_hostname}\) via (%{IP:infoblox_nios_log_dhcp_interface_ip}|%{WORD:observer_ingress_interface_name}) TransID %{DATA:infoblox_nios_log_dhcp_trans_id} uid %{DATA:infoblox_nios_log_dhcp_uid} \(%{GREEDYDATA:infoblox_nios_log_dhcp_lease_message}\)' + DHCP_3: '%{WORD:event_action} for %{IP:client_ip} from %{MAC:client_mac} \(%{DATA:infoblox_nios_log_dhcp_client_hostname}\) via (%{IP:infoblox_nios_log_dhcp_interface_ip}|%{WORD:observer_ingress_interface_name}) TransID %{DATA:infoblox_nios_log_dhcp_trans_id} uid %{DATA:infoblox_nios_log_dhcp_uid}: %{GREEDYDATA:infoblox_nios_log_dhcp_request_message}' + DHCP_4: '%{WORD:event_action} for %{IP:client_ip} \(%{IP:infoblox_nios_log_dhcp_router_ip}\) from %{MAC:client_mac} \(%{DATA:infoblox_nios_log_dhcp_client_hostname}\) via (%{IP:infoblox_nios_log_dhcp_interface_ip}|%{WORD:observer_ingress_interface_name}) TransID %{DATA:infoblox_nios_log_dhcp_trans_id} uid %{GREEDYDATA:infoblox_nios_log_dhcp_uid}' + DHCP_5: '%{WORD:event_action} for %{IP:client_ip} \(%{IP:infoblox_nios_log_dhcp_router_ip}\) from %{MAC:client_mac} via (%{IP:infoblox_nios_log_dhcp_interface_ip}|%{WORD:observer_ingress_interface_name}) TransID %{DATA:infoblox_nios_log_dhcp_trans_id} \(%{GREEDYDATA:infoblox_nios_log_dhcp_lease_message}\)' + DHCP_6: '%{WORD:event_action} for %{IP:client_ip} from %{MAC:client_mac} via (%{IP:infoblox_nios_log_dhcp_interface_ip}|%{WORD:observer_ingress_interface_name}) TransID %{DATA:infoblox_nios_log_dhcp_trans_id} uid %{DATA:infoblox_nios_log_dhcp_uid} \(%{GREEDYDATA:infoblox_nios_log_dhcp_lease_message}\)' + DHCP_7: '%{WORD:event_action} for %{IP:client_ip} \(%{IP:infoblox_nios_log_dhcp_router_ip}\) from %{MAC:client_mac} via (%{IP:infoblox_nios_log_dhcp_interface_ip}|%{WORD:observer_ingress_interface_name}) TransID %{DATA:infoblox_nios_log_dhcp_trans_id}: %{GREEDYDATA:infoblox_nios_log_dhcp_request_message}' + DHCP_8: "%{WORD:event_action} for %{IP:client_ip} from %{MAC:client_mac} via (%{IP:infoblox_nios_log_dhcp_interface_ip}|%{WORD:observer_ingress_interface_name}) TransID %{DATA:infoblox_nios_log_dhcp_trans_id}: %{GREEDYDATA:infoblox_nios_log_dhcp_request_message}" + DHCP_9: '%{WORD:event_action} for %{IP:client_ip} \(%{IP:infoblox_nios_log_dhcp_router_ip}\) from %{MAC:client_mac} via (%{IP:infoblox_nios_log_dhcp_interface_ip}|%{WORD:observer_ingress_interface_name}) TransID %{GREEDYDATA:infoblox_nios_log_dhcp_trans_id}' + DHCP_10: "%{WORD:event_action} for %{IP:client_ip} from %{MAC:client_mac} via (%{IP:infoblox_nios_log_dhcp_interface_ip}|%{WORD:observer_ingress_interface_name}) TransID %{DATA:infoblox_nios_log_dhcp_trans_id} uid %{GREEDYDATA:infoblox_nios_log_dhcp_uid}" + DHCP_11: "%{WORD:event_action} for %{IP:client_ip} from %{MAC:client_mac} via (%{IP:infoblox_nios_log_dhcp_interface_ip}|%{WORD:observer_ingress_interface_name}) TransID %{GREEDYDATA:infoblox_nios_log_dhcp_trans_id}" + DHCP_12: "%{WORD:event_action} for %{IP:client_ip} from %{MAC:client_mac} via (%{IP:infoblox_nios_log_dhcp_interface_ip}|%{WORD:observer_ingress_interface_name})" + + # Other patterns + + ## For DHCP message like: + ## Option 82: received a REQUEST DHCP packet from relay-agent eth2 with a circuit-id of "1a:02:30:00:00:00:00:76:00:00:00:00:00:00:2a:f0", a remote-id of "0a:44:70:46" for 192.168.1.222 (00:50:56:ae:b3:44) lease time is undefined seconds. (NEW) + DHCP_OTHER: 'Option %{NUMBER}: received a %{DATA:event_action} packet from %{NOTSPACE} %{DATA:infoblox_nios_log_dhcp_relay_interface_name} with a circuit-id of \"%{DATA:infoblox_nios_log_dhcp_circuit_id}\", a remote-id of \"%{DATA:infoblox_nios_log_dhcp_remote_id}\" for %{IP:client_ip} \(%{MAC:client_mac}\) lease time is %{DATA:infoblox_nios_log_dhcp_lease_time} seconds. \(%{DATA:infoblox_nios_log_dhcp_lease_message}\)' + + - name: parse_event + filter: "{{'DHCPACK' in original.message}}" + external: + name: grok.match + properties: + output_field: message + pattern: "%{DHCPACK_1}|%{DHCPACK_2}|%{DHCPACK_3}|%{DHCPACK_4}|%{DHCPACK_5}|%{DHCPACK_6}|%{DHCPACK_7}|%{DHCPACK_8}|%{DHCPACK_9}|%{DHCPACK_10}|%{DHCPACK_11}|%{DHCPACK_12}" + custom_patterns: + # Patterns are inspired by + # https://github.com/elastic/integrations/blob/main/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_dhcp.yml + DHCPACK_1: "%{WORD:event_action} on %{IP:client_ip} to %{MAC:client_mac} \\(%{DATA:infoblox_nios_log_dhcp_client_hostname}\\) via (%{IP:infoblox_nios_log_dhcp_interface_ip}|%{WORD:observer_ingress_interface_name}) relay (%{IP:infoblox_nios_log_dhcp_relay_interface_ip}|%{WORD:infoblox_nios_log_dhcp_relay_interface_name}) lease-duration %{NUMBER:infoblox_nios_log_dhcp_lease_duration} offered-duration %{NUMBER:infoblox_nios_log_dhcp_offered_duration} \\(%{DATA:infoblox_nios_log_dhcp_message}\\) uid %{GREEDYDATA:infoblox_nios_log_dhcp_uid}" + DHCPACK_2: "%{WORD:event_action} on %{IP:client_ip} to %{MAC:client_mac} via (%{IP:infoblox_nios_log_dhcp_interface_ip}|%{WORD:observer_ingress_interface_name}) relay (%{IP:infoblox_nios_log_dhcp_relay_interface_ip}|%{WORD:infoblox_nios_log_dhcp_relay_interface_name}) lease-duration %{NUMBER:infoblox_nios_log_dhcp_lease_duration} offered-duration %{NUMBER:infoblox_nios_log_dhcp_offered_duration} \\(%{DATA:infoblox_nios_log_dhcp_message}\\) uid %{GREEDYDATA:infoblox_nios_log_dhcp_uid}" + DHCPACK_3: "%{WORD:event_action} on %{IP:client_ip} to %{MAC:client_mac} \\(%{DATA:infoblox_nios_log_dhcp_client_hostname}\\) via (%{IP:infoblox_nios_log_dhcp_interface_ip}|%{WORD:observer_ingress_interface_name}) relay (%{IP:infoblox_nios_log_dhcp_relay_interface_ip}|%{WORD:infoblox_nios_log_dhcp_relay_interface_name}) lease-duration %{NUMBER:infoblox_nios_log_dhcp_lease_duration} \\(%{DATA:infoblox_nios_log_dhcp_lease_message}\\) uid %{GREEDYDATA:infoblox_nios_log_dhcp_uid}" + DHCPACK_4: "%{WORD:event_action} on %{IP:client_ip} to %{MAC:client_mac} \\(%{DATA:infoblox_nios_log_dhcp_client_hostname}\\) via (%{IP:infoblox_nios_log_dhcp_interface_ip}|%{WORD:observer_ingress_interface_name}) relay (%{IP:infoblox_nios_log_dhcp_relay_interface_ip}|%{WORD:infoblox_nios_log_dhcp_relay_interface_name}) lease-duration %{NUMBER:infoblox_nios_log_dhcp_lease_duration} offered-duration %{NUMBER:infoblox_nios_log_dhcp_offered_duration} \\(%{DATA:infoblox_nios_log_dhcp_message}\\)" + DHCPACK_5: "%{WORD:event_action} on %{IP:client_ip} to %{MAC:client_mac} via (%{IP:infoblox_nios_log_dhcp_interface_ip}|%{WORD:observer_ingress_interface_name}) relay (%{IP:infoblox_nios_log_dhcp_relay_interface_ip}|%{WORD:infoblox_nios_log_dhcp_relay_interface_name}) lease-duration %{NUMBER:infoblox_nios_log_dhcp_lease_duration} \\(%{DATA:infoblox_nios_log_dhcp_lease_message}\\) uid %{GREEDYDATA:infoblox_nios_log_dhcp_uid}" + DHCPACK_6: "%{WORD:event_action} on %{IP:client_ip} to %{MAC:client_mac} via (%{IP:infoblox_nios_log_dhcp_interface_ip}|%{WORD:observer_ingress_interface_name}) relay (%{IP:infoblox_nios_log_dhcp_relay_interface_ip}|%{WORD:infoblox_nios_log_dhcp_relay_interface_name}) lease-duration %{NUMBER:infoblox_nios_log_dhcp_lease_duration} \\(%{DATA:infoblox_nios_log_dhcp_lease_message}\\)" + DHCPACK_7: "%{WORD:event_action} on %{IP:client_ip} to %{MAC:client_mac} \\(%{DATA:infoblox_nios_log_dhcp_client_hostname}\\) via (%{IP:infoblox_nios_log_dhcp_interface_ip}|%{WORD:observer_ingress_interface_name}) relay (%{IP:infoblox_nios_log_dhcp_relay_interface_ip}|%{WORD:infoblox_nios_log_dhcp_relay_interface_name}) lease-duration %{NUMBER:infoblox_nios_log_dhcp_lease_duration} \\(%{GREEDYDATA:infoblox_nios_log_dhcp_lease_message}\\)" + DHCPACK_8: "%{WORD:event_action} on %{IP:client_ip} to %{MAC:client_mac} \\(%{DATA:infoblox_nios_log_dhcp_client_hostname}\\) via (%{IP:infoblox_nios_log_dhcp_interface_ip}|%{WORD:observer_ingress_interface_name}) relay (%{IP:infoblox_nios_log_dhcp_relay_interface_ip}|%{WORD:infoblox_nios_log_dhcp_relay_interface_name}) lease-duration %{NUMBER:infoblox_nios_log_dhcp_lease_duration} uid %{GREEDYDATA:infoblox_nios_log_dhcp_uid}" + DHCPACK_9: "%{WORD:event_action} on %{IP:client_ip} to %{MAC:client_mac} \\(%{DATA:infoblox_nios_log_dhcp_client_hostname}\\) via (%{IP:infoblox_nios_log_dhcp_interface_ip}|%{WORD:observer_ingress_interface_name}) relay (%{IP:infoblox_nios_log_dhcp_relay_interface_ip}|%{WORD:infoblox_nios_log_dhcp_relay_interface_name}) lease-duration %{GREEDYDATA:infoblox_nios_log_dhcp_lease_duration}" + DHCPACK_10: "%{WORD:event_action} on %{IP:client_ip} to %{MAC:client_mac} via (%{IP:infoblox_nios_log_dhcp_interface_ip}|%{WORD:observer_ingress_interface_name}) relay (%{IP:infoblox_nios_log_dhcp_relay_interface_ip}|%{WORD:infoblox_nios_log_dhcp_relay_interface_name}) lease-duration %{NUMBER:infoblox_nios_log_dhcp_lease_duration} uid %{GREEDYDATA:infoblox_nios_log_dhcp_uid}" + DHCPACK_11: "%{WORD:event_action} on %{IP:client_ip} to %{MAC:client_mac} via (%{IP:infoblox_nios_log_dhcp_interface_ip}|%{WORD:observer_ingress_interface_name}) relay (%{IP:infoblox_nios_log_dhcp_relay_interface_ip}|%{WORD:infoblox_nios_log_dhcp_relay_interface_name}) lease-duration %{GREEDYDATA:infoblox_nios_log_dhcp_lease_duration}" + DHCPACK_12: "%{WORD:event_action} to %{IP:client_ip} \\(%{MAC:client_mac}\\) via %{WORD:observer_ingress_interface_name}" + + - name: parse_datetime + external: + name: date.parse + properties: + input_field: "{{parse_event.message._tmp_timestamp}}" + output_field: result + format: "%d-%b-%Y %H:%M:%S.%f" + - name: set_ecs_fields stages: set_ecs_fields: actions: - set: - source.ip: "{{parse_event.message.src}}" - filter: "{{parse_event.message.src | is_ipaddress}}" + source.ip: "{{parse_event.message.client_ip}}" + filter: "{{parse_event.message.client_ip | is_ipaddress}}" - set: - source.port: "{{parse_event.message.spt}}" + "@timestamp": "{{parse_datetime.result}}" + event.action: "{{parse_event.message.event_action}}" + + source.port: "{{parse_event.message.client_port}}" + source.mac: "{{parse_event.message.client_mac}}" + + observer.ingress.interface.name: "{{parse_event.message.observer_ingress_interface_name}}" + + infoblox.dhcp.interface_ip: "{{parse_event.message.infoblox_nios_log_dhcp_interface_ip}}" + infoblox.dhcp.trans_id: "{{parse_event.message.infoblox_nios_log_dhcp_trans_id}}" + infoblox.dhcp.router_ip: "{{parse_event.message.infoblox_nios_log_dhcp_router_ip}}" + infoblox.dhcp.request_message: "{{parse_event.message.infoblox_nios_log_dhcp_request_message}}" + infoblox.dhcp.lease_message: "{{parse_event.message.infoblox_nios_log_dhcp_lease_message}}" + infoblox.dhcp.lease_time: "{{parse_event.message.infoblox_nios_log_dhcp_lease_time}}" + infoblox.dhcp.circuit_id: "{{parse_event.message.infoblox_nios_log_dhcp_circuit_id}}" + + dns.type: "query" dns.question.class: "{{parse_event.message.dns_question_class}}" dns.question.type: "{{parse_event.message.dns_question_type}}" dns.question.name: "{{parse_event.message.dns_question_name}}" - dns.type: "query" + dns.response_code: "{{parse_event.message.dns_response_code}}" + dns.header_flags: > [ {% if parse_event.message.flags_rd == "+" %}"RD",{% endif %} @@ -42,4 +148,4 @@ stages: filter: '{{parse_event.message.flags_tcp == "T"}}' - set: - infoblox.ddi.category: "{{parse_event.message.category}}" + infoblox.ddi.category: "{{parse_event.message.infoblox_nios_log_dns_category}}" diff --git a/Infoblox/ddi/tests/query_log_dhcp_1.json b/Infoblox/ddi/tests/query_log_dhcp_1.json new file mode 100644 index 000000000..43383a2e1 --- /dev/null +++ b/Infoblox/ddi/tests/query_log_dhcp_1.json @@ -0,0 +1,32 @@ +{ + "input": { + "message": "Option 82: received a REQUEST DHCP packet from relay-agent eth2 with a circuit-id of \"1a:02:30:00:00:00:00:76:00:00:00:00:00:00:2a:f0\", a remote-id of \"0a:44:70:46\" for 192.168.1.222 (00:50:56:ae:b3:44) lease time is undefined seconds. (NEW)" + }, + "expected": { + "message": "Option 82: received a REQUEST DHCP packet from relay-agent eth2 with a circuit-id of \"1a:02:30:00:00:00:00:76:00:00:00:00:00:00:2a:f0\", a remote-id of \"0a:44:70:46\" for 192.168.1.222 (00:50:56:ae:b3:44) lease time is undefined seconds. (NEW)", + "event": { + "action": "REQUEST DHCP" + }, + "dns": { + "header_flags": [], + "type": "query" + }, + "infoblox": { + "dhcp": { + "circuit_id": "1a:02:30:00:00:00:00:76:00:00:00:00:00:00:2a:f0", + "lease_message": "NEW", + "lease_time": "undefined" + } + }, + "related": { + "ip": [ + "192.168.1.222" + ] + }, + "source": { + "address": "192.168.1.222", + "ip": "192.168.1.222", + "mac": "00:50:56:ae:b3:44" + } + } +} \ No newline at end of file diff --git a/Infoblox/ddi/tests/query_log_dhcp_2.json b/Infoblox/ddi/tests/query_log_dhcp_2.json new file mode 100644 index 000000000..9474b00d8 --- /dev/null +++ b/Infoblox/ddi/tests/query_log_dhcp_2.json @@ -0,0 +1,32 @@ +{ + "input": { + "message": "Option 82: received a REQUEST DHCP packet from relay-agent 192.168.1.53 with a circuit-id of \"1a:02:30:00:00:00:00:76:00:00:00:00:00:00:2a:f0\", a remote-id of \"0a:44:70:46\" for 192.168.1.53 (00:50:56:ae:b3:44) lease time is undefined seconds. (NEW)" + }, + "expected": { + "message": "Option 82: received a REQUEST DHCP packet from relay-agent 192.168.1.53 with a circuit-id of \"1a:02:30:00:00:00:00:76:00:00:00:00:00:00:2a:f0\", a remote-id of \"0a:44:70:46\" for 192.168.1.53 (00:50:56:ae:b3:44) lease time is undefined seconds. (NEW)", + "event": { + "action": "REQUEST DHCP" + }, + "dns": { + "header_flags": [], + "type": "query" + }, + "infoblox": { + "dhcp": { + "circuit_id": "1a:02:30:00:00:00:00:76:00:00:00:00:00:00:2a:f0", + "lease_message": "NEW", + "lease_time": "undefined" + } + }, + "related": { + "ip": [ + "192.168.1.53" + ] + }, + "source": { + "address": "192.168.1.53", + "ip": "192.168.1.53", + "mac": "00:50:56:ae:b3:44" + } + } +} \ No newline at end of file diff --git a/Infoblox/ddi/tests/query_log_dhcp_3.json b/Infoblox/ddi/tests/query_log_dhcp_3.json new file mode 100644 index 000000000..bca901b0b --- /dev/null +++ b/Infoblox/ddi/tests/query_log_dhcp_3.json @@ -0,0 +1,31 @@ +{ + "input": { + "message": "DHCPREQUEST for 192.168.1.107 from e8:c8:29:5c:c8:99 via 192.168.1.107 TransID 80b994d6" + }, + "expected": { + "message": "DHCPREQUEST for 192.168.1.107 from e8:c8:29:5c:c8:99 via 192.168.1.107 TransID 80b994d6", + "event": { + "action": "DHCPREQUEST" + }, + "dns": { + "header_flags": [], + "type": "query" + }, + "infoblox": { + "dhcp": { + "interface_ip": "192.168.1.107", + "trans_id": "80b994d6" + } + }, + "related": { + "ip": [ + "192.168.1.107" + ] + }, + "source": { + "address": "192.168.1.107", + "ip": "192.168.1.107", + "mac": "e8:c8:29:5c:c8:99" + } + } +} \ No newline at end of file diff --git a/Infoblox/ddi/tests/query_log_dhcp_4.json b/Infoblox/ddi/tests/query_log_dhcp_4.json new file mode 100644 index 000000000..417007f30 --- /dev/null +++ b/Infoblox/ddi/tests/query_log_dhcp_4.json @@ -0,0 +1,38 @@ +{ + "input": { + "message": "DHCPREQUEST for 192.168.1.208 from 00:50:56:ae:17:c6 (VDPSCE080019) via eth2 TransID 823c1fa3 uid 01:00:50:56:ae:17:c6 (RENEW)" + }, + "expected": { + "message": "DHCPREQUEST for 192.168.1.208 from 00:50:56:ae:17:c6 (VDPSCE080019) via eth2 TransID 823c1fa3 uid 01:00:50:56:ae:17:c6 (RENEW)", + "event": { + "action": "DHCPREQUEST" + }, + "dns": { + "header_flags": [], + "type": "query" + }, + "infoblox": { + "dhcp": { + "lease_message": "RENEW", + "trans_id": "823c1fa3" + } + }, + "observer": { + "ingress": { + "interface": { + "name": "eth2" + } + } + }, + "related": { + "ip": [ + "192.168.1.208" + ] + }, + "source": { + "address": "192.168.1.208", + "ip": "192.168.1.208", + "mac": "00:50:56:ae:17:c6" + } + } +} \ No newline at end of file diff --git a/Infoblox/ddi/tests/query_log_dhcp_5.json b/Infoblox/ddi/tests/query_log_dhcp_5.json new file mode 100644 index 000000000..14a6991a4 --- /dev/null +++ b/Infoblox/ddi/tests/query_log_dhcp_5.json @@ -0,0 +1,33 @@ +{ + "input": { + "message": "DHCPREQUEST for 192.168.1.95 (192.168.1.95) from d8:94:03:ec:da:d1 via 192.168.1.95 TransID ac1b72c4: lease 192.168.1.95 unavailable." + }, + "expected": { + "message": "DHCPREQUEST for 192.168.1.95 (192.168.1.95) from d8:94:03:ec:da:d1 via 192.168.1.95 TransID ac1b72c4: lease 192.168.1.95 unavailable.", + "event": { + "action": "DHCPREQUEST" + }, + "dns": { + "header_flags": [], + "type": "query" + }, + "infoblox": { + "dhcp": { + "interface_ip": "192.168.1.95", + "request_message": "lease 192.168.1.95 unavailable.", + "router_ip": "192.168.1.95", + "trans_id": "ac1b72c4" + } + }, + "related": { + "ip": [ + "192.168.1.95" + ] + }, + "source": { + "address": "192.168.1.95", + "ip": "192.168.1.95", + "mac": "d8:94:03:ec:da:d1" + } + } +} \ No newline at end of file diff --git a/Infoblox/ddi/tests/query_log_dhcp_6.json b/Infoblox/ddi/tests/query_log_dhcp_6.json new file mode 100644 index 000000000..f0ca93fce --- /dev/null +++ b/Infoblox/ddi/tests/query_log_dhcp_6.json @@ -0,0 +1,32 @@ +{ + "input": { + "message": "DHCPREQUEST for 192.168.1.159 from c8:09:a8:f8:cd:e8 via 192.168.1.159 TransID e711c0c1: ignored (unknown subnet)." + }, + "expected": { + "message": "DHCPREQUEST for 192.168.1.159 from c8:09:a8:f8:cd:e8 via 192.168.1.159 TransID e711c0c1: ignored (unknown subnet).", + "event": { + "action": "DHCPREQUEST" + }, + "dns": { + "header_flags": [], + "type": "query" + }, + "infoblox": { + "dhcp": { + "interface_ip": "192.168.1.159", + "request_message": "ignored (unknown subnet).", + "trans_id": "e711c0c1" + } + }, + "related": { + "ip": [ + "192.168.1.159" + ] + }, + "source": { + "address": "192.168.1.159", + "ip": "192.168.1.159", + "mac": "c8:09:a8:f8:cd:e8" + } + } +} \ No newline at end of file diff --git a/Infoblox/ddi/tests/query_log_dhcp_7.json b/Infoblox/ddi/tests/query_log_dhcp_7.json new file mode 100644 index 000000000..301ee3a35 --- /dev/null +++ b/Infoblox/ddi/tests/query_log_dhcp_7.json @@ -0,0 +1,32 @@ +{ + "input": { + "message": "DHCPACK on 192.168.1.138 to 08:71:90:8d:0b:5d (P70955) via eth2 relay 192.168.1.138 lease-duration 172800" + }, + "expected": { + "message": "DHCPACK on 192.168.1.138 to 08:71:90:8d:0b:5d (P70955) via eth2 relay 192.168.1.138 lease-duration 172800", + "event": { + "action": "DHCPACK" + }, + "dns": { + "header_flags": [], + "type": "query" + }, + "observer": { + "ingress": { + "interface": { + "name": "eth2" + } + } + }, + "related": { + "ip": [ + "192.168.1.138" + ] + }, + "source": { + "address": "192.168.1.138", + "ip": "192.168.1.138", + "mac": "08:71:90:8d:0b:5d" + } + } +} \ No newline at end of file diff --git a/Infoblox/ddi/tests/query_log_dhcp_8.json b/Infoblox/ddi/tests/query_log_dhcp_8.json new file mode 100644 index 000000000..a0415a271 --- /dev/null +++ b/Infoblox/ddi/tests/query_log_dhcp_8.json @@ -0,0 +1,27 @@ +{ + "input": { + "message": "r-l-e:192.168.1.113,Fixed,P76984,c4:d0:e3:b4:08:4d,1732119022,1732291822,,$" + }, + "expected": { + "message": "r-l-e:192.168.1.113,Fixed,P76984,c4:d0:e3:b4:08:4d,1732119022,1732291822,,$", + "dns": { + "header_flags": [], + "type": "query" + }, + "infoblox": { + "ddi": { + "category": "Fixed" + } + }, + "related": { + "ip": [ + "192.168.1.113" + ] + }, + "source": { + "address": "192.168.1.113", + "ip": "192.168.1.113", + "mac": "c4:d0:e3:b4:08:4d" + } + } +} \ No newline at end of file diff --git a/Infoblox/ddi/tests/query_log_dns_1.json b/Infoblox/ddi/tests/query_log_dns_1.json new file mode 100644 index 000000000..2c4c9a783 --- /dev/null +++ b/Infoblox/ddi/tests/query_log_dns_1.json @@ -0,0 +1,36 @@ +{ + "input": { + "message": "FORMERR resolving 'test.testing.io/AAAA/IN': 192.168.1.136#53" + }, + "expected": { + "message": "FORMERR resolving 'test.testing.io/AAAA/IN': 192.168.1.136#53", + "event": { + "action": "FORMERR" + }, + "dns": { + "header_flags": [], + "question": { + "class": "IN", + "name": "test.testing.io", + "registered_domain": "testing.io", + "subdomain": "test", + "top_level_domain": "io", + "type": "AAAA" + }, + "type": "query" + }, + "related": { + "hosts": [ + "test.testing.io" + ], + "ip": [ + "192.168.1.136" + ] + }, + "source": { + "address": "192.168.1.136", + "ip": "192.168.1.136", + "port": 53 + } + } +} \ No newline at end of file diff --git a/Infoblox/ddi/tests/query_log_dns_2.json b/Infoblox/ddi/tests/query_log_dns_2.json new file mode 100644 index 000000000..44cae19f4 --- /dev/null +++ b/Infoblox/ddi/tests/query_log_dns_2.json @@ -0,0 +1,33 @@ +{ + "input": { + "message": "client 192.168.1.1#1130: UDP: query: test.io IN A response: NXDOMAIN +" + }, + "expected": { + "message": "client 192.168.1.1#1130: UDP: query: test.io IN A response: NXDOMAIN +", + "dns": { + "header_flags": [], + "question": { + "class": "IN", + "name": "test.io", + "registered_domain": "test.io", + "top_level_domain": "io", + "type": "A" + }, + "response_code": "NXDOMAIN", + "type": "query" + }, + "related": { + "hosts": [ + "test.io" + ], + "ip": [ + "192.168.1.1" + ] + }, + "source": { + "address": "192.168.1.1", + "ip": "192.168.1.1", + "port": 1130 + } + } +} \ No newline at end of file diff --git a/Infoblox/ddi/tests/query_log_dns_3.json b/Infoblox/ddi/tests/query_log_dns_3.json new file mode 100644 index 000000000..6b742fbb7 --- /dev/null +++ b/Infoblox/ddi/tests/query_log_dns_3.json @@ -0,0 +1,33 @@ +{ + "input": { + "message": "client 192.168.1.1#12337: UDP: query: test.org IN A response: NXDOMAIN +AE" + }, + "expected": { + "message": "client 192.168.1.1#12337: UDP: query: test.org IN A response: NXDOMAIN +AE", + "dns": { + "header_flags": [], + "question": { + "class": "IN", + "name": "test.org", + "registered_domain": "test.org", + "top_level_domain": "org", + "type": "A" + }, + "response_code": "NXDOMAIN", + "type": "query" + }, + "related": { + "hosts": [ + "test.org" + ], + "ip": [ + "192.168.1.1" + ] + }, + "source": { + "address": "192.168.1.1", + "ip": "192.168.1.1", + "port": 12337 + } + } +} \ No newline at end of file diff --git a/Infoblox/ddi/tests/query_log_dns_4.json b/Infoblox/ddi/tests/query_log_dns_4.json new file mode 100644 index 000000000..4806d5bab --- /dev/null +++ b/Infoblox/ddi/tests/query_log_dns_4.json @@ -0,0 +1,34 @@ +{ + "input": { + "message": "client 192.168.1.1#37188: UDP: query: _ldap._tcp.test.test.net IN SRV response: NXDOMAIN +A" + }, + "expected": { + "message": "client 192.168.1.1#37188: UDP: query: _ldap._tcp.test.test.net IN SRV response: NXDOMAIN +A", + "dns": { + "header_flags": [], + "question": { + "class": "IN", + "name": "_ldap._tcp.test.test.net", + "registered_domain": "test.net", + "subdomain": "_ldap._tcp.test", + "top_level_domain": "net", + "type": "SRV" + }, + "response_code": "NXDOMAIN", + "type": "query" + }, + "related": { + "hosts": [ + "_ldap._tcp.test.test.net" + ], + "ip": [ + "192.168.1.1" + ] + }, + "source": { + "address": "192.168.1.1", + "ip": "192.168.1.1", + "port": 37188 + } + } +} \ No newline at end of file diff --git a/Infoblox/ddi/tests/query_log_dns_5.json b/Infoblox/ddi/tests/query_log_dns_5.json new file mode 100644 index 000000000..bd240d9a7 --- /dev/null +++ b/Infoblox/ddi/tests/query_log_dns_5.json @@ -0,0 +1,34 @@ +{ + "input": { + "message": "client 192.168.1.1#37521: UDP: query: test.test.io IN AAAA response: NOERROR +A test.test.io. 86400 IN CNAME test.test.io." + }, + "expected": { + "message": "client 192.168.1.1#37521: UDP: query: test.test.io IN AAAA response: NOERROR +A test.test.io. 86400 IN CNAME test.test.io.", + "dns": { + "header_flags": [], + "question": { + "class": "IN", + "name": "test.test.io", + "registered_domain": "test.io", + "subdomain": "test", + "top_level_domain": "io", + "type": "AAAA" + }, + "response_code": "NOERROR", + "type": "query" + }, + "related": { + "hosts": [ + "test.test.io" + ], + "ip": [ + "192.168.1.1" + ] + }, + "source": { + "address": "192.168.1.1", + "ip": "192.168.1.1", + "port": 37521 + } + } +} \ No newline at end of file diff --git a/Infoblox/ddi/tests/query_log_dns_6.json b/Infoblox/ddi/tests/query_log_dns_6.json new file mode 100644 index 000000000..1129be865 --- /dev/null +++ b/Infoblox/ddi/tests/query_log_dns_6.json @@ -0,0 +1,34 @@ +{ + "input": { + "message": "client 192.168.1.1#40432: UDP: query: test.test.org IN A response: NOERROR + test.test.org. 365 IN A 192.168.1.1; test.test.org. 365 IN A 192.168.1.1; test.test.org. 365 IN A 192.168.1.1; test.test.org. 365 IN A 192.168.1.1; test.test.org. 365 IN A 192.168.1.1; test.test.org. 365 IN A 192.168.1.1" + }, + "expected": { + "message": "client 192.168.1.1#40432: UDP: query: test.test.org IN A response: NOERROR + test.test.org. 365 IN A 192.168.1.1; test.test.org. 365 IN A 192.168.1.1; test.test.org. 365 IN A 192.168.1.1; test.test.org. 365 IN A 192.168.1.1; test.test.org. 365 IN A 192.168.1.1; test.test.org. 365 IN A 192.168.1.1", + "dns": { + "header_flags": [], + "question": { + "class": "IN", + "name": "test.test.org", + "registered_domain": "test.org", + "subdomain": "test", + "top_level_domain": "org", + "type": "A" + }, + "response_code": "NOERROR", + "type": "query" + }, + "related": { + "hosts": [ + "test.test.org" + ], + "ip": [ + "192.168.1.1" + ] + }, + "source": { + "address": "192.168.1.1", + "ip": "192.168.1.1", + "port": 40432 + } + } +} \ No newline at end of file diff --git a/Infoblox/ddi/tests/query_log_dns_7.json b/Infoblox/ddi/tests/query_log_dns_7.json new file mode 100644 index 000000000..b695d7a1a --- /dev/null +++ b/Infoblox/ddi/tests/query_log_dns_7.json @@ -0,0 +1,33 @@ +{ + "input": { + "message": "client 192.168.1.1#49943: UDP: query: test.dev IN A response: NOERROR + test.dev. 11720 IN CNAME test.dev.; thmwh.l46l2i c8.c3r2fb7.81hxxxxxx.dev. 67 IN CNAME test.dev.; test.dev. 52 IN CNAME test.dev.; test.dev. 235 IN A 192.168.1.1; test.dev. 235 IN A 192.168.1.1; test.dev. 235 IN A 192.168.1.1; test.dev. 235 IN A 192.168.1.1; th mwh.xxxxxxxx.c3r2fb7.81hxxxxxx.dev. 235 IN A 192.168.1.1; test.dev. 235 IN A 192.168.1.1; test.dev. 235 IN A 192.168.1.1; thmwh.xxxxxxxx.c3r2fb7.81h xxxxxx.dev. 235 IN A 192.168.1.1;" + }, + "expected": { + "message": "client 192.168.1.1#49943: UDP: query: test.dev IN A response: NOERROR + test.dev. 11720 IN CNAME test.dev.; thmwh.l46l2i c8.c3r2fb7.81hxxxxxx.dev. 67 IN CNAME test.dev.; test.dev. 52 IN CNAME test.dev.; test.dev. 235 IN A 192.168.1.1; test.dev. 235 IN A 192.168.1.1; test.dev. 235 IN A 192.168.1.1; test.dev. 235 IN A 192.168.1.1; th mwh.xxxxxxxx.c3r2fb7.81hxxxxxx.dev. 235 IN A 192.168.1.1; test.dev. 235 IN A 192.168.1.1; test.dev. 235 IN A 192.168.1.1; thmwh.xxxxxxxx.c3r2fb7.81h xxxxxx.dev. 235 IN A 192.168.1.1;", + "dns": { + "header_flags": [], + "question": { + "class": "IN", + "name": "test.dev", + "registered_domain": "test.dev", + "top_level_domain": "dev", + "type": "A" + }, + "response_code": "NOERROR", + "type": "query" + }, + "related": { + "hosts": [ + "test.dev" + ], + "ip": [ + "192.168.1.1" + ] + }, + "source": { + "address": "192.168.1.1", + "ip": "192.168.1.1", + "port": 49943 + } + } +} \ No newline at end of file diff --git a/Infoblox/ddi/tests/query_log_dns_8.json b/Infoblox/ddi/tests/query_log_dns_8.json new file mode 100644 index 000000000..2204069d9 --- /dev/null +++ b/Infoblox/ddi/tests/query_log_dns_8.json @@ -0,0 +1,33 @@ +{ + "input": { + "message": "28-Nov-2024 15:26:27.498 client 1.2.3.4#36615: UDP: query: PD2LORA2.enim.l2 IN A response: NOERROR +A test.dev. 3600 IN A 10.56.12.201;" + }, + "expected": { + "message": "28-Nov-2024 15:26:27.498 client 1.2.3.4#36615: UDP: query: PD2LORA2.enim.l2 IN A response: NOERROR +A test.dev. 3600 IN A 10.56.12.201;", + "@timestamp": "2024-11-28T15:26:27.498000Z", + "dns": { + "header_flags": [], + "question": { + "class": "IN", + "name": "PD2LORA2.enim.l2", + "subdomain": "PD2LORA2.enim", + "type": "A" + }, + "response_code": "NOERROR", + "type": "query" + }, + "related": { + "hosts": [ + "PD2LORA2.enim.l2" + ], + "ip": [ + "1.2.3.4" + ] + }, + "source": { + "address": "1.2.3.4", + "ip": "1.2.3.4", + "port": 36615 + } + } +} \ No newline at end of file diff --git a/Infoblox/ddi/tests/query_log_dns_9.json b/Infoblox/ddi/tests/query_log_dns_9.json new file mode 100644 index 000000000..f4a0e6e07 --- /dev/null +++ b/Infoblox/ddi/tests/query_log_dns_9.json @@ -0,0 +1,35 @@ +{ + "input": { + "message": "28-Nov-2024 15:26:27.359 client 1.2.3.4#63175: UDP: query: www.bing.com IN A response: NOERROR + www.bing.com. 7072 IN CNAME www-www.bing.com.trafficmanager.net.; www-www.bing.com.trafficmanager.net. 56 IN CNAME www.bing.com.edgekey.net.; www.bing.com.edgekey.net. 7154 IN CNAME e86303.test.xxxxx.net.; e86303.test.xxxxx.net. 17 IN A 1.2.3.181; e86303.test.xxxxx.net. 17 IN A 1.2.3.173; e86303.test.xxxxx.net. 17 IN A 1.2.3.184; e86303.test.xxxxx.net. 17 IN A 1.2.3.185; e86303.test.xxxxx.net. 17 IN A 1.2.3.174; e86303.test.xxxxx.net. 17 IN A 1.2.3.183; e86303.test.xxxxx.net. 17 IN A 1.2.3.177; e86303.test.xxxxx.net. 17 IN A 1.2.3.179; e86303.test.xxxxx.net. 17 IN A 1.2.3.175;" + }, + "expected": { + "message": "28-Nov-2024 15:26:27.359 client 1.2.3.4#63175: UDP: query: www.bing.com IN A response: NOERROR + www.bing.com. 7072 IN CNAME www-www.bing.com.trafficmanager.net.; www-www.bing.com.trafficmanager.net. 56 IN CNAME www.bing.com.edgekey.net.; www.bing.com.edgekey.net. 7154 IN CNAME e86303.test.xxxxx.net.; e86303.test.xxxxx.net. 17 IN A 1.2.3.181; e86303.test.xxxxx.net. 17 IN A 1.2.3.173; e86303.test.xxxxx.net. 17 IN A 1.2.3.184; e86303.test.xxxxx.net. 17 IN A 1.2.3.185; e86303.test.xxxxx.net. 17 IN A 1.2.3.174; e86303.test.xxxxx.net. 17 IN A 1.2.3.183; e86303.test.xxxxx.net. 17 IN A 1.2.3.177; e86303.test.xxxxx.net. 17 IN A 1.2.3.179; e86303.test.xxxxx.net. 17 IN A 1.2.3.175;", + "@timestamp": "2024-11-28T15:26:27.359000Z", + "dns": { + "header_flags": [], + "question": { + "class": "IN", + "name": "www.bing.com", + "registered_domain": "bing.com", + "subdomain": "www", + "top_level_domain": "com", + "type": "A" + }, + "response_code": "NOERROR", + "type": "query" + }, + "related": { + "hosts": [ + "www.bing.com" + ], + "ip": [ + "1.2.3.4" + ] + }, + "source": { + "address": "1.2.3.4", + "ip": "1.2.3.4", + "port": 63175 + } + } +} \ No newline at end of file From 122c55a987f7f2ba91d921fd34af5050cf4b613a Mon Sep 17 00:00:00 2001 From: vg-svitla Date: Fri, 29 Nov 2024 12:50:58 +0200 Subject: [PATCH 2/7] Fix smart description --- Infoblox/ddi/_meta/smart-descriptions.json | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Infoblox/ddi/_meta/smart-descriptions.json b/Infoblox/ddi/_meta/smart-descriptions.json index 32aeb23c0..2b9898cb2 100644 --- a/Infoblox/ddi/_meta/smart-descriptions.json +++ b/Infoblox/ddi/_meta/smart-descriptions.json @@ -16,5 +16,24 @@ "type": "request resolution of" } ] + }, + { + "value": "{source.ip} perform {event.action}", + "conditions": [ + { + "field": "source.ip" + }, + { + "field": "event.action" + } + ] + }, + { + "value": "Query on {source.ip}", + "conditions": [ + { + "field": "source.ip" + } + ] } ] From 526f76676800e8db48495abb88e3981595473e3d Mon Sep 17 00:00:00 2001 From: Antoine Ryon Date: Sat, 30 Nov 2024 09:50:19 +0100 Subject: [PATCH 3/7] DNS answers handling and additional improvements --- Infoblox/ddi/ingest/parser.yml | 44 +++++++++--- Infoblox/ddi/tests/query_log_dns_2.json | 3 + Infoblox/ddi/tests/query_log_dns_3.json | 3 + Infoblox/ddi/tests/query_log_dns_4.json | 3 + Infoblox/ddi/tests/query_log_dns_5.json | 12 ++++ Infoblox/ddi/tests/query_log_dns_6.json | 47 +++++++++++++ Infoblox/ddi/tests/query_log_dns_7.json | 3 + Infoblox/ddi/tests/query_log_dns_8.json | 12 ++++ Infoblox/ddi/tests/query_log_dns_9.json | 89 +++++++++++++++++++++++++ 9 files changed, 206 insertions(+), 10 deletions(-) diff --git a/Infoblox/ddi/ingest/parser.yml b/Infoblox/ddi/ingest/parser.yml index 2fb950af6..e54b3ef79 100644 --- a/Infoblox/ddi/ingest/parser.yml +++ b/Infoblox/ddi/ingest/parser.yml @@ -5,7 +5,7 @@ pipeline: name: grok.match properties: output_field: message - pattern: "%{DNS_FORMERR}|%{DNS_OTHER}|%{DNS_0}|%{DNS_1}|%{DNS_2}|%{DNS_3}|%{DNS_4}|%{DNS_5}|%{DNS_6}|%{DNS_7}|%{DNS_8}|%{DNS_9}|%{DNS_10}|%{DNS_11}|%{DNS_12}|%{DNS_13}" + pattern: "%{DNS_FORMERR}|%{DNS_OTHER}|%{DNS_0}|%{DNS_1}|%{DNS_2}|%{DNS_3}|%{DNS_4}|%{DNS_5}|%{DNS_6}|%{DNS_7}|%{DNS_8}|%{DNS_9}|%{DNS_10}|%{DNS_11}|%{DNS_12}|%{DNS_13}|%{DNS_14}" custom_patterns: QUERY_FLAGS: "%{QUERY_FLAGS_RD:flags_rd}%{QUERY_FLAGS_EDNS:flags_edns}?%{QUERY_FLAGS_TCP:flags_tcp}?%{QUERY_FLAGS_DNSSEC:flags_dnssec}?%{QUERY_FLAGS_CD:flags_cd}?%{QUERY_FLAGS_DNS_SERVER_COOKIE}?%{QUERY_FLAGS_DNS_SERVER_COOKIE_WITHOUT_VALID_SERVER}?" QUERY_FLAGS_RD: '[\+\-]' @@ -28,12 +28,13 @@ pipeline: DNS_5: "(%{NOTSPACE:infoblox_nios_log_dns_category}:)?\\s*%{CLIENT} \\(%{DATA}\\): %{VIEW}?query failed %{GREEDYDATA:infoblox_nios_log_dns_message}" DNS_6: "(%{NOTSPACE:infoblox_nios_log_dns_category}:)?\\s*%{CLIENT} \\(%{DATA:infoblox_nios_log_dns_before_query}\\): rewriting query name %{DATA} to '%{DATA:infoblox_nios_log_dns_after_query}', type %{DATA:dns_question_type}" DNS_7: "(%{NOTSPACE:infoblox_nios_log_dns_category}:)?\\s*%{CLIENT} \\(%{DATA}\\): %{VIEW}?query: %{DATA:dns_question_name} %{DATA:dns_question_class} %{WORD:dns_question_type} %{DATA:infoblox_nios_log_dns_header_flags} \\(%{IP:server_ip}\\)" - DNS_8: "(%{NOTSPACE:infoblox_nios_log_dns_category}:)?\\s*%{CLIENT} %{DATA:network_transport}: %{VIEW}?query: %{DATA:dns_question_name} %{DATA:dns_question_class} %{WORD:dns_question_type} response: %{DATA:dns_response_code} %{DATA:infoblox_nios_log_dns_header_flags}" - DNS_9: "(%{NOTSPACE:infoblox_nios_log_dns_category}:)?\\s*%{CLIENT} \\(%{DATA}\\): transfer of '%{DATA:dns_question_name}/%{DATA:dns_question_class}': %{GREEDYDATA:infoblox_nios_log_dns_message}" - DNS_10: "(%{NOTSPACE:infoblox_nios_log_dns_category}:)?\\s*CEF:0\\|Infoblox\\|NIOS\\|%{GREEDYDATA:infoblox_nios_log_dns_version}\\|RPZ-%{DATA:dns_answers_type}\\|%{DATA:infoblox_nios_log_dns_answers_policy}\\|\\d+\\|app=DNS dst=%{IP:server_ip} src=%{IP:client_ip} spt=%{NUMBER:client_port} view=%{DATA:infoblox_nios_log_dns_view_name} qtype=%{WORD:dns_question_type} msg=%{GREEDYDATA:infoblox_nios_log_dns_message}" - DNS_11: "(%{NOTSPACE:infoblox_nios_log_dns_category}:)?\\s*%{GREEDYDATA:_tmp_timestamp} %{CLIENT} %{DATA:network_transport}: %{VIEW}?query: %{DATA:dns_question_name} %{DATA:dns_question_class} %{WORD:dns_question_type} response: %{DATA:dns_response_code} %{DATA:infoblox_nios_log_dns_header_flags} %{GREEDYDATA:repeat_message}" - DNS_12: "(%{NOTSPACE:infoblox_nios_log_dns_category}:)?\\s*%{GREEDYDATA:_tmp_timestamp} %{CLIENT} %{DATA:network_transport}: %{VIEW}?query: %{DATA:dns_question_name} %{DATA:dns_question_class} %{WORD:dns_question_type} response: %{DATA:dns_response_code} %{DATA:infoblox_nios_log_dns_header_flags}" - DNS_13: "(%{NOTSPACE:infoblox_nios_log_dns_category}:)?\\s*%{CLIENT} %{GREEDYDATA:infoblox_nios_log_dns_message}" + DNS_8: "(%{NOTSPACE:infoblox_nios_log_dns_category}:)?\\s*%{CLIENT} %{DATA:network_transport}: %{VIEW}?query: %{DATA:dns_question_name} %{DATA:dns_question_class} %{WORD:dns_question_type} response: %{DATA:dns_response_code} %{DATA:infoblox_nios_log_dns_header_flags} %{GREEDYDATA:dns_records}" + DNS_9: "(%{NOTSPACE:infoblox_nios_log_dns_category}:)?\\s*%{CLIENT} %{DATA:network_transport}: %{VIEW}?query: %{DATA:dns_question_name} %{DATA:dns_question_class} %{WORD:dns_question_type} response: %{DATA:dns_response_code} %{DATA:infoblox_nios_log_dns_header_flags}" + DNS_10: "(%{NOTSPACE:infoblox_nios_log_dns_category}:)?\\s*%{CLIENT} \\(%{DATA}\\): transfer of '%{DATA:dns_question_name}/%{DATA:dns_question_class}': %{GREEDYDATA:infoblox_nios_log_dns_message}" + DNS_11: "(%{NOTSPACE:infoblox_nios_log_dns_category}:)?\\s*CEF:0\\|Infoblox\\|NIOS\\|%{GREEDYDATA:infoblox_nios_log_dns_version}\\|RPZ-%{DATA:dns_answers_type}\\|%{DATA:infoblox_nios_log_dns_answers_policy}\\|\\d+\\|app=DNS dst=%{IP:server_ip} src=%{IP:client_ip} spt=%{NUMBER:client_port} view=%{DATA:infoblox_nios_log_dns_view_name} qtype=%{WORD:dns_question_type} msg=%{GREEDYDATA:infoblox_nios_log_dns_message}" + DNS_12: "(%{NOTSPACE:infoblox_nios_log_dns_category}:)?\\s*%{GREEDYDATA:_tmp_timestamp} %{CLIENT} %{DATA:network_transport}: %{VIEW}?query: %{DATA:dns_question_name} %{DATA:dns_question_class} %{WORD:dns_question_type} response: %{DATA:dns_response_code} %{DATA:infoblox_nios_log_dns_header_flags} %{GREEDYDATA:dns_records}" + DNS_13: "(%{NOTSPACE:infoblox_nios_log_dns_category}:)?\\s*%{GREEDYDATA:_tmp_timestamp} %{CLIENT} %{DATA:network_transport}: %{VIEW}?query: %{DATA:dns_question_name} %{DATA:dns_question_class} %{WORD:dns_question_type} response: %{DATA:dns_response_code} %{DATA:infoblox_nios_log_dns_header_flags}" + DNS_14: "(%{NOTSPACE:infoblox_nios_log_dns_category}:)?\\s*%{CLIENT} %{GREEDYDATA:infoblox_nios_log_dns_message}" # Original pattern DNS_0: '(%{WORD:infoblox_nios_log_dns_category}: )?client ?(%{DATA}) %{IP:client_ip}#%{INT:client_port} (%{DATA}): query: %{DATA:dns_question_name} %{WORD:dns_question_class} %{WORD:dns_question_type} %{QUERY_FLAGS} \(%{IP}\)' @@ -132,7 +133,6 @@ stages: infoblox.dhcp.lease_time: "{{parse_event.message.infoblox_nios_log_dhcp_lease_time}}" infoblox.dhcp.circuit_id: "{{parse_event.message.infoblox_nios_log_dhcp_circuit_id}}" - dns.type: "query" dns.question.class: "{{parse_event.message.dns_question_class}}" dns.question.type: "{{parse_event.message.dns_question_type}}" dns.question.name: "{{parse_event.message.dns_question_name}}" @@ -143,9 +143,33 @@ stages: {% if parse_event.message.flags_rd == "+" %}"RD",{% endif %} {% if parse_event.message.flags_cd == "C" %}"CD",{% endif %} ] + - set: - network.transport: "tcp" - filter: '{{parse_event.message.flags_tcp == "T"}}' + dns.type: query + filter: '{{parse_event.message.get("response_code") == None}}' + - set: + dns.type: answer + dns.response_code: '{{parse_event.message.response_code}}' + filter: '{{parse_event.message.get("response_code") != None}}' + + - set: + dns.answers: | + [ + {%- for record in parse_event.message.dns_records.split() | batch(5) -%} + {"name": "{{record[0]}}", "ttl": {{record[1]}}, "class": "{{record[2]}}", "type": "{{record[3]}}", "data": "{{record[4][:-1]}}"}, + {%- endfor -%} + ] + filter: '{{parse_event.message.get(''dns_records'') != None}}' + + - set: + network.transport: tcp + filter: '{{parse_event.message.get("flags_tcp") != None and parse_event.message.flags_tcp == "T"}}' + - set: + network.transport: udp + filter: '{{parse_event.message.get("flags_tcp") != None and parse_event.message.flags_tcp != "T"}}' + - set: + network.transport: '{{parse_event.message.network_transport | lower }}' + filter: '{{parse_event.message.get("network_transport") != None}}' - set: infoblox.ddi.category: "{{parse_event.message.infoblox_nios_log_dns_category}}" diff --git a/Infoblox/ddi/tests/query_log_dns_2.json b/Infoblox/ddi/tests/query_log_dns_2.json index 44cae19f4..1d4135768 100644 --- a/Infoblox/ddi/tests/query_log_dns_2.json +++ b/Infoblox/ddi/tests/query_log_dns_2.json @@ -16,6 +16,9 @@ "response_code": "NXDOMAIN", "type": "query" }, + "network": { + "transport": "udp" + }, "related": { "hosts": [ "test.io" diff --git a/Infoblox/ddi/tests/query_log_dns_3.json b/Infoblox/ddi/tests/query_log_dns_3.json index 6b742fbb7..959a20d36 100644 --- a/Infoblox/ddi/tests/query_log_dns_3.json +++ b/Infoblox/ddi/tests/query_log_dns_3.json @@ -16,6 +16,9 @@ "response_code": "NXDOMAIN", "type": "query" }, + "network": { + "transport": "udp" + }, "related": { "hosts": [ "test.org" diff --git a/Infoblox/ddi/tests/query_log_dns_4.json b/Infoblox/ddi/tests/query_log_dns_4.json index 4806d5bab..e52e2b96b 100644 --- a/Infoblox/ddi/tests/query_log_dns_4.json +++ b/Infoblox/ddi/tests/query_log_dns_4.json @@ -17,6 +17,9 @@ "response_code": "NXDOMAIN", "type": "query" }, + "network": { + "transport": "udp" + }, "related": { "hosts": [ "_ldap._tcp.test.test.net" diff --git a/Infoblox/ddi/tests/query_log_dns_5.json b/Infoblox/ddi/tests/query_log_dns_5.json index bd240d9a7..126d59135 100644 --- a/Infoblox/ddi/tests/query_log_dns_5.json +++ b/Infoblox/ddi/tests/query_log_dns_5.json @@ -5,6 +5,15 @@ "expected": { "message": "client 192.168.1.1#37521: UDP: query: test.test.io IN AAAA response: NOERROR +A test.test.io. 86400 IN CNAME test.test.io.", "dns": { + "answers": [ + { + "class": "IN", + "data": "test.test.io", + "name": "test.test.io.", + "ttl": 86400, + "type": "CNAME" + } + ], "header_flags": [], "question": { "class": "IN", @@ -17,6 +26,9 @@ "response_code": "NOERROR", "type": "query" }, + "network": { + "transport": "udp" + }, "related": { "hosts": [ "test.test.io" diff --git a/Infoblox/ddi/tests/query_log_dns_6.json b/Infoblox/ddi/tests/query_log_dns_6.json index 1129be865..606b02ef2 100644 --- a/Infoblox/ddi/tests/query_log_dns_6.json +++ b/Infoblox/ddi/tests/query_log_dns_6.json @@ -5,6 +5,50 @@ "expected": { "message": "client 192.168.1.1#40432: UDP: query: test.test.org IN A response: NOERROR + test.test.org. 365 IN A 192.168.1.1; test.test.org. 365 IN A 192.168.1.1; test.test.org. 365 IN A 192.168.1.1; test.test.org. 365 IN A 192.168.1.1; test.test.org. 365 IN A 192.168.1.1; test.test.org. 365 IN A 192.168.1.1", "dns": { + "answers": [ + { + "class": "IN", + "data": "192.168.1.1", + "name": "test.test.org.", + "ttl": 365, + "type": "A" + }, + { + "class": "IN", + "data": "192.168.1.1", + "name": "test.test.org.", + "ttl": 365, + "type": "A" + }, + { + "class": "IN", + "data": "192.168.1.1", + "name": "test.test.org.", + "ttl": 365, + "type": "A" + }, + { + "class": "IN", + "data": "192.168.1.1", + "name": "test.test.org.", + "ttl": 365, + "type": "A" + }, + { + "class": "IN", + "data": "192.168.1.1", + "name": "test.test.org.", + "ttl": 365, + "type": "A" + }, + { + "class": "IN", + "data": "192.168.1.", + "name": "test.test.org.", + "ttl": 365, + "type": "A" + } + ], "header_flags": [], "question": { "class": "IN", @@ -17,6 +61,9 @@ "response_code": "NOERROR", "type": "query" }, + "network": { + "transport": "udp" + }, "related": { "hosts": [ "test.test.org" diff --git a/Infoblox/ddi/tests/query_log_dns_7.json b/Infoblox/ddi/tests/query_log_dns_7.json index b695d7a1a..ca3673036 100644 --- a/Infoblox/ddi/tests/query_log_dns_7.json +++ b/Infoblox/ddi/tests/query_log_dns_7.json @@ -16,6 +16,9 @@ "response_code": "NOERROR", "type": "query" }, + "network": { + "transport": "udp" + }, "related": { "hosts": [ "test.dev" diff --git a/Infoblox/ddi/tests/query_log_dns_8.json b/Infoblox/ddi/tests/query_log_dns_8.json index 2204069d9..72f737796 100644 --- a/Infoblox/ddi/tests/query_log_dns_8.json +++ b/Infoblox/ddi/tests/query_log_dns_8.json @@ -6,6 +6,15 @@ "message": "28-Nov-2024 15:26:27.498 client 1.2.3.4#36615: UDP: query: PD2LORA2.enim.l2 IN A response: NOERROR +A test.dev. 3600 IN A 10.56.12.201;", "@timestamp": "2024-11-28T15:26:27.498000Z", "dns": { + "answers": [ + { + "class": "IN", + "data": "10.56.12.201", + "name": "test.dev.", + "ttl": 3600, + "type": "A" + } + ], "header_flags": [], "question": { "class": "IN", @@ -16,6 +25,9 @@ "response_code": "NOERROR", "type": "query" }, + "network": { + "transport": "udp" + }, "related": { "hosts": [ "PD2LORA2.enim.l2" diff --git a/Infoblox/ddi/tests/query_log_dns_9.json b/Infoblox/ddi/tests/query_log_dns_9.json index f4a0e6e07..ae3c85ca0 100644 --- a/Infoblox/ddi/tests/query_log_dns_9.json +++ b/Infoblox/ddi/tests/query_log_dns_9.json @@ -6,6 +6,92 @@ "message": "28-Nov-2024 15:26:27.359 client 1.2.3.4#63175: UDP: query: www.bing.com IN A response: NOERROR + www.bing.com. 7072 IN CNAME www-www.bing.com.trafficmanager.net.; www-www.bing.com.trafficmanager.net. 56 IN CNAME www.bing.com.edgekey.net.; www.bing.com.edgekey.net. 7154 IN CNAME e86303.test.xxxxx.net.; e86303.test.xxxxx.net. 17 IN A 1.2.3.181; e86303.test.xxxxx.net. 17 IN A 1.2.3.173; e86303.test.xxxxx.net. 17 IN A 1.2.3.184; e86303.test.xxxxx.net. 17 IN A 1.2.3.185; e86303.test.xxxxx.net. 17 IN A 1.2.3.174; e86303.test.xxxxx.net. 17 IN A 1.2.3.183; e86303.test.xxxxx.net. 17 IN A 1.2.3.177; e86303.test.xxxxx.net. 17 IN A 1.2.3.179; e86303.test.xxxxx.net. 17 IN A 1.2.3.175;", "@timestamp": "2024-11-28T15:26:27.359000Z", "dns": { + "answers": [ + { + "class": "IN", + "data": "www-www.bing.com.trafficmanager.net.", + "name": "www.bing.com.", + "ttl": 7072, + "type": "CNAME" + }, + { + "class": "IN", + "data": "www.bing.com.edgekey.net.", + "name": "www-www.bing.com.trafficmanager.net.", + "ttl": 56, + "type": "CNAME" + }, + { + "class": "IN", + "data": "e86303.test.xxxxx.net.", + "name": "www.bing.com.edgekey.net.", + "ttl": 7154, + "type": "CNAME" + }, + { + "class": "IN", + "data": "1.2.3.181", + "name": "e86303.test.xxxxx.net.", + "ttl": 17, + "type": "A" + }, + { + "class": "IN", + "data": "1.2.3.173", + "name": "e86303.test.xxxxx.net.", + "ttl": 17, + "type": "A" + }, + { + "class": "IN", + "data": "1.2.3.184", + "name": "e86303.test.xxxxx.net.", + "ttl": 17, + "type": "A" + }, + { + "class": "IN", + "data": "1.2.3.185", + "name": "e86303.test.xxxxx.net.", + "ttl": 17, + "type": "A" + }, + { + "class": "IN", + "data": "1.2.3.174", + "name": "e86303.test.xxxxx.net.", + "ttl": 17, + "type": "A" + }, + { + "class": "IN", + "data": "1.2.3.183", + "name": "e86303.test.xxxxx.net.", + "ttl": 17, + "type": "A" + }, + { + "class": "IN", + "data": "1.2.3.177", + "name": "e86303.test.xxxxx.net.", + "ttl": 17, + "type": "A" + }, + { + "class": "IN", + "data": "1.2.3.179", + "name": "e86303.test.xxxxx.net.", + "ttl": 17, + "type": "A" + }, + { + "class": "IN", + "data": "1.2.3.175", + "name": "e86303.test.xxxxx.net.", + "ttl": 17, + "type": "A" + } + ], "header_flags": [], "question": { "class": "IN", @@ -18,6 +104,9 @@ "response_code": "NOERROR", "type": "query" }, + "network": { + "transport": "udp" + }, "related": { "hosts": [ "www.bing.com" From 6a8f65023adad6d1ffe2c55447233833f3ff4db5 Mon Sep 17 00:00:00 2001 From: vg-svitla Date: Mon, 2 Dec 2024 18:32:45 +0200 Subject: [PATCH 4/7] Fix comments --- Infoblox/ddi/_meta/fields.yml | 10 ---------- Infoblox/ddi/ingest/parser.yml | 10 ++++++---- Infoblox/ddi/tests/query_log_dhcp_1.json | 7 +++---- Infoblox/ddi/tests/query_log_dhcp_2.json | 7 +++---- Infoblox/ddi/tests/query_log_dhcp_4.json | 4 ++-- Infoblox/ddi/tests/query_log_dhcp_5.json | 4 ++-- Infoblox/ddi/tests/query_log_dhcp_6.json | 4 ++-- Infoblox/ddi/tests/query_log_dns_1.json | 10 +++++----- 8 files changed, 23 insertions(+), 33 deletions(-) diff --git a/Infoblox/ddi/_meta/fields.yml b/Infoblox/ddi/_meta/fields.yml index 69f6818da..947dad6cf 100644 --- a/Infoblox/ddi/_meta/fields.yml +++ b/Infoblox/ddi/_meta/fields.yml @@ -13,21 +13,11 @@ infoblox.dhcp.interface_ip: name: infoblox.dhcp.interface_ip type: ip -infoblox.dhcp.lease_message: - description: The lease message. - name: infoblox.dhcp.lease_message - type: keyword - infoblox.dhcp.lease_time: description: The lease time. name: infoblox.dhcp.lease_time type: keyword -infoblox.dhcp.request_message: - description: The request message. - name: infoblox.dhcp.request_message - type: keyword - infoblox.dhcp.router_ip: description: The IP address of the router. name: infoblox.dhcp.router_ip diff --git a/Infoblox/ddi/ingest/parser.yml b/Infoblox/ddi/ingest/parser.yml index e54b3ef79..2597e3675 100644 --- a/Infoblox/ddi/ingest/parser.yml +++ b/Infoblox/ddi/ingest/parser.yml @@ -43,7 +43,7 @@ pipeline: ## For DNS message like: ## FORMERR resolving 'test.testing.io/AAAA/IN': 192.168.1.136#53 - DNS_FORMERR: "%{WORD:event_action} resolving '%{DATA:dns_question_name}/%{DATA:dns_question_type}/%{DATA:dns_question_class}': %{IP:client_ip}#%{NUMBER:client_port}" + DNS_FORMERR: "%{WORD:event_action} resolving '%{DATA:dns_question_name}/%{DATA:dns_question_type}/%{DATA:dns_question_class}': %{IP:destination_ip}#%{NUMBER:destination_port}" ## For other message like: ## r-l-e:192.168.1.113,Fixed,P76984,c4:d0:e3:b4:08:4d,1732119022,1732291822,,$ @@ -74,7 +74,7 @@ pipeline: ## For DHCP message like: ## Option 82: received a REQUEST DHCP packet from relay-agent eth2 with a circuit-id of "1a:02:30:00:00:00:00:76:00:00:00:00:00:00:2a:f0", a remote-id of "0a:44:70:46" for 192.168.1.222 (00:50:56:ae:b3:44) lease time is undefined seconds. (NEW) - DHCP_OTHER: 'Option %{NUMBER}: received a %{DATA:event_action} packet from %{NOTSPACE} %{DATA:infoblox_nios_log_dhcp_relay_interface_name} with a circuit-id of \"%{DATA:infoblox_nios_log_dhcp_circuit_id}\", a remote-id of \"%{DATA:infoblox_nios_log_dhcp_remote_id}\" for %{IP:client_ip} \(%{MAC:client_mac}\) lease time is %{DATA:infoblox_nios_log_dhcp_lease_time} seconds. \(%{DATA:infoblox_nios_log_dhcp_lease_message}\)' + DHCP_OTHER: 'Option %{NUMBER}: received a %{DATA:event_action} packet from %{NOTSPACE} %{DATA:infoblox_nios_log_dhcp_relay_interface_name} with a circuit-id of \"%{DATA:infoblox_nios_log_dhcp_circuit_id}\", a remote-id of \"%{DATA:infoblox_nios_log_dhcp_remote_id}\" for %{IP:client_ip} \(%{MAC:client_mac}\) %{GREEDYDATA:infoblox_nios_log_dhcp_lease_message}' - name: parse_event filter: "{{'DHCPACK' in original.message}}" @@ -119,17 +119,19 @@ stages: - set: "@timestamp": "{{parse_datetime.result}}" event.action: "{{parse_event.message.event_action}}" + event.reason: "{{parse_event.message.infoblox_nios_log_dhcp_request_message or parse_event.message.infoblox_nios_log_dhcp_lease_message}}" source.port: "{{parse_event.message.client_port}}" source.mac: "{{parse_event.message.client_mac}}" + destination.ip: "{{parse_event.message.destination_ip}}" + destination.port: "{{parse_event.message.destination_port}}" + observer.ingress.interface.name: "{{parse_event.message.observer_ingress_interface_name}}" infoblox.dhcp.interface_ip: "{{parse_event.message.infoblox_nios_log_dhcp_interface_ip}}" infoblox.dhcp.trans_id: "{{parse_event.message.infoblox_nios_log_dhcp_trans_id}}" infoblox.dhcp.router_ip: "{{parse_event.message.infoblox_nios_log_dhcp_router_ip}}" - infoblox.dhcp.request_message: "{{parse_event.message.infoblox_nios_log_dhcp_request_message}}" - infoblox.dhcp.lease_message: "{{parse_event.message.infoblox_nios_log_dhcp_lease_message}}" infoblox.dhcp.lease_time: "{{parse_event.message.infoblox_nios_log_dhcp_lease_time}}" infoblox.dhcp.circuit_id: "{{parse_event.message.infoblox_nios_log_dhcp_circuit_id}}" diff --git a/Infoblox/ddi/tests/query_log_dhcp_1.json b/Infoblox/ddi/tests/query_log_dhcp_1.json index 43383a2e1..0e2ff27e2 100644 --- a/Infoblox/ddi/tests/query_log_dhcp_1.json +++ b/Infoblox/ddi/tests/query_log_dhcp_1.json @@ -5,7 +5,8 @@ "expected": { "message": "Option 82: received a REQUEST DHCP packet from relay-agent eth2 with a circuit-id of \"1a:02:30:00:00:00:00:76:00:00:00:00:00:00:2a:f0\", a remote-id of \"0a:44:70:46\" for 192.168.1.222 (00:50:56:ae:b3:44) lease time is undefined seconds. (NEW)", "event": { - "action": "REQUEST DHCP" + "action": "REQUEST DHCP", + "reason": "lease time is undefined seconds. (NEW)" }, "dns": { "header_flags": [], @@ -13,9 +14,7 @@ }, "infoblox": { "dhcp": { - "circuit_id": "1a:02:30:00:00:00:00:76:00:00:00:00:00:00:2a:f0", - "lease_message": "NEW", - "lease_time": "undefined" + "circuit_id": "1a:02:30:00:00:00:00:76:00:00:00:00:00:00:2a:f0" } }, "related": { diff --git a/Infoblox/ddi/tests/query_log_dhcp_2.json b/Infoblox/ddi/tests/query_log_dhcp_2.json index 9474b00d8..44aebdb62 100644 --- a/Infoblox/ddi/tests/query_log_dhcp_2.json +++ b/Infoblox/ddi/tests/query_log_dhcp_2.json @@ -5,7 +5,8 @@ "expected": { "message": "Option 82: received a REQUEST DHCP packet from relay-agent 192.168.1.53 with a circuit-id of \"1a:02:30:00:00:00:00:76:00:00:00:00:00:00:2a:f0\", a remote-id of \"0a:44:70:46\" for 192.168.1.53 (00:50:56:ae:b3:44) lease time is undefined seconds. (NEW)", "event": { - "action": "REQUEST DHCP" + "action": "REQUEST DHCP", + "reason": "lease time is undefined seconds. (NEW)" }, "dns": { "header_flags": [], @@ -13,9 +14,7 @@ }, "infoblox": { "dhcp": { - "circuit_id": "1a:02:30:00:00:00:00:76:00:00:00:00:00:00:2a:f0", - "lease_message": "NEW", - "lease_time": "undefined" + "circuit_id": "1a:02:30:00:00:00:00:76:00:00:00:00:00:00:2a:f0" } }, "related": { diff --git a/Infoblox/ddi/tests/query_log_dhcp_4.json b/Infoblox/ddi/tests/query_log_dhcp_4.json index 417007f30..826be7f66 100644 --- a/Infoblox/ddi/tests/query_log_dhcp_4.json +++ b/Infoblox/ddi/tests/query_log_dhcp_4.json @@ -5,7 +5,8 @@ "expected": { "message": "DHCPREQUEST for 192.168.1.208 from 00:50:56:ae:17:c6 (VDPSCE080019) via eth2 TransID 823c1fa3 uid 01:00:50:56:ae:17:c6 (RENEW)", "event": { - "action": "DHCPREQUEST" + "action": "DHCPREQUEST", + "reason": "RENEW" }, "dns": { "header_flags": [], @@ -13,7 +14,6 @@ }, "infoblox": { "dhcp": { - "lease_message": "RENEW", "trans_id": "823c1fa3" } }, diff --git a/Infoblox/ddi/tests/query_log_dhcp_5.json b/Infoblox/ddi/tests/query_log_dhcp_5.json index 14a6991a4..681472682 100644 --- a/Infoblox/ddi/tests/query_log_dhcp_5.json +++ b/Infoblox/ddi/tests/query_log_dhcp_5.json @@ -5,7 +5,8 @@ "expected": { "message": "DHCPREQUEST for 192.168.1.95 (192.168.1.95) from d8:94:03:ec:da:d1 via 192.168.1.95 TransID ac1b72c4: lease 192.168.1.95 unavailable.", "event": { - "action": "DHCPREQUEST" + "action": "DHCPREQUEST", + "reason": "lease 192.168.1.95 unavailable." }, "dns": { "header_flags": [], @@ -14,7 +15,6 @@ "infoblox": { "dhcp": { "interface_ip": "192.168.1.95", - "request_message": "lease 192.168.1.95 unavailable.", "router_ip": "192.168.1.95", "trans_id": "ac1b72c4" } diff --git a/Infoblox/ddi/tests/query_log_dhcp_6.json b/Infoblox/ddi/tests/query_log_dhcp_6.json index f0ca93fce..ad6128a51 100644 --- a/Infoblox/ddi/tests/query_log_dhcp_6.json +++ b/Infoblox/ddi/tests/query_log_dhcp_6.json @@ -5,7 +5,8 @@ "expected": { "message": "DHCPREQUEST for 192.168.1.159 from c8:09:a8:f8:cd:e8 via 192.168.1.159 TransID e711c0c1: ignored (unknown subnet).", "event": { - "action": "DHCPREQUEST" + "action": "DHCPREQUEST", + "reason": "ignored (unknown subnet)." }, "dns": { "header_flags": [], @@ -14,7 +15,6 @@ "infoblox": { "dhcp": { "interface_ip": "192.168.1.159", - "request_message": "ignored (unknown subnet).", "trans_id": "e711c0c1" } }, diff --git a/Infoblox/ddi/tests/query_log_dns_1.json b/Infoblox/ddi/tests/query_log_dns_1.json index 2c4c9a783..042f12e70 100644 --- a/Infoblox/ddi/tests/query_log_dns_1.json +++ b/Infoblox/ddi/tests/query_log_dns_1.json @@ -7,6 +7,11 @@ "event": { "action": "FORMERR" }, + "destination": { + "address": "192.168.1.136", + "ip": "192.168.1.136", + "port": 53 + }, "dns": { "header_flags": [], "question": { @@ -26,11 +31,6 @@ "ip": [ "192.168.1.136" ] - }, - "source": { - "address": "192.168.1.136", - "ip": "192.168.1.136", - "port": 53 } } } \ No newline at end of file From 5dd9ece3f663615741b67ba65d8a99d536197e57 Mon Sep 17 00:00:00 2001 From: vg-svitla Date: Mon, 2 Dec 2024 18:35:01 +0200 Subject: [PATCH 5/7] Apply linter --- Infoblox/ddi/ingest/parser.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Infoblox/ddi/ingest/parser.yml b/Infoblox/ddi/ingest/parser.yml index 2597e3675..0fd9855e8 100644 --- a/Infoblox/ddi/ingest/parser.yml +++ b/Infoblox/ddi/ingest/parser.yml @@ -145,13 +145,13 @@ stages: {% if parse_event.message.flags_rd == "+" %}"RD",{% endif %} {% if parse_event.message.flags_cd == "C" %}"CD",{% endif %} ] - + - set: dns.type: query filter: '{{parse_event.message.get("response_code") == None}}' - set: dns.type: answer - dns.response_code: '{{parse_event.message.response_code}}' + dns.response_code: "{{parse_event.message.response_code}}" filter: '{{parse_event.message.get("response_code") != None}}' - set: @@ -161,7 +161,7 @@ stages: {"name": "{{record[0]}}", "ttl": {{record[1]}}, "class": "{{record[2]}}", "type": "{{record[3]}}", "data": "{{record[4][:-1]}}"}, {%- endfor -%} ] - filter: '{{parse_event.message.get(''dns_records'') != None}}' + filter: "{{parse_event.message.get('dns_records') != None}}" - set: network.transport: tcp @@ -170,7 +170,7 @@ stages: network.transport: udp filter: '{{parse_event.message.get("flags_tcp") != None and parse_event.message.flags_tcp != "T"}}' - set: - network.transport: '{{parse_event.message.network_transport | lower }}' + network.transport: "{{parse_event.message.network_transport | lower }}" filter: '{{parse_event.message.get("network_transport") != None}}' - set: From 243d7ed895f12d4dc48ed17c9c1047d8ccc3f6e0 Mon Sep 17 00:00:00 2001 From: vg-svitla Date: Mon, 2 Dec 2024 18:37:18 +0200 Subject: [PATCH 6/7] Fix smart-descriptions.json --- Infoblox/ddi/_meta/smart-descriptions.json | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Infoblox/ddi/_meta/smart-descriptions.json b/Infoblox/ddi/_meta/smart-descriptions.json index 2b9898cb2..86ef44018 100644 --- a/Infoblox/ddi/_meta/smart-descriptions.json +++ b/Infoblox/ddi/_meta/smart-descriptions.json @@ -29,11 +29,19 @@ ] }, { - "value": "Query on {source.ip}", + "value": "Query from {source.ip}", "conditions": [ { "field": "source.ip" } ] + }, + { + "value": "Query to {destination.ip}", + "conditions": [ + { + "field": "destination.ip" + } + ] } ] From a4754df9ef5b6fadafade5360e39be9c88359dab Mon Sep 17 00:00:00 2001 From: vg-svitla Date: Tue, 10 Dec 2024 17:42:47 +0200 Subject: [PATCH 7/7] Fix comments --- Infoblox/ddi/ingest/parser.yml | 7 ++- Infoblox/ddi/tests/query_log_dns_5.json | 2 +- Infoblox/ddi/tests/query_log_dns_6.json | 2 +- Infoblox/ddi/tests/query_log_dns_7.json | 79 +++++++++++++++++++++++++ 4 files changed, 86 insertions(+), 4 deletions(-) diff --git a/Infoblox/ddi/ingest/parser.yml b/Infoblox/ddi/ingest/parser.yml index 0fd9855e8..c84be32fe 100644 --- a/Infoblox/ddi/ingest/parser.yml +++ b/Infoblox/ddi/ingest/parser.yml @@ -157,8 +157,11 @@ stages: - set: dns.answers: | [ - {%- for record in parse_event.message.dns_records.split() | batch(5) -%} - {"name": "{{record[0]}}", "ttl": {{record[1]}}, "class": "{{record[2]}}", "type": "{{record[3]}}", "data": "{{record[4][:-1]}}"}, + {%- for data in parse_event.message.dns_records.split(';') -%} + {%- if data != "" -%} + {%- set record = data.split(' ') -%} + {"name": "{{record[-5]}}", "ttl": {{record[-4]}}, "class": "{{record[-3]}}", "type": "{{record[-2]}}", "data": "{{record[-1]}}"}, + {%- endif -%} {%- endfor -%} ] filter: "{{parse_event.message.get('dns_records') != None}}" diff --git a/Infoblox/ddi/tests/query_log_dns_5.json b/Infoblox/ddi/tests/query_log_dns_5.json index 126d59135..e8b9350f6 100644 --- a/Infoblox/ddi/tests/query_log_dns_5.json +++ b/Infoblox/ddi/tests/query_log_dns_5.json @@ -8,7 +8,7 @@ "answers": [ { "class": "IN", - "data": "test.test.io", + "data": "test.test.io.", "name": "test.test.io.", "ttl": 86400, "type": "CNAME" diff --git a/Infoblox/ddi/tests/query_log_dns_6.json b/Infoblox/ddi/tests/query_log_dns_6.json index 606b02ef2..c3e9d8ddc 100644 --- a/Infoblox/ddi/tests/query_log_dns_6.json +++ b/Infoblox/ddi/tests/query_log_dns_6.json @@ -43,7 +43,7 @@ }, { "class": "IN", - "data": "192.168.1.", + "data": "192.168.1.1", "name": "test.test.org.", "ttl": 365, "type": "A" diff --git a/Infoblox/ddi/tests/query_log_dns_7.json b/Infoblox/ddi/tests/query_log_dns_7.json index ca3673036..7e8b80b01 100644 --- a/Infoblox/ddi/tests/query_log_dns_7.json +++ b/Infoblox/ddi/tests/query_log_dns_7.json @@ -5,6 +5,85 @@ "expected": { "message": "client 192.168.1.1#49943: UDP: query: test.dev IN A response: NOERROR + test.dev. 11720 IN CNAME test.dev.; thmwh.l46l2i c8.c3r2fb7.81hxxxxxx.dev. 67 IN CNAME test.dev.; test.dev. 52 IN CNAME test.dev.; test.dev. 235 IN A 192.168.1.1; test.dev. 235 IN A 192.168.1.1; test.dev. 235 IN A 192.168.1.1; test.dev. 235 IN A 192.168.1.1; th mwh.xxxxxxxx.c3r2fb7.81hxxxxxx.dev. 235 IN A 192.168.1.1; test.dev. 235 IN A 192.168.1.1; test.dev. 235 IN A 192.168.1.1; thmwh.xxxxxxxx.c3r2fb7.81h xxxxxx.dev. 235 IN A 192.168.1.1;", "dns": { + "answers": [ + { + "class": "IN", + "data": "test.dev.", + "name": "test.dev.", + "ttl": 11720, + "type": "CNAME" + }, + { + "class": "IN", + "data": "test.dev.", + "name": "c8.c3r2fb7.81hxxxxxx.dev.", + "ttl": 67, + "type": "CNAME" + }, + { + "class": "IN", + "data": "test.dev.", + "name": "test.dev.", + "ttl": 52, + "type": "CNAME" + }, + { + "class": "IN", + "data": "192.168.1.1", + "name": "test.dev.", + "ttl": 235, + "type": "A" + }, + { + "class": "IN", + "data": "192.168.1.1", + "name": "test.dev.", + "ttl": 235, + "type": "A" + }, + { + "class": "IN", + "data": "192.168.1.1", + "name": "test.dev.", + "ttl": 235, + "type": "A" + }, + { + "class": "IN", + "data": "192.168.1.1", + "name": "test.dev.", + "ttl": 235, + "type": "A" + }, + { + "class": "IN", + "data": "192.168.1.1", + "name": "mwh.xxxxxxxx.c3r2fb7.81hxxxxxx.dev.", + "ttl": 235, + "type": "A" + }, + { + "class": "IN", + "data": "192.168.1.1", + "name": "test.dev.", + "ttl": 235, + "type": "A" + }, + { + "class": "IN", + "data": "192.168.1.1", + "name": "test.dev.", + "ttl": 235, + "type": "A" + }, + { + "class": "IN", + "data": "192.168.1.1", + "name": "xxxxxx.dev.", + "ttl": 235, + "type": "A" + } + ], "header_flags": [], "question": { "class": "IN",