Merge pull request #141 from CybercentreCanada/testing/result_parsing

Add tests for result parsing
CybercentreCanada · Jul 25, 2024 · b8bc4cb · b8bc4cb
2 parents 093be4f + 0d15306
commit b8bc4cb
Show file tree

Hide file tree

Showing 20 changed files with 155 additions and 57 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -29,4 +29,9 @@
         //Added the ignore of E203 for now : https://github.com/PyCQA/pycodestyle/issues/373
         "--ignore=E203,W503"
     ],
+    "python.testing.pytestArgs": [
+        "tests"
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true,
 }
diff --git a/pipelines/azure-tests.yaml b/pipelines/azure-tests.yaml
@@ -0,0 +1,72 @@
+name: tests
+
+variables:
+  - group: unittest-samples
+  - name: self_location
+    value: "self_location"
+  - name: full_self_location
+    value: "$(Agent.BuildDirectory)/$(self_location)"
+  - name: samples_location
+    value: "samples_location"
+  - name: full_samples_location
+    value: "$(Agent.BuildDirectory)/$(samples_location)"
+
+resources:
+  repositories:
+  - repository: unittest-samples
+    type: github
+    name: $(unittest_samples_repository)
+    ref: main
+    endpoint: github-repo-sa
+    trigger: none
+
+trigger: ["*"]
+pr: ["*"]
+
+pool:
+  vmImage: "ubuntu-20.04"
+
+jobs:
+  - job: run_test
+    strategy:
+      matrix:
+        Python3_9:
+          python.version: "3.9"
+        Python3_10:
+         python.version: "3.10"
+        Python3_11:
+         python.version: "3.11"
+
+    timeoutInMinutes: 10
+
+    steps:
+      - task: UsePythonVersion@0
+        displayName: Set python version
+        inputs:
+          versionSpec: "$(python.version)"
+      - checkout: self
+        fetchDepth: 1
+        path: $(self_location)
+      - checkout: unittest-samples
+        fetchDepth: 1
+        path: $(samples_location)
+      - script: |
+          [ ! -d "$(pwd)/tests" ] && echo "No tests found" && exit
+          sudo apt-get update
+          sudo apt-get install -y libfuzzy-dev libfuzzy2
+          if [[ -f "$(pwd)/pkglist.txt" ]]; then
+            grep -vE '^#' "$(pwd)/pkglist.txt" | xargs sudo apt install -y
+          fi
+          sudo rm -rf /var/lib/apt/lists/*
+          sudo env "PATH=$PATH" python -m pip install -U --no-cache-dir assemblyline assemblyline_v4_service
+          [ -f $(pwd)/requirements.txt ] && sudo env "PATH=$PATH" python -m pip install -U --no-cache-dir -r $(pwd)/requirements.txt
+          [ -f $(pwd)/tests/requirements.txt ] && sudo env "PATH=$PATH" python -m pip install -U --no-cache-dir -r $(pwd)/tests/requirements.txt
+          sudo rm -rf /tmp/* /var/lib/apt/lists/* ~/.cache/pip
+        workingDirectory: $(full_self_location)
+        displayName: Setup environment
+      - script: |
+          [ ! -d "$(pwd)/tests" ] && echo "No tests found" && exit
+          export REPO_NAME=${BUILD_REPOSITORY_NAME##*/}
+          python -m pytest -p no:cacheprovider --durations=10 -rsx -vv --disable-warnings
+        workingDirectory: $(full_self_location)
+        displayName: Test
diff --git a/suricata_/helper.py b/suricata_/helper.py
@@ -211,43 +211,37 @@ def attach_network_connection(data: dict):
                     network_part: NetworkConnection = ontology._result_parts.get(source["ontology_id"])
                     if not regex.match(IP_ONLY_REGEX, ext_hostname):
                         attribute["domain"] = ext_hostname
-                    if record.get("http") and record["http"].get("hostname") and network_part.http_details:
+                    if app_proto == "http" and not network_part.http_details:
+                        # Alert pertains to an HTTP event
+                        continue
+                    elif record.get("http") and record["http"].get("hostname") and network_part.http_details:
                         # Only alerts containing HTTP details can provide URI-relevant information
                         http_record = record["http"]
                         network_part_http_details = network_part.http_details
 
-                        if (
-                            "content_range" in http_record
-                            and http_record["content_range"]["raw"]
-                            != network_part_http_details.response_headers.get("content_range")
-                        ):
+                        if "content_range" in http_record and http_record["content_range"][
+                            "raw"
+                        ] != network_part_http_details.response_headers.get("content_range"):
                             # Content range doesn't match
                             continue
-                        elif (
-                            "http_user_agent" in http_record
-                            and http_record["http_user_agent"]
-                            != network_part_http_details.request_headers.get("user_agent")
-                        ):
+                        elif "http_user_agent" in http_record and http_record[
+                            "http_user_agent"
+                        ] != network_part_http_details.request_headers.get("user_agent"):
                             # User agent doesn't match
                             continue
-                        elif (
-                            "http_content_type" in http_record
-                            and http_record["http_content_type"]
-                            != network_part_http_details.response_headers.get("content_type")
-                        ):
+                        elif "http_content_type" in http_record and http_record[
+                            "http_content_type"
+                        ] != network_part_http_details.response_headers.get("content_type"):
                             # Content type doesn't match
                             continue
                         elif (
                             "status" in http_record
-                            and http_record["status"]
-                            != network_part_http_details.response_status_code
+                            and http_record["status"] != network_part_http_details.response_status_code
                         ):
                             # Status code doesn't match
-                            continue                            
+                            continue
 
-                        if not (
-                            http_record["http_method"] == network_part_http_details.request_method
-                        ):
+                        if not (http_record["http_method"] == network_part_http_details.request_method):
                             # Request method or status code doesn't match
                             continue
 
@@ -332,6 +326,6 @@ def attach_network_connection(data: dict):
         "urls": urls,
         "email_addresses": email_addresses,
         "tls": tls_dict,
-        "extracted_files": extracted_files.values(),
+        "extracted_files": list(extracted_files.values()),
         "reverse_lookup": reverse_lookup,
     }
diff --git a/test/create_updater_config.py b/test/create_updater_config.py
diff --git a/test/files/async-oneside-test.rules b/test/files/async-oneside-test.rules
diff --git a/test/files/http_redirects.pcapng b/test/files/http_redirects.pcapng
diff --git a/test/files/rsasnakeoil2.cap b/test/files/rsasnakeoil2.cap
diff --git a/test/files/simple_http_download.onesided.nosyn.noack.pcap b/test/files/simple_http_download.onesided.nosyn.noack.pcap
diff --git a/test/files/simple_http_download.onesided.nosyn.pcap b/test/files/simple_http_download.onesided.nosyn.pcap
diff --git a/test/files/simple_http_download.onesided.pcap b/test/files/simple_http_download.onesided.pcap
diff --git a/test/files/simple_http_download.pcap b/test/files/simple_http_download.pcap
diff --git a/test/files/smb2_putty_xfer.pcap b/test/files/smb2_putty_xfer.pcap
diff --git a/test/files/smb_putty_xfer.pcap b/test/files/smb_putty_xfer.pcap
diff --git a/test/files/smtp.pcap b/test/files/smtp.pcap
diff --git a/tests/files/alert_flow/eve.json b/tests/files/alert_flow/eve.json
@@ -0,0 +1,4 @@
+{"timestamp":"2023-01-26T16:10:11.251909+0000","flow_id":1,"pcap_cnt":180,"event_type":"alert","src_ip":"0.0.0.1","src_port":1,"dest_ip":"0.0.0.2","dest_port":2,"proto":"TCP","pkt_src":"wire/pcap","metadata":{"flowbits":["TEST"]},"community_id":"community_id","alert":{"action":"allowed","gid":0,"signature_id":1,"rev":2,"signature":"TEST","category":"Unknown Classtype","severity":3,"metadata":{"created_at":["2017_03_22"],"updated_at":["2019_07_26"]}},"direction":"to_server","flow":{"pkts_toserver":5,"pkts_toclient":3,"bytes_toserver":635,"bytes_toclient":186,"start":"2023-01-26T16:10:07.939143+0000","src_ip":"0.0.0.1","dest_ip":"0.0.0.2","src_port":1,"dest_port":2}}
+{"timestamp":"2023-01-26T16:03:18.121508+0000","flow_id":1,"event_type":"flow","src_ip":"0.0.0.1","src_port":1,"dest_ip":"0.0.0.2","dest_port":2,"proto":"TCP","app_proto":"failed","flow":{"pkts_toserver":7,"pkts_toclient":5,"bytes_toserver":755,"bytes_toclient":331,"start":"2023-01-26T16:10:07.939143+0000","end":"2023-01-26T16:10:14.938611+0000","age":7,"state":"closed","reason":"shutdown","alerted":true},"metadata":{"flowbits":["TEST"]},"community_id":"community_id","tcp":{"tcp_flags":"1e","tcp_flags_ts":"1e","tcp_flags_tc":"1a","syn":true,"rst":true,"psh":true,"ack":true,"state":"closed","ts_max_regions":1,"tc_max_regions":1}}
+{"timestamp":"2023-01-26T16:03:18.121508+0000","flow_id":1,"event_type":"netflow","src_ip":"0.0.0.1","src_port":1,"dest_ip":"0.0.0.2","dest_port":2,"proto":"TCP","app_proto":"failed","netflow":{"pkts":7,"bytes":755,"start":"2023-01-26T16:10:07.939143+0000","end":"2023-01-26T16:10:14.938611+0000","age":7,"min_ttl":128,"max_ttl":128},"tcp":{"tcp_flags":"1e","syn":true,"rst":true,"psh":true,"ack":true},"metadata":{"flowbits":["TEST"]},"community_id":"community_id"}
+{"timestamp":"2023-01-26T16:03:18.121508+0000","flow_id":1,"event_type":"netflow","src_ip":"0.0.0.2","src_port":2,"dest_ip":"0.0.0.1","dest_port":1,"proto":"TCP","app_proto":"failed","netflow":{"pkts":5,"bytes":331,"start":"2023-01-26T16:10:07.939143+0000","end":"2023-01-26T16:10:14.938611+0000","age":7,"min_ttl":64,"max_ttl":64},"tcp":{"tcp_flags":"1a","syn":true,"psh":true,"ack":true},"metadata":{"flowbits":["TEST"]},"community_id":"community_id"}
diff --git a/tests/files/alert_flow/result.json b/tests/files/alert_flow/result.json
@@ -0,0 +1,18 @@
+{
+  "signatures": {
+      "0:2825564": {
+          "attributes": [
+              {
+                  "source": {
+                      "ontology_id": "network_5vAUsPCHBRjuVw2VD9msTF",
+                      "service_name": null,
+                      "tag": "0.0.0.1:2",
+                      "time_observed": "2023-01-26 16:03:18.121508+00:00"
+                  }
+              }
+          ],
+          "malware_family": [],
+          "signature": "TEST"
+      }
+  }
+}
diff --git a/tests/files/alert_http/eve.json b/tests/files/alert_http/eve.json
@@ -0,0 +1,5 @@
+{"timestamp":"2023-09-28T13:37:57.854302+0000","flow_id":1,"pcap_cnt":30,"event_type":"alert","src_ip":"0.0.0.1","src_port":1,"dest_ip":"0.0.0.2","dest_port":2,"proto":"TCP","pkt_src":"wire/pcap","community_id":"community_id","tx_id":0,"alert":{"action":"allowed","gid":0,"signature_id":1,"rev":4,"signature":"TEST","category":"Unknown Classtype","severity":3,"metadata":{"created_at":["2015_05_30"],"updated_at":["2020_08_17"]}},"http":{"hostname":"bad.com","url":"/bad_path","http_user_agent":"test_agent","http_content_type":"text/html","http_method":"POST","protocol":"HTTP/1.1","status":200,"length":86},"files":[{"filename":"/bad_path","gaps":false,"state":"CLOSED","md5":"md5","sha256":"sha256","stored":false,"storing":true,"size":65,"tx_id":0}],"app_proto":"http","direction":"to_server","flow":{"pkts_toserver":4,"pkts_toclient":4,"bytes_toserver":540,"bytes_toclient":465,"start":"2023-09-28T13:37:57.825895+0000","src_ip":"0.0.0.1","dest_ip":"0.0.0.2","src_port":1,"dest_port":2}}
+{"timestamp":"2023-09-28T13:37:57.854302+0000","flow_id":1,"pcap_cnt":30,"event_type":"fileinfo","src_ip":"0.0.0.1","src_port":1,"dest_ip":"0.0.0.2","dest_port":2,"proto":"TCP","pkt_src":"wire/pcap","community_id":"community_id","http":{"hostname":"bad.com","url":"/bad_path","http_user_agent":"test_agent","http_content_type":"text/html","http_method":"POST","protocol":"HTTP/1.1","status":200,"length":86},"app_proto":"http","fileinfo":{"filename":"/bad_path","gaps":false,"state":"CLOSED","md5":"md5","sha256":"sha256","stored":true,"file_id":1,"size":65,"tx_id":0}}
+{"timestamp":"2023-09-28T13:37:57.854302+0000","flow_id":1,"pcap_cnt":30,"event_type":"http","src_ip":"0.0.0.1","src_port":1,"dest_ip":"0.0.0.2","dest_port":2,"proto":"TCP","pkt_src":"wire/pcap","community_id":"community_id","tx_id":0,"http":{"hostname":"bad.com","url":"/bad_path","http_user_agent":"test_agent","http_content_type":"text/html","http_method":"POST","protocol":"HTTP/1.1","status":200,"length":86,"request_headers":[{"name":"User-Agent","value":"test_agent"},{"name":"Host","value":"bad.com"},{"name":"Pragma","value":"no-cache"},{"name":"Content-type","value":"application/x-www-form-urlencoded"},{"name":"Content-length","value":"65"}],"response_headers":[{"name":"Server","value":"INetSim HTTP Server"},{"name":"Connection","value":"Close"},{"name":"Content-Length","value":"86"},{"name":"Content-Type","value":"text/html"},{"name":"Date","value":"Thu, 28 Sep 2023 13:37:57 GMT"}]}}
+{"timestamp":"2023-09-28T13:31:06.236569+0000","flow_id":1,"event_type":"flow","src_ip":"0.0.0.1","src_port":1,"dest_ip":"0.0.0.2","dest_port":2,"proto":"TCP","app_proto":"http","flow":{"pkts_toserver":5,"pkts_toclient":5,"bytes_toserver":600,"bytes_toclient":525,"start":"2023-09-28T13:37:57.825895+0000","end":"2023-09-28T13:37:57.854373+0000","age":0,"state":"closed","reason":"shutdown","alerted":true},"community_id":"community_id","tcp":{"tcp_flags":"1b","tcp_flags_ts":"1b","tcp_flags_tc":"1b","syn":true,"fin":true,"psh":true,"ack":true,"state":"closed","ts_max_regions":1,"tc_max_regions":1}}
+{"timestamp":"2023-09-28T13:31:06.236569+0000","flow_id":1,"event_type":"netflow","src_ip":"0.0.0.1","src_port":1,"dest_ip":"0.0.0.2","dest_port":2,"proto":"TCP","app_proto":"http","netflow":{"pkts":5,"bytes":600,"start":"2023-09-28T13:37:57.825895+0000","end":"2023-09-28T13:37:57.854373+0000","age":0,"min_ttl":128,"max_ttl":128},"tcp":{"tcp_flags":"1b","syn":true,"fin":true,"psh":true,"ack":true},"community_id":"community_id"}
diff --git a/tests/files/alert_http/result.json b/tests/files/alert_http/result.json
@@ -0,0 +1,19 @@
+{
+  "signatures": {
+    "0:2811170": {
+      "attributes": [
+        {
+          "source": {
+            "ontology_id": "network_http_fkW8ZA1YJOfyF8ULRt4sI",
+            "service_name": null,
+            "tag": "0.0.0.1:2",
+            "time_observed": "2023-09-28 13:37:57.854302+00:00"
+          },
+          "uri": "http://bad.com/bad_path"
+        }
+      ],
+      "malware_family": [],
+      "signature": "TEST"
+    }
+  }
+}
diff --git a/tests/requirements.txt b/tests/requirements.txt
@@ -0,0 +1,2 @@
+assemblyline-service-utilities
+pytest
diff --git a/tests/test_helper.py b/tests/test_helper.py
@@ -0,0 +1,13 @@
+import os
+import pytest
+
+from suricata_.helper import parse_suricata_output
+
+
+@pytest.mark.parametrize("sample_dir", ["files/alert_http", "files/alert_flow"])
+def test_alert_signature_correlation(sample_dir):
+    sample_dir = os.path.join(os.path.dirname(__file__), sample_dir)
+    result = parse_suricata_output(sample_dir)
+    for s in result["signatures"].values():
+        # In most cases, an alert event should correspond to a other single event (ie. http, dns, netflow)
+        assert len(s["attributes"]) == 1