Merge pull request #8 from CybercentreCanada/AL-3076-why-is-url-creat…

…or-always-scaled-very-high-on-pb

Al 3076 why is url creator always scaled very high [dev]

tests/results/dadc624d4454e10293dbd1b701b9ee9f99ef83b4cd07b695111d37eb95abcff8_email_found_skip/params.json

@@ -5,7 +5,7 @@
     "tags": [
         {
             "type": "network.email.address",
-            "short_type": "URI",
+            "short_type": "address",
             "value": "[email protected]",
             "score": 0
         },

tests/results/dadc624d4454e10293dbd1b701b9ee9f99ef83b4cd07b695111d37eb95abcff8_no_tags/params.json

@@ -0,0 +1,3 @@
+{
+    "tags": []
+}

tests/results/dadc624d4454e10293dbd1b701b9ee9f99ef83b4cd07b695111d37eb95abcff8_no_tags/result.json

@@ -0,0 +1,16 @@
+{
+  "extra": {
+    "drop_file": false,
+    "score": 0,
+    "sections": []
+  },
+  "files": {
+    "extracted": [],
+    "supplementary": []
+  },
+  "results": {
+    "heuristics": [],
+    "tags": {},
+    "temp_submission_data": {}
+  }
+}

urlcreator/network.py

@@ -10,13 +10,16 @@
 from multidecoder.decoders.network import DOMAIN_TYPE, EMAIL_TYPE, IP_TYPE, URL_TYPE, parse_url
 from multidecoder.multidecoder import Multidecoder
 from multidecoder.node import Node
+from multidecoder.registry import build_registry
 from multidecoder.string_helper import make_bytes, make_str
 
 NETWORK_IOC_TYPES = ["domain", "ip", "uri"]
 
 
 def url_analysis(url: str) -> Tuple[ResultTableSection, Dict[str, List[str]]]:
-    md = Multidecoder()
+    # There is no point in searching for keywords in a URL
+    md_registry = build_registry(exclude=["get_keywords"])
+    md = Multidecoder(decoders=md_registry)
 
     analysis_table = ResultTableSection(url[:128] + "..." if len(url) > 128 else url)
     network_iocs = {ioc_type: [] for ioc_type in NETWORK_IOC_TYPES}

urlcreator/urlcreator.py

@@ -3,6 +3,7 @@
 from collections import Counter, defaultdict
 from urllib.parse import urlparse
 
+import urlcreator.network
 from assemblyline.odm.base import IP_ONLY_REGEX, IPV4_ONLY_REGEX
 from assemblyline_v4_service.common.base import ServiceBase
 from assemblyline_v4_service.common.request import ServiceRequest
@@ -16,8 +17,6 @@
 )
 from assemblyline_v4_service.common.task import MaxExtractedExceeded
 
-import urlcreator.network
-
 # Threshold to trigger heuristic regarding high port usage in URI
 HIGH_PORT_MINIMUM = 1024
 
@@ -51,12 +50,16 @@ def __init__(self, config) -> None:
 
     def execute(self, request: ServiceRequest) -> None:
         request.result = Result()
-
-        minimum_maliciousness = max(int(request.get_param("minimum_maliciousness")), self.minimum_maliciousness_limit)
         tags = request.task.tags
 
         # Only concerned with static/dynamic URIs found by prior services
         urls = tags.get("network.static.uri", []) + tags.get("network.dynamic.uri", [])
+
+        # No tags of interest? Exit fast!
+        if not urls:
+            return
+
+        minimum_maliciousness = max(int(request.get_param("minimum_maliciousness")), self.minimum_maliciousness_limit)
         emails = [x[0].lower() for x in tags.get("network.email.address", [])]
 
         scoring_uri = ResultTableSection(title_text="High scoring URI")