Merge pull request #13 from CybercentreCanada/submitted_url_only

Analyze Submitted URL (optional)

urldownloader/urldownloader.py

@@ -48,11 +48,11 @@ def execute(self, request: ServiceRequest) -> None:
         if request.get_param('user_agent'):
             headers['User-Agent'] = request.get_param('user_agent')
 
-        # Code to be used when responsibility of fetching submitted_url is moved to service from UI
-        # -----------------------------------------------------------------------------------------
+        urls = []
         submitted_url = request.task.metadata.get('submitted_url')
-        # Make sure this is the first URL fetched
-        urls = [(submitted_url, 10000)] if submitted_url and request.task.depth == 0 else []
+        if request.get_param('analyze_submitted_url') and submitted_url and request.task.depth == 0:
+            # Make sure this is the first URL fetched
+            urls = [(submitted_url, 10000)]
 
         tags = request.task.tags
         # Distinguish between only fetching the submitted_url vs all in the submission
@@ -62,7 +62,7 @@ def execute(self, request: ServiceRequest) -> None:
 
         request.temp_submission_data.setdefault('visited_urls', {})
 
-       # Check if current file is malicious, if so tag URL that downloaded the file
+        # Check if current file is malicious, if so tag URL that downloaded the file
         task_score = 0
         for tags_tuples in tags.values():
             task_score += sum([tag_tuple[1] for tag_tuple in tags_tuples])
@@ -92,8 +92,9 @@ def execute(self, request: ServiceRequest) -> None:
                 fp, sha256 = self.fetch_uri(tag_value, headers=headers)
                 if isinstance(fp, str):
                     self.log.info(f'Success, writing to {fp}...')
-                    request.add_extracted(fp, tag_value, f"Response from {tag_value}",
-                                          safelist_interface=self.api_interface)
+                    if sha256 != request.sha256:
+                        request.add_extracted(fp, tag_value, f"Response from {tag_value}",
+                                              safelist_interface=self.api_interface)
                 else:
                     self.log.debug(f'Server response except occurred: {fp.reason}')
                     exception_table.add_row(TableRow({'URI': tag_value, 'REASON': fp.reason}))