add support for new flags (#113)

* add support for new flags #111 #112 * Update requirements.txt * updating views --------- Co-authored-by: David G <[email protected]>
muchdogesec · Feb 7, 2025 · f93faca · f93faca
1 parent 302da67
commit f93faca
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 10 deletions.
diff --git a/requirements.txt b/requirements.txt
@@ -49,6 +49,6 @@ urllib3==2.2.3
 vine==5.1.0
 wcwidth==0.2.13
 zipp==3.20.2
-dogesec_commons @ https://github.com/muchdogesec/dogesec_commons/releases/download/main-2025-01-31-07-34-17/dogesec_commons-0.0.4rc0-py3-none-any.whl
-arango_cve_processor @ https://github.com/muchdogesec/arango_cve_processor/releases/download/main-2025-01-27-11-39-19/arango_cve_processor-0.0.1-py3-none-any.whl
-stix2arango @ https://github.com/muchdogesec/stix2arango/releases/download/main-2025-02-01-18-38-26/stix2arango-0.0.3-py3-none-any.whl
+dogesec_commons @ https://github.com/muchdogesec/dogesec_commons/releases/download/main-2025-02-06-06-30-23/dogesec_commons-0.0.5-py3-none-any.whl
+arango_cve_processor @ https://github.com/muchdogesec/arango_cve_processor/releases/download/main-2025-02-06-16-08-34/arango_cve_processor-0.0.1-py3-none-any.whl
+stix2arango @ https://github.com/muchdogesec/stix2arango/releases/download/main-2025-02-04-14-14-39/stix2arango-0.0.4rc0-py3-none-any.whl
diff --git a/vulmatch/server/serializers.py b/vulmatch/server/serializers.py
@@ -23,8 +23,10 @@ class Meta:
 class NVDTaskSerializer(serializers.Serializer):
     last_modified_earliest = serializers.DateField(help_text="(`YYYY-MM-DD`): earliest date")
     last_modified_latest = serializers.DateField(help_text="(`YYYY-MM-DD`): latest date \n* default is `1980-01-01`")
-    ignore_embedded_relationships = serializers.BooleanField(default=False)
-    always_latest = serializers.BooleanField(default=False, help_text="Skip _is_latest check when uploading, this should only be used when working with large backfill tasks")
+    ignore_embedded_relationships = serializers.BooleanField(default=True)
+    ignore_embedded_relationships_sro = serializers.BooleanField(default=True)
+    ignore_embedded_relationships_smo = serializers.BooleanField(default=True)
+    always_latest = serializers.BooleanField(default=True, help_text="Skip _is_latest check when uploading, this should only be used when working with large backfill tasks")
 
     def validate(self, attrs):
         if attrs.get('last_modified_earliest') and attrs.get('last_modified_latest') and attrs['last_modified_earliest'] > attrs['last_modified_latest']:
@@ -38,6 +40,8 @@ class StixVersionsSerializer(serializers.Serializer):
 
 class ACPSerializer(serializers.Serializer):
     ignore_embedded_relationships = serializers.BooleanField(default=False)
+    ignore_embedded_relationships_sro = serializers.BooleanField(default=True)
+    ignore_embedded_relationships_smo = serializers.BooleanField(default=True)
     modified_min = serializers.DateTimeField(required=False)
     created_min = serializers.DateTimeField(required=False)
 

diff --git a/vulmatch/server/views.py b/vulmatch/server/views.py
@@ -41,8 +41,10 @@ class VulnerabilityStatus(models.models.TextChoices):
 
             * `last_modified_earliest` (required - `YYYY-MM-DD`): earliest modified time for vulnerability
             * `last_modified_latest` (required - `YYYY-MM-DD`): latest modified time for vulnerability
-            * `ignore_embedded_relationships` (optional - default: `false`): Most objects contains embedded relationships inside them (e.g. `created_by_ref`). Setting this to `false` (recommended) will get stix2arango to generate SROs for these embedded relationships so they can be searched. `true` will ignore them.
-            * `always_latest`: this is a stix2arango setting that defines how updates happen. Default is `true`, setting this to `false` will get stix2arango to bypass the check for updated objects. The only time you should ever set this to `false` is on first backfill where you can be sure no duplicate objects exist.
+            * `ignore_embedded_relationships` (optional - default: `true`): Most objects contains embedded relationships inside them (e.g. `created_by_ref`). Setting this to `false` is not recommended as it will get stix2arango to generate SROs for these embedded relationships so they can be searched (this will create millions of additional relationships). `true` will ignore them. This is a stix2arango setting.
+            * `ignore_embedded_relationships_sro` (optional): boolean, if `true` passed (recommended), will stop any embedded relationships from being generated from SRO objects (`type` = `relationship`). Default is `true`. This is a stix2arango setting.
+            * `ignore_embedded_relationships_smo` (optional): boolean, if `true` passed (recommended), will stop any embedded relationships from being generated from SMO objects (`type` = `marking-definition`, `extension-definition`, `language-content`). Default is `true`. This is a stix2arango setting.
+            * `always_latest`: this is a stix2arango setting that defines how updates happen. Default is `true`, setting this to `false` will get stix2arango to bypass the check for updated objects. The only time you should ever set this to `false` is on first run AND when dates are between `1988-01-01` and `2024-12-31` (because of the way we generated cve2stix data).
 
             The data for updates is requested from `https://cve2stix.vulmatch.com` (managed by the [DOGESEC](https://www.dogesec.com/) team).
             """
@@ -517,7 +519,9 @@ def retrieve_object_relationships(self, request, *args, cpe_name=None, **kwargs)
 
             The following key/values are accepted in the body of the request:
 
-            * `ignore_embedded_relationships` (optional - default: `true`): arango_cve_processor generates SROs to link knowledge-bases. These SROs have embedded relationships inside them. Setting this to `true` (recommended) will generate SROs for these embedded relationships so they can be searched. `false` will ignore them
+            * `ignore_embedded_relationships` (optional - default: `false`): arango_cve_processor generates SROs to link knowledge-bases. These SROs have embedded relationships inside them. Setting this to `false` is generally recommended, but ALWAYS when running `cve-epss` and `cve-kev` to ensure the Report objects created are correctly joined to the CVE.
+            * `ignore_embedded_relationships_sro` (optional): boolean, if `true` passed (recommended), will stop any embedded relationships from being generated from SRO objects (`type` = `relationship`). Default is `true`. This is a stix2arango setting.
+            * `ignore_embedded_relationships_smo` (optional): boolean, if `true` passed (recommended), will stop any embedded relationships from being generated from SMO objects (`type` = `marking-definition`, `extension-definition`, `language-content`). Default is `true`. This is a stix2arango setting.
             * `modified_min` (optional - default: all time - format: `YYYY-MM-DDTHH:MM:SS.sssZ`): by default arango_cve_processor will run over all objects in the latest version of a framework (e.g. ATT&CK). This is not always efficient, especially when updating CVE records. As such, you can ask the script to only consider objects with a `modified` time greater than that specified for this field.
             * `created_min` (optional - default: all time- format: `YYYY-MM-DDTHH:MM:SS.sssZ`): same as `modified_min`, but this time considers `created` time of the object (not `modified` time).
             """

diff --git a/vulmatch/worker/tasks.py b/vulmatch/worker/tasks.py
@@ -128,11 +128,12 @@ def upload_file(filename, collection_name, stix2arango_note=None, job_id=None, p
         database=settings.ARANGODB_DATABASE,
         collection=collection_name,
         stix2arango_note=stix2arango_note,
-        ignore_embedded_relationships=params.get('ignore_embedded_relationships', False),
         host_url=settings.ARANGODB_HOST_URL,
         username=settings.ARANGODB_USERNAME,
         password=settings.ARANGODB_PASSWORD,
-        always_latest=params.get('always_latest', False),
+        # always_latest=params.get('always_latest', False),
+        # ignore_embedded_relationships=params.get('ignore_embedded_relationships', False),
+        **params
     )
     s2a.run()