Always latest param (#85)

* expose s2a's ALWAYS_LATEST #80 * add missing env * Update requirements.txt * Update views.py --------- Co-authored-by: David G <[email protected]>
muchdogesec · Jan 21, 2025 · 07a3fbd · 07a3fbd
1 parent 81b7624
commit 07a3fbd
Show file tree

Hide file tree

Showing 5 changed files with 7 additions and 14 deletions.
diff --git a/Dockerfile.deploy b/Dockerfile.deploy
@@ -24,6 +24,8 @@ ENV ARANGODB_HOST_URL=http://host.docker.internal:8529
 ENV POSTGRES_USER=vulmatch
 ENV POSTGRES_DB=vulmatch_database
 ENV POSTGRES_HOST=host.docker.internal
+ENV CELERY_BROKER_URL=redis://host.docker.internal:6379/0
+ENV result_backend=redis://host.docker.internal:6379/1
 
 WORKDIR /usr/src/app
 COPY requirements.txt ./

diff --git a/requirements.txt b/requirements.txt
@@ -51,4 +51,4 @@ wcwidth==0.2.13
 zipp==3.20.2
 dogesec_commons @ https://github.com/muchdogesec/dogesec_commons/releases/download/main-2024-12-23-10-31-09/dogesec_commons-0.0.3-py3-none-any.whl
 arango_cve_processor @ https://github.com/muchdogesec/arango_cve_processor/releases/download/main-2025-01-10-10-35-43/arango_cve_processor-0.0.1-py3-none-any.whl
-stix2arango @ https://github.com/muchdogesec/stix2arango/releases/download/main-2024-12-23-10-31-37/stix2arango-0.0.3-py3-none-any.whl
+stix2arango @ https://github.com/muchdogesec/stix2arango/releases/download/main-2025-01-17-16-50-57/stix2arango-0.0.3-py3-none-any.whl
diff --git a/vulmatch/server/serializers.py b/vulmatch/server/serializers.py
@@ -24,28 +24,17 @@ class NVDTaskSerializer(serializers.Serializer):
     last_modified_earliest = serializers.DateField(help_text="(`YYYY-MM-DD`): earliest date")
     last_modified_latest = serializers.DateField(help_text="(`YYYY-MM-DD`): latest date \n* default is `1980-01-01`")
     ignore_embedded_relationships = serializers.BooleanField(default=False)
+    always_latest = serializers.BooleanField(default=False, help_text="Skip _is_latest check when uploading, this should only be used when working with large backfill tasks")
 
     def validate(self, attrs):
         if attrs.get('last_modified_earliest') and attrs.get('last_modified_latest') and attrs['last_modified_earliest'] > attrs['last_modified_latest']:
             raise serializers.ValidationError(f'last_modified_earliest cannot be greater than last_modified_latest')
         return super().validate(attrs)
 
-class MitreTaskSerializer(serializers.Serializer):
-    version = serializers.CharField(help_text="mitre version passed to the script")
-    ignore_embedded_relationships = serializers.BooleanField(default=False)
-
-class MitreVersionsSerializer(serializers.Serializer):
-    latest = serializers.CharField(required=False, allow_null=True)
-    versions = serializers.ListField(child=serializers.CharField())
-
 class StixVersionsSerializer(serializers.Serializer):
     latest = serializers.DateTimeField(required=False, allow_null=True)
     versions = serializers.ListField(child=serializers.DateTimeField())
 
-class MitreObjectVersions(serializers.Serializer):
-    modified = serializers.DateTimeField(required=False, allow_null=True)
-    notes = serializers.ListField(child=serializers.CharField())
-
 
 class ACPSerializer(serializers.Serializer):
     ignore_embedded_relationships = serializers.BooleanField(default=False)

diff --git a/vulmatch/server/views.py b/vulmatch/server/views.py
@@ -42,8 +42,9 @@ class VulnerabilityStatus(models.models.TextChoices):
             * `last_modified_earliest` (required - `YYYY-MM-DD`): earliest modified time for vulnerability
             * `last_modified_latest` (required - `YYYY-MM-DD`): latest modified time for vulnerability
             * `ignore_embedded_relationships` (optional - default: `false`): Most objects contains embedded relationships inside them (e.g. `created_by_ref`). Setting this to `false` (recommended) will get stix2arango to generate SROs for these embedded relationships so they can be searched. `true` will ignore them.
+            * `always_latest`: this is a stix2arango setting that defines how updates happen. Default is `true`, setting this to `false` will get stix2arango to bypass the check for updated objects. The only time you should ever set this to `false` is on first backfill where you can be sure no duplicate objects exist.
 
-            The data for updates is requested from `https://downloads.ctibutler.com` (managed by the [DOGESEC](https://www.dogesec.com/) team).
+            The data for updates is requested from `https://cve2stix.vulmatch.com` (managed by the [DOGESEC](https://www.dogesec.com/) team).
             """
         ),
     ),

diff --git a/vulmatch/worker/tasks.py b/vulmatch/worker/tasks.py
@@ -132,6 +132,7 @@ def upload_file(filename, collection_name, stix2arango_note=None, job_id=None, p
         host_url=settings.ARANGODB_HOST_URL,
         username=settings.ARANGODB_USERNAME,
         password=settings.ARANGODB_PASSWORD,
+        always_latest=params.get('always_latest', False),
     )
     s2a.run()