diff --git a/Dockerfile.deploy b/Dockerfile.deploy index 02106fe..008288d 100644 --- a/Dockerfile.deploy +++ b/Dockerfile.deploy @@ -24,6 +24,8 @@ ENV ARANGODB_HOST_URL=http://host.docker.internal:8529 ENV POSTGRES_USER=vulmatch ENV POSTGRES_DB=vulmatch_database ENV POSTGRES_HOST=host.docker.internal +ENV CELERY_BROKER_URL=redis://host.docker.internal:6379/0 +ENV result_backend=redis://host.docker.internal:6379/1 WORKDIR /usr/src/app COPY requirements.txt ./ diff --git a/requirements.txt b/requirements.txt index 91d055d..e085239 100644 --- a/requirements.txt +++ b/requirements.txt @@ -51,4 +51,4 @@ wcwidth==0.2.13 zipp==3.20.2 dogesec_commons @ https://github.com/muchdogesec/dogesec_commons/releases/download/main-2024-12-23-10-31-09/dogesec_commons-0.0.3-py3-none-any.whl arango_cve_processor @ https://github.com/muchdogesec/arango_cve_processor/releases/download/main-2025-01-10-10-35-43/arango_cve_processor-0.0.1-py3-none-any.whl -stix2arango @ https://github.com/muchdogesec/stix2arango/releases/download/main-2024-12-23-10-31-37/stix2arango-0.0.3-py3-none-any.whl \ No newline at end of file +stix2arango @ https://github.com/muchdogesec/stix2arango/releases/download/main-2025-01-17-16-50-57/stix2arango-0.0.3-py3-none-any.whl \ No newline at end of file diff --git a/vulmatch/server/serializers.py b/vulmatch/server/serializers.py index b07147f..0eac866 100644 --- a/vulmatch/server/serializers.py +++ b/vulmatch/server/serializers.py @@ -24,28 +24,17 @@ class NVDTaskSerializer(serializers.Serializer): last_modified_earliest = serializers.DateField(help_text="(`YYYY-MM-DD`): earliest date") last_modified_latest = serializers.DateField(help_text="(`YYYY-MM-DD`): latest date \n* default is `1980-01-01`") ignore_embedded_relationships = serializers.BooleanField(default=False) + always_latest = serializers.BooleanField(default=False, help_text="Skip _is_latest check when uploading, this should only be used when working with large backfill tasks") def validate(self, attrs): if attrs.get('last_modified_earliest') and attrs.get('last_modified_latest') and attrs['last_modified_earliest'] > attrs['last_modified_latest']: raise serializers.ValidationError(f'last_modified_earliest cannot be greater than last_modified_latest') return super().validate(attrs) -class MitreTaskSerializer(serializers.Serializer): - version = serializers.CharField(help_text="mitre version passed to the script") - ignore_embedded_relationships = serializers.BooleanField(default=False) - -class MitreVersionsSerializer(serializers.Serializer): - latest = serializers.CharField(required=False, allow_null=True) - versions = serializers.ListField(child=serializers.CharField()) - class StixVersionsSerializer(serializers.Serializer): latest = serializers.DateTimeField(required=False, allow_null=True) versions = serializers.ListField(child=serializers.DateTimeField()) -class MitreObjectVersions(serializers.Serializer): - modified = serializers.DateTimeField(required=False, allow_null=True) - notes = serializers.ListField(child=serializers.CharField()) - class ACPSerializer(serializers.Serializer): ignore_embedded_relationships = serializers.BooleanField(default=False) diff --git a/vulmatch/server/views.py b/vulmatch/server/views.py index 11a5426..eb1f91e 100644 --- a/vulmatch/server/views.py +++ b/vulmatch/server/views.py @@ -42,8 +42,9 @@ class VulnerabilityStatus(models.models.TextChoices): * `last_modified_earliest` (required - `YYYY-MM-DD`): earliest modified time for vulnerability * `last_modified_latest` (required - `YYYY-MM-DD`): latest modified time for vulnerability * `ignore_embedded_relationships` (optional - default: `false`): Most objects contains embedded relationships inside them (e.g. `created_by_ref`). Setting this to `false` (recommended) will get stix2arango to generate SROs for these embedded relationships so they can be searched. `true` will ignore them. + * `always_latest`: this is a stix2arango setting that defines how updates happen. Default is `true`, setting this to `false` will get stix2arango to bypass the check for updated objects. The only time you should ever set this to `false` is on first backfill where you can be sure no duplicate objects exist. - The data for updates is requested from `https://downloads.ctibutler.com` (managed by the [DOGESEC](https://www.dogesec.com/) team). + The data for updates is requested from `https://cve2stix.vulmatch.com` (managed by the [DOGESEC](https://www.dogesec.com/) team). """ ), ), diff --git a/vulmatch/worker/tasks.py b/vulmatch/worker/tasks.py index 4733911..8ea4235 100644 --- a/vulmatch/worker/tasks.py +++ b/vulmatch/worker/tasks.py @@ -132,6 +132,7 @@ def upload_file(filename, collection_name, stix2arango_note=None, job_id=None, p host_url=settings.ARANGODB_HOST_URL, username=settings.ARANGODB_USERNAME, password=settings.ARANGODB_PASSWORD, + always_latest=params.get('always_latest', False), ) s2a.run()