move acvep-related index creation into acvep, remove sort (#103)

* move acvep-related index creation into acvep, remove sort

* add space

* fix typo

requirements.txt

@@ -50,5 +50,5 @@ vine==5.1.0
 wcwidth==0.2.13
 zipp==3.20.2
 dogesec_commons @ https://github.com/muchdogesec/dogesec_commons/releases/download/main-2024-12-23-10-31-09/dogesec_commons-0.0.3-py3-none-any.whl
-arango_cve_processor @ https://github.com/muchdogesec/arango_cve_processor/releases/download/main-2025-01-10-10-35-43/arango_cve_processor-0.0.1-py3-none-any.whl
+arango_cve_processor @ https://github.com/muchdogesec/arango_cve_processor/archive/refs/heads/chunk-object-retrieval.zip
 stix2arango @ https://github.com/muchdogesec/stix2arango/releases/download/main-2025-01-17-16-50-57/stix2arango-0.0.3-py3-none-any.whl

vulmatch/server/arango_helpers.py

@@ -1,5 +1,6 @@
 import contextlib
 import json
+import logging
 from pathlib import Path
 import re
 import typing
@@ -292,12 +293,14 @@ def __init__(self, collection, request, container='objects') -> None:
         self.page, self.count = self.get_page_params(request)
         self.request = request
         self.query = request.query_params.dict()
+
     def execute_query(self, query, bind_vars={}, paginate=True, relationship_mode=False, container=None):
         if relationship_mode:
             return self.get_relationships(query, bind_vars)
         if paginate:
             bind_vars['offset'], bind_vars['count'] = self.get_offset_and_count(self.count, self.page)
         cursor = self.db.aql.execute(query, bind_vars=bind_vars, count=True, full_count=True)
+        logging.info("AQL stat: %s", cursor.statistics())
         if paginate:
             return self.get_paginated_response(container or self.container, cursor, self.page, self.page_size, cursor.statistics()["fullCount"])
         return list(cursor)
@@ -420,7 +423,7 @@ def get_vulnerabilities(self):
             filters.append('''
                 FILTER LENGTH(
                     FOR d IN nvd_cve_edge_collection
-                        FILTER doc._id == d._from AND d.relationship_type == 'exploited-using' AND d._arango_cve_processor_note == "cve-attack" AND NOT doc._is_ref AND d.external_references
+                        FILTER doc._id == d._from AND d.relationship_type == 'exploited-using' AND d._arango_cve_processor_note == "cve-attack" AND doc._is_ref != TRUE AND d.external_references != NULL
                         FILTER FIRST(FOR c IN d.external_references FILTER c.source_name == 'mitre-attack' RETURN c.external_id) IN @attack_ids
                         LIMIT 1
                         RETURN TRUE
@@ -432,7 +435,7 @@ def get_vulnerabilities(self):
             filters.append('''
                 FILTER LENGTH(
                     FOR d IN nvd_cve_edge_collection
-                        FILTER doc._id == d._from AND d.relationship_type == 'exploited-using' AND d._arango_cve_processor_note == "cve-capec" AND NOT doc._is_ref AND d.external_references
+                        FILTER doc._id == d._from AND d.relationship_type == 'exploited-using' AND d._arango_cve_processor_note == "cve-capec" AND doc._is_ref != TRUE AND d.external_references != NULL
                         FILTER FIRST(FOR c IN d.external_references FILTER c.source_name == 'capec' RETURN c.external_id) IN @capec_ids
                         LIMIT 1
                         RETURN TRUE
@@ -519,13 +522,13 @@ def get_cve_bundle(self, cve_id: str):
         query = '''
 LET cve_data_ids = (
   FOR doc IN nvd_cve_vertex_collection
-  FILTER doc._is_latest AND doc.external_references[0].external_id == @cve_id AND ( @@@vertex_filters )
+  FILTER doc._is_latest == TRUE AND doc.external_references[0].external_id == @cve_id AND ( @@@vertex_filters )
   RETURN doc._id
 )
 
 LET default_object_ids = (
   FOR doc IN nvd_cve_vertex_collection
-  FILTER doc.id IN @default_imports AND doc._is_latest
+  FILTER doc.id IN @default_imports AND doc._is_latest == TRUE
   RETURN doc._id
 )
 
@@ -551,7 +554,7 @@ def get_cve_bundle(self, cve_id: str):
     def get_cxe_object(self, cve_id, type="vulnerability", var='name', version_param='cve_version', relationship_mode=False):
         bind_vars={'@collection': self.collection, 'obj_name': cve_id, "type":type, 'var':var}
         #return Response(bind_vars)
-        filters = ['FILTER doc._is_latest']
+        filters = ['FILTER doc._is_latest == TRUE']
         if q := self.query.get(version_param):
             bind_vars['stix_modified'] = q
             filters[0] = 'FILTER doc.modified == @stix_modified'
@@ -627,21 +630,20 @@ def get_softwares(self):
 
 
         if q := self.query.get('name'):
-            bind_vars['name'] = q
-            filters.append('FILTER CONTAINS(doc.name, @name)')
+            bind_vars['name'] = self.like_string(q).lower()
+            filters.append('FILTER doc.name LIKE @name')
 
         query = """
             FOR doc in @@collection OPTIONS {indexHint: "cpe_search_inv", forceIndexHint: true}
-            FILTER doc.type == 'software' AND doc._is_latest == TRUE
+            FILTER doc.type == 'software' 
             LET cve_matches = (FOR d in nvd_cve_edge_collection FILTER d._to == doc._id AND d.relationship_type IN ['exploits', 'relies-on'] RETURN [d.relationship_type, d.external_references[0].external_id])
 
             @filters
-            @sort_stmt
+            FILTER doc._is_latest == TRUE //*/
             LIMIT @offset, @count
             RETURN KEEP(doc, KEYS(doc, true))
-        """.replace('@filters', '\n'.join(filters))\
-            .replace('@sort_stmt', self.get_sort_stmt(CPE_SORT_FIELDS, doc_name='doc.x_cpe_struct'))
-        # return HttpResponse(f"""{query}\n// {json.dumps(bind_vars)}""")
+        """.replace('@filters', '\n'.join(filters))
+
         return self.execute_query(query, bind_vars=bind_vars)
 
     def get_relationships(self, docs_query, binds):

vulmatch/server/ctibutler_views/arango_helpers.py

@@ -401,7 +401,7 @@ def get_attack_objects(self, matrix):
 
         query = """
             FOR doc in @@collection
-            FILTER doc.type IN @types AND doc._arango_cve_processor_note == 'cve-attack' AND (@include_revoked OR NOT doc.revoked) AND (@include_deprecated OR NOT doc.x_mitre_deprecated)
+            FILTER doc.type IN @types AND doc._arango_cve_processor_note == 'cve-attack' AND (@include_revoked OR doc.revoked != TRUE) AND (@include_deprecated OR doc.x_mitre_deprecated != TRUE)
             @filters
             LIMIT @offset, @count
             RETURN KEEP(doc, KEYS(doc, true))

vulmatch/server/views.py

@@ -487,12 +487,6 @@ class filterset_class(FilterSet):
         target_hw = CharFilter(help_text='Characterises the instruction set architecture (e.g., x86) on which the product being described or identified operates (this is the 11th value in the CPE URI).')
         other = CharFilter(help_text='Capture any other general descriptive or identifying information which is vendor- or product-specific and which does not logically fit in any other attribute value (this is the 12th value in the CPE URI).')
 
-
-    @extend_schema(
-            parameters=[
-                OpenApiParameter('sort', enum=CPE_SORT_FIELDS, description="Sort results by"),
-            ]
-    )
     @decorators.action(methods=['GET'], url_path="objects", detail=False)
     def list_objects(self, request, *args, **kwargs):
         return ArangoDBHelper('', request).get_softwares()

vulmatch/urls.py

@@ -38,19 +38,20 @@
 router.register("kev", views.KevView, "kev-view")
 router.register("epss", views.EPSSView, "epss-view")
 ## objects
-router.register('objects/smos', arango_views.SMOView, "object-view-smo")
-router.register('objects/scos', arango_views.SCOView, "object-view-sco")
-router.register('objects/sros', arango_views.SROView, "object-view-sro")
-router.register('objects/sdos', arango_views.SDOView, "object-view-sdo")
-router.register("object", arango_views.SingleObjectView, "object-view-orig")
-
+obj_router = routers.SimpleRouter(use_regex_path=True)
+obj_router.register("", arango_views.SingleObjectView, "object-view-orig")
+obj_router.register('smos', arango_views.SMOView, "object-view-smo")
+obj_router.register('scos', arango_views.SCOView, "object-view-sco")
+obj_router.register('sros', arango_views.SROView, "object-view-sro")
+obj_router.register('sdos', arango_views.SDOView, "object-view-sdo")
 ### more views
 router.register("cwe", ctibutler_views.CweView, "cwe-view")
 router.register("capec", ctibutler_views.CapecView, "capec-view")
 router.register("attack", ctibutler_views.AttackView, "attack-view")
 
 urlpatterns = [
     path(f'api/{API_VERSION}/', include(router.urls)),
+    path(f'api/{API_VERSION}/objects/', include(obj_router.urls)),
     path('admin/', admin.site.urls),
     # YOUR PATTERNS
     path('api/schema/', SpectacularAPIView.as_view(), name='schema'),

vulmatch/worker/populate_dbs.py

@@ -8,6 +8,7 @@
 
 
 from stix2arango.stix2arango import Stix2Arango
+from arango_cve_processor.tools.utils import create_indexes as create_acvep_indexes
 
 
 collections_to_create = ['nvd_cve']
@@ -43,9 +44,10 @@ def create_indexes(db: StandardDatabase):
         "id",
         "type",
         *[dict(name=f'x_cpe_struct.{name}', analyzer='norm_en') for name in ['product', 'vendor', 'version', 'update', 'edition', 'language', 'sw_edition', 'target_sw', 'target_hw', 'other']],
-        "x_cpe_struct.part"
+        "x_cpe_struct.part",
+        "_is_latest"
     ], inBackground=True))
-    vertex_collection.add_index(dict(type='persistent', fields=["_arango_cve_processor_note", "type"], inBackground=True, name=f"acvep_imports-type", sparse=True))
+    create_acvep_indexes(db)