sqlite backend (#107)

* Fixes for CVE 5 conversion Signed-off-by: Prabhu Subramanian <[email protected]> Store to sqlite Signed-off-by: Prabhu Subramanian <[email protected]> * Remove sync option Signed-off-by: Prabhu Subramanian <[email protected]> * Set user interaction Signed-off-by: Prabhu Subramanian <[email protected]> * Convert to abspath Signed-off-by: Prabhu Subramanian <[email protected]> * Convert to CVE model Signed-off-by: Prabhu Subramanian <[email protected]> * lint fixes Signed-off-by: Prabhu Subramanian <[email protected]> * Added search by cve and stats method Signed-off-by: Prabhu Subramanian <[email protected]> * Added search by cve and stats method Signed-off-by: Prabhu Subramanian <[email protected]> * Improve insert performance Signed-off-by: Prabhu Subramanian <[email protected]> * Use apsw to force sqlite version and switch to jsonb for improved performance Signed-off-by: Prabhu Subramanian <[email protected]> * nested transaction is not supported Signed-off-by: Prabhu Subramanian <[email protected]> * Add new vdb.lib.cve_model package to pyproject.toml, remove tests from distribution. Signed-off-by: Caroline Russell <[email protected]> * Review comments Signed-off-by: Prabhu Subramanian <[email protected]> * Reduce duplicates Signed-off-by: Prabhu Subramanian <[email protected]> * lint fixes Signed-off-by: Prabhu Subramanian <[email protected]> * test with auto vacuum Signed-off-by: Prabhu Subramanian <[email protected]> * go back to json which is surprisingly more efficient than blob Signed-off-by: Prabhu Subramanian <[email protected]> * go back to blob Signed-off-by: Prabhu Subramanian <[email protected]> * Remove incorrect provider metadata Signed-off-by: Prabhu Subramanian <[email protected]> * Revert removal of provider metadata Signed-off-by: Prabhu Subramanian <[email protected]> * Adds a new purl prefix column to the index to assist with purl based searches Signed-off-by: Prabhu Subramanian <[email protected]> * Added git osv source Signed-off-by: Prabhu Subramanian <[email protected]> * Bug fix Signed-off-by: Prabhu Subramanian <[email protected]> * Create more aliases based on git url in references Signed-off-by: Prabhu Subramanian <[email protected]> * Improve ghsa parsing Signed-off-by: Prabhu Subramanian <[email protected]> * Bug fixes Signed-off-by: Prabhu Subramanian <[email protected]> --------- Signed-off-by: Prabhu Subramanian <[email protected]> Signed-off-by: Caroline Russell <[email protected]> Co-authored-by: Caroline Russell <[email protected]>
AppThreat · Mar 19, 2024 · 4950a83 · 4950a83
1 parent 81f6a29
commit 4950a83
Show file tree

Hide file tree

Showing 24 changed files with 1,745 additions and 1,282 deletions.
diff --git a/README.md b/README.md
@@ -37,7 +37,7 @@ pip install appthreat-vulnerability-db
 
 ## Usage
 
-This package is ideal as a library for managing vulnerabilities. This is used by [dep-scan](http://github.com/AppThreat/dep-scan), a free open-source dependency audit tool. However, there is a limited cli capability available with few features to test this tool directly.
+This package is ideal as a library for managing vulnerabilities. This is used by [owasp-dep-scan](http://github.com/owasp-dep-scan/dep-scan), a free open-source dependency audit tool. However, there is a limited cli capability available with few features to test this tool directly.
 
 ### Download pre-built database
 
@@ -52,23 +52,23 @@ oras pull ghcr.io/appthreat/vdb:v5 -o $VDB_HOME
 
 Cache application vulnerabilities
 
-```bash
+```shell
 vdb --cache
 ```
 
 Typical size of this database is over 1.1 GB.
 
 Cache application and OS vulnerabilities
 
-```bash
+```shell
 vdb --cache-os
 ```
 
 Note the size of the database with OS vulnerabilities is over 3.1 GB.
 
 Cache from just [OSV](https://osv.dev)
 
-```bash
+```shell
 vdb --cache --only-osv
 ```
 
@@ -77,31 +77,24 @@ It is possible to customise the cache behaviour by increasing the historic data
 - NVD_START_YEAR - Default: 2018. Supports upto 2002
 - GITHUB_PAGE_COUNT - Default: 2. Supports upto 20
 
-### Periodic sync
-
-To periodically sync the latest vulnerabilities and update the database cache.
-
-```bash
-vdb --sync
-```
-
 ### Basic search
 
 It is possible to perform simple search using the cli.
 
-```bash
-vdb --search android:8.0
-
-vdb --search google:android:8.0
-
-vdb --search android:8.0,simplesamlphp:1.14.11
-
+```shell
 vdb --search pkg:pypi/[email protected]
 
 # Full url and short form for swift
 vdb --search "pkg:swift/github.com/vapor/[email protected]"
 
 vdb --search "pkg:swift/vapor/[email protected]"
-```
 
-Syntax is package:version,package:version or vendor : package : version (Without space)
+# Search by cpe
+vdb --search "cpe:2.3:a:npm:gitblame:*:*:*:*:*:*:*:*"
+
+# Search by colon separated values
+vdb --search "npm:gitblame:0.0.1"
+
+# Search by CVE id
+vdb --search CVE-2024-25169
+```
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,20 +1,20 @@
 [project]
 name = "appthreat-vulnerability-db"
 version = "6.0.0"
-description = "AppThreat's vulnerability database and package search library with a built-in file based storage. OSV, CVE, GitHub, npm are the primary sources of vulnerabilities."
+description = "AppThreat's vulnerability database and package search library with a built-in sqlite based storage. OSV, CVE, GitHub, npm are the primary sources of vulnerabilities."
 authors = [
     {name = "Team AppThreat", email = "[email protected]"},
 ]
 dependencies = [
     "httpx[http2]",
     "appdirs",
-    "tabulate",
-    "msgpack==1.0.5",
     "orjson",
     "semver>=3.0.0",
     "packageurl-python",
     "cvss",
-    "pydantic[email]"
+    "pydantic[email]",
+    "rich",
+    "apsw>=3.45.2.0"
 ]
 requires-python = ">=3.10"
 readme = "README.md"
@@ -47,12 +47,13 @@ dev = [
     "black",
     "bandit",
     "flake8",
+    "pylint",
     "pytest",
     "pytest-cov"
 ]
 
 [tool.setuptools]
-packages = ["test", "vdb", "vdb.lib"]
+packages = ["vdb", "vdb.lib", "vdb.lib.cve_model"]
 
 [tool.pytest.ini_options]
 addopts="--showlocals -v --cov-report=term-missing --no-cov-on-fail --cov vdb"

diff --git a/test/data/CVE-2023-52426.json b/test/data/CVE-2023-52426.json
@@ -0,0 +1,95 @@
+{
+  "id": "CVE-2023-52426",
+  "sourceIdentifier": "[email protected]",
+  "published": "2024-02-04T20:15:46.120",
+  "lastModified": "2024-02-09T02:02:39.800",
+  "vulnStatus": "Analyzed",
+  "descriptions": [
+    {
+      "lang": "en",
+      "value": "libexpat through 2.5.0 allows recursive XML Entity Expansion if XML_DTD is undefined at compile time."
+    },
+    {
+      "lang": "es",
+      "value": "libexpat hasta 2.5.0 permite la expansión recursiva de entidades XML si XML_DTD no está definido en el momento de la compilación."
+    }
+  ],
+  "metrics": {
+    "cvssMetricV31": [
+      {
+        "source": "[email protected]",
+        "type": "Primary",
+        "cvssData": {
+          "version": "3.1",
+          "vectorString": "CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:N/I:N/A:H",
+          "attackVector": "LOCAL",
+          "attackComplexity": "LOW",
+          "privilegesRequired": "LOW",
+          "userInteraction": "NONE",
+          "scope": "UNCHANGED",
+          "confidentialityImpact": "NONE",
+          "integrityImpact": "NONE",
+          "availabilityImpact": "HIGH",
+          "baseScore": 5.5,
+          "baseSeverity": "MEDIUM"
+        },
+        "exploitabilityScore": 1.8,
+        "impactScore": 3.6
+      }
+    ]
+  },
+  "weaknesses": [
+    {
+      "source": "[email protected]",
+      "type": "Primary",
+      "description": [
+        {
+          "lang": "en",
+          "value": "CWE-776"
+        }
+      ]
+    }
+  ],
+  "configurations": [
+    {
+      "nodes": [
+        {
+          "operator": "OR",
+          "negate": false,
+          "cpeMatch": [
+            {
+              "vulnerable": true,
+              "criteria": "cpe:2.3:a:libexpat_project:libexpat:*:*:*:*:*:*:*:*",
+              "matchCriteriaId": "1C50909D-8A18-484B-A7DB-7EF4CA67C2CB",
+              "versionEndIncluding": "2.5.0"
+            }
+          ]
+        }
+      ]
+    }
+  ],
+  "references": [
+    {
+      "url": "https://cwe.mitre.org/data/definitions/776.html",
+      "source": "[email protected]",
+      "tags": [
+        "Technical Description"
+      ]
+    },
+    {
+      "url": "https://github.com/libexpat/libexpat/commit/0f075ec8ecb5e43f8fdca5182f8cca4703da0404",
+      "source": "[email protected]",
+      "tags": [
+        "Patch",
+        "Vendor Advisory"
+      ]
+    },
+    {
+      "url": "https://github.com/libexpat/libexpat/pull/777",
+      "source": "[email protected]",
+      "tags": [
+        "Vendor Advisory"
+      ]
+    }
+  ]
+}