Add tests for models

Signed-off-by: Tushar Goel <[email protected]>

Author: TG1999

vulnerabilities/importers/__init__.py

@@ -33,8 +33,6 @@
 from vulnerabilities.importers import ubuntu_usn
 from vulnerabilities.importers import vulnrichment
 from vulnerabilities.importers import xen
-from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline
-from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
 from vulnerabilities.pipelines import alpine_linux_importer
 from vulnerabilities.pipelines import github_importer
 from vulnerabilities.pipelines import gitlab_importer
@@ -51,56 +49,51 @@
 from vulnerabilities.pipelines.v2_importers import pypa_importer as pypa_importer_v2
 from vulnerabilities.pipelines.v2_importers import pysec_importer as pysec_importer_v2
 from vulnerabilities.pipelines.v2_importers import vulnrichment_importer as vulnrichment_importer_v2
+from vulnerabilities.utils import create_registry
 
-IMPORTERS_REGISTRY = [
-    nvd_importer_v2.NVDImporterPipeline,
-    github_importer_v2.GitHubAPIImporterPipeline,
-    npm_importer_v2.NpmImporterPipeline,
-    vulnrichment_importer_v2.VulnrichImporterPipeline,
-    apache_httpd_v2.ApacheHTTPDImporterPipeline,
-    pypa_importer_v2.PyPaImporterPipeline,
-    gitlab_importer_v2.GitLabImporterPipeline,
-    pysec_importer_v2.PyPIImporterPipeline,
-    nvd_importer.NVDImporterPipeline,
-    github_importer.GitHubAPIImporterPipeline,
-    gitlab_importer.GitLabImporterPipeline,
-    github_osv.GithubOSVImporter,
-    pypa_importer.PyPaImporterPipeline,
-    npm_importer.NpmImporterPipeline,
-    nginx_importer.NginxImporterPipeline,
-    pysec_importer.PyPIImporterPipeline,
-    apache_tomcat.ApacheTomcatImporter,
-    postgresql.PostgreSQLImporter,
-    debian.DebianImporter,
-    curl.CurlImporter,
-    epss.EPSSImporter,
-    vulnrichment.VulnrichImporter,
-    alpine_linux_importer.AlpineLinuxImporterPipeline,
-    ruby.RubyImporter,
-    apache_kafka.ApacheKafkaImporter,
-    openssl.OpensslImporter,
-    redhat.RedhatImporter,
-    archlinux.ArchlinuxImporter,
-    ubuntu.UbuntuImporter,
-    debian_oval.DebianOvalImporter,
-    retiredotnet.RetireDotnetImporter,
-    apache_httpd.ApacheHTTPDImporter,
-    mozilla.MozillaImporter,
-    gentoo.GentooImporter,
-    istio.IstioImporter,
-    project_kb_msr2019.ProjectKBMSRImporter,
-    suse_scores.SUSESeverityScoreImporter,
-    elixir_security.ElixirSecurityImporter,
-    xen.XenImporter,
-    ubuntu_usn.UbuntuUSNImporter,
-    fireeye.FireyeImporter,
-    oss_fuzz.OSSFuzzImporter,
-]
-
-IMPORTERS_REGISTRY = {
-    x.pipeline_id
-    if issubclass(x, VulnerableCodeBaseImporterPipeline)
-    or issubclass(x, VulnerableCodeBaseImporterPipelineV2)
-    else x.qualified_name: x
-    for x in IMPORTERS_REGISTRY
-}
+IMPORTERS_REGISTRY = create_registry(
+    [
+        nvd_importer_v2.NVDImporterPipeline,
+        github_importer_v2.GitHubAPIImporterPipeline,
+        npm_importer_v2.NpmImporterPipeline,
+        vulnrichment_importer_v2.VulnrichImporterPipeline,
+        apache_httpd_v2.ApacheHTTPDImporterPipeline,
+        pypa_importer_v2.PyPaImporterPipeline,
+        gitlab_importer_v2.GitLabImporterPipeline,
+        pysec_importer_v2.PyPIImporterPipeline,
+        nvd_importer.NVDImporterPipeline,
+        github_importer.GitHubAPIImporterPipeline,
+        gitlab_importer.GitLabImporterPipeline,
+        github_osv.GithubOSVImporter,
+        pypa_importer.PyPaImporterPipeline,
+        npm_importer.NpmImporterPipeline,
+        nginx_importer.NginxImporterPipeline,
+        pysec_importer.PyPIImporterPipeline,
+        apache_tomcat.ApacheTomcatImporter,
+        postgresql.PostgreSQLImporter,
+        debian.DebianImporter,
+        curl.CurlImporter,
+        epss.EPSSImporter,
+        vulnrichment.VulnrichImporter,
+        alpine_linux_importer.AlpineLinuxImporterPipeline,
+        ruby.RubyImporter,
+        apache_kafka.ApacheKafkaImporter,
+        openssl.OpensslImporter,
+        redhat.RedhatImporter,
+        archlinux.ArchlinuxImporter,
+        ubuntu.UbuntuImporter,
+        debian_oval.DebianOvalImporter,
+        retiredotnet.RetireDotnetImporter,
+        apache_httpd.ApacheHTTPDImporter,
+        mozilla.MozillaImporter,
+        gentoo.GentooImporter,
+        istio.IstioImporter,
+        project_kb_msr2019.ProjectKBMSRImporter,
+        suse_scores.SUSESeverityScoreImporter,
+        elixir_security.ElixirSecurityImporter,
+        xen.XenImporter,
+        ubuntu_usn.UbuntuUSNImporter,
+        fireeye.FireyeImporter,
+        oss_fuzz.OSSFuzzImporter,
+    ]
+)

vulnerabilities/improvers/__init__.py

@@ -31,45 +31,43 @@
     enhance_with_metasploit as enhance_with_metasploit_v2,
 )
 from vulnerabilities.pipelines.v2_improvers import flag_ghost_packages as flag_ghost_packages_v2
+from vulnerabilities.utils import create_registry
 
-IMPROVERS_REGISTRY = [
-    valid_versions.GitHubBasicImprover,
-    valid_versions.GitLabBasicImprover,
-    valid_versions.NginxBasicImprover,
-    valid_versions.ApacheHTTPDImprover,
-    valid_versions.DebianBasicImprover,
-    valid_versions.NpmImprover,
-    valid_versions.ElixirImprover,
-    valid_versions.ApacheTomcatImprover,
-    valid_versions.ApacheKafkaImprover,
-    valid_versions.IstioImprover,
-    valid_versions.DebianOvalImprover,
-    valid_versions.UbuntuOvalImprover,
-    valid_versions.OSSFuzzImprover,
-    valid_versions.RubyImprover,
-    valid_versions.GithubOSVImprover,
-    vulnerability_status.VulnerabilityStatusImprover,
-    valid_versions.CurlImprover,
-    flag_ghost_packages.FlagGhostPackagePipeline,
-    enhance_with_kev.VulnerabilityKevPipeline,
-    enhance_with_metasploit.MetasploitImproverPipeline,
-    enhance_with_exploitdb.ExploitDBImproverPipeline,
-    compute_package_risk.ComputePackageRiskPipeline,
-    compute_package_version_rank.ComputeVersionRankPipeline,
-    collect_commits.CollectFixCommitsPipeline,
-    add_cvss31_to_CVEs.CVEAdvisoryMappingPipeline,
-    remove_duplicate_advisories.RemoveDuplicateAdvisoriesPipeline,
-    populate_vulnerability_summary_pipeline.PopulateVulnerabilitySummariesPipeline,
-    exploitdb_v2.ExploitDBImproverPipeline,
-    enhance_with_kev_v2.VulnerabilityKevPipeline,
-    flag_ghost_packages_v2.FlagGhostPackagePipeline,
-    enhance_with_metasploit_v2.MetasploitImproverPipeline,
-    compute_package_risk_v2.ComputePackageRiskPipeline,
-    compute_version_rank_v2.ComputeVersionRankPipeline,
-    collect_commits_v2.CollectFixCommitsPipeline,
-]
-
-IMPROVERS_REGISTRY = {
-    x.pipeline_id if issubclass(x, VulnerableCodePipeline) else x.qualified_name: x
-    for x in IMPROVERS_REGISTRY
-}
+IMPROVERS_REGISTRY = create_registry(
+    [
+        valid_versions.GitHubBasicImprover,
+        valid_versions.GitLabBasicImprover,
+        valid_versions.NginxBasicImprover,
+        valid_versions.ApacheHTTPDImprover,
+        valid_versions.DebianBasicImprover,
+        valid_versions.NpmImprover,
+        valid_versions.ElixirImprover,
+        valid_versions.ApacheTomcatImprover,
+        valid_versions.ApacheKafkaImprover,
+        valid_versions.IstioImprover,
+        valid_versions.DebianOvalImprover,
+        valid_versions.UbuntuOvalImprover,
+        valid_versions.OSSFuzzImprover,
+        valid_versions.RubyImprover,
+        valid_versions.GithubOSVImprover,
+        vulnerability_status.VulnerabilityStatusImprover,
+        valid_versions.CurlImprover,
+        flag_ghost_packages.FlagGhostPackagePipeline,
+        enhance_with_kev.VulnerabilityKevPipeline,
+        enhance_with_metasploit.MetasploitImproverPipeline,
+        enhance_with_exploitdb.ExploitDBImproverPipeline,
+        compute_package_risk.ComputePackageRiskPipeline,
+        compute_package_version_rank.ComputeVersionRankPipeline,
+        collect_commits.CollectFixCommitsPipeline,
+        add_cvss31_to_CVEs.CVEAdvisoryMappingPipeline,
+        remove_duplicate_advisories.RemoveDuplicateAdvisoriesPipeline,
+        populate_vulnerability_summary_pipeline.PopulateVulnerabilitySummariesPipeline,
+        exploitdb_v2.ExploitDBImproverPipeline,
+        enhance_with_kev_v2.VulnerabilityKevPipeline,
+        flag_ghost_packages_v2.FlagGhostPackagePipeline,
+        enhance_with_metasploit_v2.MetasploitImproverPipeline,
+        compute_package_risk_v2.ComputePackageRiskPipeline,
+        compute_version_rank_v2.ComputeVersionRankPipeline,
+        collect_commits_v2.CollectFixCommitsPipeline,
+    ]
+)

vulnerabilities/models.py

@@ -2556,6 +2556,15 @@ class AdvisoryV2(models.Model):
         help_text="Unique ID for the datasource used for this advisory ." "e.g.: nginx_importer_v2",
     )
 
+    advisory_id = models.CharField(
+        max_length=50,
+        blank=False,
+        null=False,
+        unique=False,
+        help_text="An advisory is a unique vulnerability identifier in some database, "
+        "such as PYSEC-2020-2233",
+    )
+
     avid = models.CharField(
         max_length=500,
         blank=False,
@@ -2565,14 +2574,6 @@ class AdvisoryV2(models.Model):
     )
 
     # This is similar to a name
-    advisory_id = models.CharField(
-        max_length=50,
-        blank=False,
-        null=False,
-        unique=False,
-        help_text="An advisory is a unique vulnerability identifier in some database, "
-        "such as PYSEC-2020-2233",
-    )
 
     # This is similar to a version
     unique_content_id = models.CharField(

vulnerabilities/pipelines/__init__.py

@@ -159,14 +159,6 @@ def on_failure(self):
         """
         pass
 
-    @classproperty
-    def pipeline_id(cls):
-        """Return unique pipeline_id set in cls.pipeline_id"""
-
-        if cls.pipeline_id is None or cls.pipeline_id == "":
-            raise NotImplementedError("pipeline_id is not defined or is empty")
-        return cls.pipeline_id
-
 
 class VulnerableCodeBaseImporterPipeline(VulnerableCodePipeline):
     """
@@ -273,10 +265,8 @@ class VulnerableCodeBaseImporterPipelineV2(VulnerableCodePipeline):
 
     pipeline_id = None  # Unique Pipeline ID, this should be the name of pipeline module.
     license_url = None
-    label = None
     spdx_license_expression = None
     repo_url = None
-    importer_name = None
     advisory_confidence = MAX_CONFIDENCE
     ignorable_versions = []
     unfurl_version_ranges = False

vulnerabilities/pipelines/v2_importers/apache_httpd_importer.py

@@ -139,11 +139,15 @@ def get_weaknesses(cve_data):
 
 
 class ApacheHTTPDImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
+    """
+    Apache HTTPD Importer Pipeline
+
+    This pipeline imports security advisories from the Apache HTTPD project.
+    """
+
     pipeline_id = "apache_httpd_importer_v2"
-    label = "Apache-Httpd"
     spdx_license_expression = "Apache-2.0"
     license_url = "https://www.apache.org/licenses/LICENSE-2.0"
-    importer_name = "Apache HTTPD Importer"
     base_url = "https://httpd.apache.org/security/json/"
     unfurl_version_ranges = True
 

vulnerabilities/pipelines/v2_importers/elixir_security_importer.py

@@ -25,13 +25,17 @@
 
 
 class ElixirSecurityImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
+    """
+    Elixir Security Advisiories Importer Pipeline
+
+    This pipeline imports security advisories for elixir.
+    """
 
     pipeline_id = "elixir_security_importer_v2"
-    label = "Elixir Security"
-    repo_url = "git+https://github.com/dependabot/elixir-security-advisories"
-    license_url = "https://github.com/dependabot/elixir-security-advisories/blob/master/LICENSE.txt"
     spdx_license_expression = "CC0-1.0"
-    importer_name = "Elixir Security Importer"
+    license_url = "https://github.com/dependabot/elixir-security-advisories/blob/master/LICENSE.txt"
+    repo_url = "git+https://github.com/dependabot/elixir-security-advisories"
+    unfurl_version_ranges = True
 
     @classmethod
     def steps(cls):

vulnerabilities/pipelines/v2_importers/github_importer.py

@@ -33,15 +33,17 @@
 
 
 class GitHubAPIImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
-    """Collect GitHub advisories."""
+    """
+    GitHub Importer Pipeline
+
+    This pipeline imports security advisories from GitHub Security Advisories.
+    """
 
     pipeline_id = "github_importer_v2"
-    label = "GitHub"
     spdx_license_expression = "CC-BY-4.0"
     license_url = "https://github.com/github/advisory-database/blob/main/LICENSE.md"
-    importer_name = "GHSA Importer"
-
     unfurl_version_ranges = True
+
     ignorable_versions = frozenset(
         [
             "0.1-bulbasaur",

vulnerabilities/pipelines/v2_importers/gitlab_importer.py

@@ -34,15 +34,16 @@
 
 
 class GitLabImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
-    """Collect advisory from GitLab Advisory Database (Open Source Edition)."""
+    """
+    GitLab Importer Pipeline
+
+    Collect advisory from GitLab Advisory Database (Open Source Edition).
+    """
 
     pipeline_id = "gitlab_importer_v2"
-    label = "GitLab"
     spdx_license_expression = "MIT"
     license_url = "https://gitlab.com/gitlab-org/advisories-community/-/blob/main/LICENSE"
-    importer_name = "GitLab Importer"
     repo_url = "git+https://gitlab.com/gitlab-org/advisories-community/"
-
     unfurl_version_ranges = True
 
     @classmethod

vulnerabilities/pipelines/v2_importers/npm_importer.py

@@ -30,14 +30,16 @@
 
 
 class NpmImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
-    """Collect advisories from nodejs GitHub repository."""
+    """
+    Node.js Security Working Group importer pipeline
+
+    Import advisories from nodejs security working group including node proper advisories and npm advisories.
+    """
 
     pipeline_id = "nodejs_security_wg"
     spdx_license_expression = "MIT"
     license_url = "https://github.com/nodejs/security-wg/blob/main/LICENSE.md"
     repo_url = "git+https://github.com/nodejs/security-wg"
-    importer_name = "npm Importer"
-
     unfurl_version_ranges = True
 
     @classmethod
@@ -120,7 +122,7 @@ def to_advisory_data(self, file: Path) -> Iterable[AdvisoryData]:
         advsisory_aliases = data.get("cves") or []
 
         return AdvisoryData(
-            advisory_id=f"NODESEC-NPM-{id}",
+            advisory_id=f"npm-{id}",
             aliases=advsisory_aliases,
             summary=build_description(summary=summary, description=description),
             date_published=date_published,

vulnerabilities/pipelines/v2_importers/nvd_importer.py

@@ -28,10 +28,13 @@
 
 
 class NVDImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
-    """Collect advisories from NVD."""
+    """
+    NVD Importer Pipeline
+
+    Collect advisories from NVD.
+    """
 
     pipeline_id = "nvd_importer_v2"
-    label = "NVD"
     # See https://github.com/nexB/vulnerablecode/issues/665 for follow up
     spdx_license_expression = (
         "LicenseRef-scancode-us-govt-public-domain  AND LicenseRef-scancode-cve-tou"
@@ -66,7 +69,6 @@ class NVDImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
         INFORMATION THEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF
         MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
     """
-    importer_name = "NVD Importer"
 
     @classmethod
     def steps(cls):