From e9e34ad514163f2fa985ffa8acd44535f5269d81 Mon Sep 17 00:00:00 2001 From: Prabhu Subramanian Date: Sun, 24 Mar 2024 20:18:06 +0000 Subject: [PATCH 1/2] Module to identify tags for references Signed-off-by: Prabhu Subramanian --- contrib/cpe_research.py | 10 ++- vdb/lib/cve_model/tagger.py | 164 ++++++++++++++++++++++++++++++++++++ 2 files changed, 172 insertions(+), 2 deletions(-) create mode 100644 vdb/lib/cve_model/tagger.py diff --git a/contrib/cpe_research.py b/contrib/cpe_research.py index c2a7ff2..0868e94 100644 --- a/contrib/cpe_research.py +++ b/contrib/cpe_research.py @@ -7,7 +7,7 @@ from rich.table import Table from vdb.lib import KNOWN_PKG_TYPES, db6, CPE_FULL_REGEX -from vdb.lib.cve_model import CVE, CVE1 +from vdb.lib.cve_model import CVE, CVE1, tagger console = Console(markup=False, highlight=False, emoji=False) @@ -70,6 +70,8 @@ def propose_pseudo_purls() -> list: "netapp", "synology", "citrix", + "juniper", + "qnap", ] ) raw_hits = index_conn.execute( @@ -79,6 +81,7 @@ def propose_pseudo_purls() -> list: table.add_column("PURL prefix") table.add_column("CPEs") table.add_column("References") + table.add_column("Tags") with Live( table, console=console, refresh_per_second=4, vertical_overflow="visible" ): @@ -92,7 +95,7 @@ def propose_pseudo_purls() -> list: ref_urls = [ str(a.url.root).lower() for a in references - if "git" in str(a.url.root).lower() + if "git" not in str(a.url.root).lower() ] if not ref_urls: continue @@ -102,6 +105,8 @@ def propose_pseudo_purls() -> list: generic_cpes = [ acpe for acpe in cpes if acpe.startswith("cpe:2.3:a:generic") ] + tags = tagger.get_reference_tags(ref_urls) + tags = [a["tag"] for a in tags] proposed_purls = [] for generic_cpe in generic_cpes: all_parts = CPE_FULL_REGEX.match(generic_cpe) @@ -118,6 +123,7 @@ def propose_pseudo_purls() -> list: purl_prefix + "\n" + "\n".join(proposed_purls), cpes[0], "\n".join(ref_urls), + "\n".join(tags), ) diff --git a/vdb/lib/cve_model/tagger.py b/vdb/lib/cve_model/tagger.py new file mode 100644 index 0000000..c7e5e30 --- /dev/null +++ b/vdb/lib/cve_model/tagger.py @@ -0,0 +1,164 @@ +from vdb.lib.cve_model import Reference + +REFERENCE_TAGS_MAP = { + "exploit": [ + "exploit-db.com/", + "exploit-database", + "seebug.org", + "seclists.org", + "nu11secur1ty", + "packetstormsecurity.com", + "coresecurity.com", + "project-zero", + "0dd.zone", + "snyk.io/research/", + "chromium.googlesource.com/infra", + "synacktiv.com", + "bishopfox.com", + "zerodayinitiative.com", + "www.samba.org/samba/security/", + "www.synology.com/support/security/", + "us-cert.gov/advisories" + ], + "government-resource": [ + ".gov", + "cisa", + "kevc", + ], + "issue-tracking": [ + "bugzilla", + "bugs.", + "chat.", + "/issues", + "/merge_request", + "oss-fuzz", + "trac.", + "security-tracker.", + "/bugs", + ], + "mailing-list": [ + "openwall.com", + "oss-security", + "www.mail-archive.com", + "lists.", + "mail.", + "/discussion/", + "/archives/", + "groups.", + "/community", + "/forum", + "/discuss", + "-announce", + ], + "mitigation": [], + "not-applicable": [], + "patch": [ + "/commit", + "/pull", + "/code/ci", + "patch", + ], + "media-coverage": [ + "blog", + "support", + "media", + "tech-updates", + "/news", + "/article", + ".html", + "/entry", + ".txt", + "/comments/", + "youtube.com", + "medium.com", + "twitter.com", + ], + "release-notes": [ + "/release", + ".md", + "/changeset" + ], + "technical-description": [ + "poc", + "hackerone", + "bugcrowd", + "bounty", + "huntr.dev", + "bounties", + "attackerkb", + "support.", + ".pdf", + "docs.google.com", + ], + "third-party-advisory": [ + "research", + "xss", + "csrf", + "ssrf", + "sqli", + "disclosure", + "rapid7", + "reference", + ".me/" + ], + "vendor-advisory": [ + "oracle.com", + "curl.haxx.se", + "nodejs.org", + "/security.", + "/securityadvisories.", + "sec-consult.com", + "jenkins.io/security", + "support.f5.com", + "suricata-ids.org/", + "foxitsoftware.com/support/", + "success.trendmicro.com/", + "docs.jamf.com/", + "www.postgresql.org/about", + "access.redhat.com", + "support.apple.com", + "rubyonrails-security", + "usn.ubuntu.com", + "security.gentoo.org", + "debian.org", + "apache.org", + "gitlab.alpinelinux.org", + "bugs.busybox.net", + "/security-advisor", + "/alert", + "wordpress", + "wpvulndb", + "/bug/view/", + ], + "vdb-entry": [ + "/advisories", + "/vulnerabilit", + "cve-", + "ghsa-", + "dsa-", + "mal-", + "snyk.io/", + "portal.msrc.microsoft.com", + "/id/", + "/bid/", + "kb.", + "jvn.jp/", + "vulndb", + "vulncheck", + "glsa", + "rhsa-", + ] +} + + +def get_reference_tags(ref_urls: list[str | Reference]) -> list[dict[str, str]]: + """Tag the urls under references""" + tags = [] + for aref in ref_urls: + theurl = aref.url if isinstance(aref, Reference) else aref + for reference_tag, tag_patterns in REFERENCE_TAGS_MAP.items(): + for keyword_str in tag_patterns: + if keyword_str in theurl: + tags.append({"url": theurl, "tag": reference_tag}) + break + return tags From 710d938a5805e3963bd9f1403ee8b1d825a0c810 Mon Sep 17 00:00:00 2001 From: Prabhu Subramanian Date: Sun, 24 Mar 2024 20:24:43 +0000 Subject: [PATCH 2/2] Module to identify tags for references Signed-off-by: Prabhu Subramanian --- contrib/cpe_research.py | 4 ++-- vdb/lib/cve_model/tagger.py | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/contrib/cpe_research.py b/contrib/cpe_research.py index 0868e94..eab6895 100644 --- a/contrib/cpe_research.py +++ b/contrib/cpe_research.py @@ -106,7 +106,7 @@ def propose_pseudo_purls() -> list: acpe for acpe in cpes if acpe.startswith("cpe:2.3:a:generic") ] tags = tagger.get_reference_tags(ref_urls) - tags = [a["tag"] for a in tags] + tags = set([a["tag"] for a in tags]) proposed_purls = [] for generic_cpe in generic_cpes: all_parts = CPE_FULL_REGEX.match(generic_cpe) @@ -123,7 +123,7 @@ def propose_pseudo_purls() -> list: purl_prefix + "\n" + "\n".join(proposed_purls), cpes[0], "\n".join(ref_urls), - "\n".join(tags), + "\n".join(list(tags)), ) diff --git a/vdb/lib/cve_model/tagger.py b/vdb/lib/cve_model/tagger.py index c7e5e30..7250caf 100644 --- a/vdb/lib/cve_model/tagger.py +++ b/vdb/lib/cve_model/tagger.py @@ -56,7 +56,7 @@ "/commit", "/pull", "/code/ci", - "patch", + ".patch", ], "media-coverage": [ "blog", @@ -76,7 +76,8 @@ "release-notes": [ "/release", ".md", - "/changeset" + "/changeset", + "releases/" ], "technical-description": [ "poc", @@ -137,7 +138,7 @@ "ghsa-", "dsa-", "mal-", - "snyk.io/", + "/vuln/", "portal.msrc.microsoft.com", "/id/", "/bid/",