Skip to content

Commit e579e7f

Browse files
authored
"Check Vulnerabilities" pipeline using VulnerableCodeDB #284 (#551)
Signed-off-by: Thomas Druez <[email protected]>
1 parent 3546dfd commit e579e7f

File tree

12 files changed

+384
-13
lines changed

12 files changed

+384
-13
lines changed

CHANGELOG.rst

+6
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,12 @@ Changelog
44
v31.1.0 (unreleased)
55
--------------------
66

7+
- Add a new "check vulnerabilities" pipeline to lookup vulnerabilities in the
8+
VulnerableCode database for all project discovered packages.
9+
Vulnerability data is stored in the extra_data field of each package.
10+
More details about VulnerableCode at https://github.com/nexB/vulnerablecode/
11+
https://github.com/nexB/scancode.io/issues/101
12+
713
- Generate SBOM (Software Bill of Materials) compliant with the SPDX 2.3 specification
814
as a new downloadable output.
915
https://github.com/nexB/scancode.io/issues/389

docs/built-in-pipelines.rst

+8
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,14 @@ Pipeline Base Class
1515
:members:
1616
:member-order: bysource
1717

18+
.. _pipeline_check_vulnerabilities:
19+
20+
Check Vulnerabilities
21+
---------------------
22+
.. autoclass:: scanpipe.pipelines.check_vulnerabilities.CheckVulnerabilities()
23+
:members:
24+
:member-order: bysource
25+
1826
.. _pipeline_docker:
1927

2028
Docker Image Analysis

scancodeio/settings.py

+7
Original file line numberDiff line numberDiff line change
@@ -311,3 +311,10 @@
311311
REST_FRAMEWORK["DEFAULT_PERMISSION_CLASSES"] = (
312312
"rest_framework.permissions.AllowAny",
313313
)
314+
315+
# VulnerableCode integration
316+
317+
VULNERABLECODE_URL = env.str("VULNERABLECODE_URL", default="")
318+
VULNERABLECODE_USER = env.str("VULNERABLECODE_USER", default="")
319+
VULNERABLECODE_PASSWORD = env.str("VULNERABLECODE_PASSWORD", default="")
320+
VULNERABLECODE_API_KEY = env.str("VULNERABLECODE_API_KEY", default="")

scancodeio/static/main.js

+25-10
Original file line numberDiff line numberDiff line change
@@ -63,17 +63,32 @@ function setupCloseModalButtons() {
6363
function setupTabs() {
6464
const $tabLinks = getAll('.tabs a');
6565

66+
function activateTab($tabLink) {
67+
const activeLink = document.querySelector('.tabs .is-active');
68+
const activeTabContent = document.querySelector('.tab-content.is-active');
69+
const targetId = $tabLink.dataset.target;
70+
const targetTabContent = document.getElementById(targetId);
71+
72+
activeLink.classList.remove('is-active');
73+
$tabLink.parentNode.classList.add('is-active');
74+
if (activeTabContent) activeTabContent.classList.remove('is-active');
75+
if (targetTabContent) targetTabContent.classList.add('is-active');
76+
77+
// Set the active tab in the URL hash. The "tab-" prefix is removed to avoid
78+
// un-wanted scrolling to the related "id" element
79+
document.location.hash = targetId.replace('tab-', '');
80+
}
81+
82+
// Activate the related tab if hash is present in URL
83+
if (document.location.hash !== "") {
84+
let tabName = document.location.hash.slice(1);
85+
let tabLink = document.querySelector(`a[data-target="tab-${tabName}"]`);
86+
if (tabLink) activateTab(tabLink);
87+
}
88+
6689
$tabLinks.forEach(function ($el) {
67-
$el.addEventListener('click', function (event) {
68-
const activeLink = document.querySelector('.tabs .is-active');
69-
const activeTabContent = document.querySelector('.tab-content.is-active');
70-
const target_id = $el.dataset.target;
71-
const targetTabContent = document.getElementById(target_id);
72-
73-
activeLink.classList.remove('is-active');
74-
$el.parentNode.classList.add('is-active');
75-
if (activeTabContent) activeTabContent.classList.remove('is-active');
76-
if (targetTabContent) targetTabContent.classList.add('is-active');
90+
$el.addEventListener('click', function () {
91+
activateTab($el)
7792
});
7893
});
7994
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
# SPDX-License-Identifier: Apache-2.0
2+
#
3+
# http://nexb.com and https://github.com/nexB/scancode.io
4+
# The ScanCode.io software is licensed under the Apache License version 2.0.
5+
# Data generated with ScanCode.io is provided as-is without warranties.
6+
# ScanCode is a trademark of nexB Inc.
7+
#
8+
# You may not use this software except in compliance with the License.
9+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
10+
# Unless required by applicable law or agreed to in writing, software distributed
11+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
13+
# specific language governing permissions and limitations under the License.
14+
#
15+
# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
16+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
17+
# ScanCode.io should be considered or used as legal advice. Consult an Attorney
18+
# for any legal advice.
19+
#
20+
# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
21+
# Visit https://github.com/nexB/scancode.io for support and download.
22+
23+
from scanpipe.pipelines import Pipeline
24+
from scanpipe.pipes import vulnerablecode
25+
26+
27+
class CheckVulnerabilities(Pipeline):
28+
"""
29+
A pipeline to check for discovered packages vulnerabilities in the VulnerableCode
30+
database.
31+
32+
Vulnerability data is stored in the extra_data field of each package.
33+
"""
34+
35+
@classmethod
36+
def steps(cls):
37+
return (
38+
cls.check_vulnerablecode_service_availability,
39+
cls.lookup_vulnerabilities,
40+
)
41+
42+
def check_vulnerablecode_service_availability(self):
43+
"""
44+
Check if the VulnerableCode service if configured and available.
45+
"""
46+
if not vulnerablecode.is_configured():
47+
raise Exception("VulnerableCode is not configured.")
48+
49+
if not vulnerablecode.is_available():
50+
raise Exception("VulnerableCode is not available.")
51+
52+
def lookup_vulnerabilities(self):
53+
"""
54+
Check for vulnerabilities on each of the project's discovered package.
55+
"""
56+
packages = self.project.discoveredpackages.all()
57+
58+
for package in packages:
59+
purl = vulnerablecode.get_base_purl(package.package_url)
60+
vulnerabilities = vulnerablecode.get_vulnerabilities_by_purl(purl)
61+
package.update_extra_data({"discovered_vulnerabilities": vulnerabilities})

scanpipe/pipes/vulnerablecode.py

+216
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,216 @@
1+
# SPDX-License-Identifier: Apache-2.0
2+
#
3+
# http://nexb.com and https://github.com/nexB/scancode.io
4+
# The ScanCode.io software is licensed under the Apache License version 2.0.
5+
# Data generated with ScanCode.io is provided as-is without warranties.
6+
# ScanCode is a trademark of nexB Inc.
7+
#
8+
# You may not use this software except in compliance with the License.
9+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
10+
# Unless required by applicable law or agreed to in writing, software distributed
11+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
13+
# specific language governing permissions and limitations under the License.
14+
#
15+
# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
16+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
17+
# ScanCode.io should be considered or used as legal advice. Consult an Attorney
18+
# for any legal advice.
19+
#
20+
# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
21+
# Visit https://github.com/nexB/scancode.io for support and download.
22+
23+
import logging
24+
25+
from django.conf import settings
26+
27+
import requests
28+
29+
label = "VulnerableCode"
30+
logger = logging.getLogger(__name__)
31+
session = requests.Session()
32+
33+
34+
VULNERABLECODE_API_URL = None
35+
VULNERABLECODE_URL = settings.VULNERABLECODE_URL
36+
if VULNERABLECODE_URL:
37+
VULNERABLECODE_API_URL = f'{VULNERABLECODE_URL.rstrip("/")}/api/'
38+
39+
# Basic Authentication
40+
VULNERABLECODE_USER = settings.VULNERABLECODE_USER
41+
VULNERABLECODE_PASSWORD = settings.VULNERABLECODE_PASSWORD
42+
basic_auth_enabled = VULNERABLECODE_USER and VULNERABLECODE_PASSWORD
43+
if basic_auth_enabled:
44+
session.auth = (VULNERABLECODE_USER, VULNERABLECODE_PASSWORD)
45+
46+
# Authentication with single API key
47+
VULNERABLECODE_API_KEY = settings.VULNERABLECODE_API_KEY
48+
if VULNERABLECODE_API_KEY:
49+
session.headers.update({"Authorization": f"Token {VULNERABLECODE_API_KEY}"})
50+
51+
52+
def is_configured():
53+
"""
54+
Returns True if the required VulnerableCode settings have been set.
55+
"""
56+
if VULNERABLECODE_API_URL:
57+
return True
58+
return False
59+
60+
61+
def is_available():
62+
"""
63+
Returns True if the configured VulnerableCode server is available.
64+
"""
65+
if not is_configured():
66+
return False
67+
68+
try:
69+
response = session.head(VULNERABLECODE_API_URL)
70+
response.raise_for_status()
71+
except requests.exceptions.RequestException as request_exception:
72+
logger.debug(f"{label} is_available() error: {request_exception}")
73+
return False
74+
75+
return response.status_code == requests.codes.ok
76+
77+
78+
def get_base_purl(purl):
79+
"""
80+
Returns the `purl` without qualifiers and subpath.
81+
"""
82+
return purl.split("?")[0]
83+
84+
85+
def get_purls(packages, base=False):
86+
"""
87+
Returns the PURLs for the given list of `packages`.
88+
Do not include qualifiers nor subpath when `base` is provided.
89+
"""
90+
return [
91+
get_base_purl(package_url) if base else package_url
92+
for package in packages
93+
if (package_url := package.package_url)
94+
]
95+
96+
97+
def request_get(
98+
url,
99+
payload=None,
100+
timeout=None,
101+
):
102+
"""
103+
Wraps the HTTP request calls on the API.
104+
"""
105+
if not url:
106+
return
107+
108+
params = {"format": "json"}
109+
if payload:
110+
params.update(payload)
111+
112+
logger.debug(f"VulnerableCode: url={url} params={params}")
113+
try:
114+
response = session.get(url, params=params, timeout=timeout)
115+
response.raise_for_status()
116+
return response.json()
117+
except (requests.RequestException, ValueError, TypeError) as exception:
118+
logger.debug(f"{label} [Exception] {exception}")
119+
120+
121+
def request_post(
122+
url,
123+
data,
124+
timeout=None,
125+
):
126+
try:
127+
response = session.post(url, json=data, timeout=timeout)
128+
response.raise_for_status()
129+
return response.json()
130+
except (requests.RequestException, ValueError, TypeError) as exception:
131+
logger.debug(f"{label} [Exception] {exception}")
132+
133+
134+
def _get_vulnerabilities(
135+
url,
136+
field_name,
137+
field_value,
138+
timeout=None,
139+
):
140+
"""
141+
Get the list of vulnerabilities.
142+
"""
143+
payload = {field_name: field_value}
144+
145+
response = request_get(url=url, payload=payload, timeout=timeout)
146+
if response and response.get("count"):
147+
results = response["results"]
148+
return results
149+
150+
151+
def get_vulnerabilities_by_purl(
152+
purl,
153+
timeout=None,
154+
api_url=VULNERABLECODE_API_URL,
155+
):
156+
"""
157+
Get the list of vulnerabilities providing a package `purl`.
158+
"""
159+
return _get_vulnerabilities(
160+
url=f"{api_url}packages/",
161+
field_name="purl",
162+
field_value=get_base_purl(purl),
163+
timeout=timeout,
164+
)
165+
166+
167+
def get_vulnerabilities_by_cpe(
168+
cpe,
169+
timeout=None,
170+
api_url=VULNERABLECODE_API_URL,
171+
):
172+
"""
173+
Get the list of vulnerabilities providing a package or component `cpe`.
174+
"""
175+
return _get_vulnerabilities(
176+
url=f"{api_url}cpes/",
177+
field_name="cpe",
178+
field_value=cpe,
179+
timeout=timeout,
180+
)
181+
182+
183+
def bulk_search_by_purl(
184+
purls,
185+
timeout=None,
186+
api_url=VULNERABLECODE_API_URL,
187+
):
188+
"""
189+
Bulk search of vulnerabilities using the provided list of `purls`.
190+
"""
191+
url = f"{api_url}packages/bulk_search"
192+
193+
data = {
194+
"purls": purls,
195+
}
196+
197+
logger.debug(f"VulnerableCode: url={url} purls_count={len(purls)}")
198+
return request_post(url, data, timeout)
199+
200+
201+
def bulk_search_by_cpes(
202+
cpes,
203+
timeout=None,
204+
api_url=VULNERABLECODE_API_URL,
205+
):
206+
"""
207+
Bulk search of vulnerabilities using the provided list of `cpes`.
208+
"""
209+
url = f"{api_url}cpes/bulk_search"
210+
211+
data = {
212+
"cpes": cpes,
213+
}
214+
215+
logger.debug(f"VulnerableCode: url={url} cpes_count={len(cpes)}")
216+
return request_post(url, data, timeout)

scanpipe/templates/scanpipe/package_list.html

+5
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,11 @@
2424
<tr class="break-word">
2525
<td style="min-width: 500px;" title="{{ package.package_uid }}">
2626
<a href="{{ package.get_absolute_url }}">{{ package.package_url }}</a>
27+
{% if package.extra_data.discovered_vulnerabilities %}
28+
<a href="{{ package.get_absolute_url }}#extra_data">
29+
<i class="fas fa-bug fa-sm has-text-danger" title="Vulnerabilities"></i>
30+
</a>
31+
{% endif %}
2732
</td>
2833
<td style="min-width: 300px; max-width: 400px;">
2934
{{ package.license_expression|linebreaksbr }}

scanpipe/templates/scanpipe/tabset/tab_default.html

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
{{ field_data.label }}
55
</dt>
66
<dd class="mb-4">
7-
<pre class="break-all is-small">{{ field_data.value|default:''|default_if_none:''|urlize }}</pre>
7+
<pre class="break-all wrap is-small">{{ field_data.value|default:''|default_if_none:''|urlize }}</pre>
88
</dd>
99
{% endfor %}
1010
</dl>

scanpipe/templates/scanpipe/tabset/tabset.html

+7-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,13 @@
66
{% if tab_data.icon_class %}
77
<span class="icon is-small"><i class="{{ tab_data.icon_class }}"></i></span>
88
{% endif %}
9-
<span>{{ label|capfirst }}</span>
9+
<span>
10+
{% if tab_data.verbose_name %}
11+
{{ tab_data.verbose_name }}
12+
{% else %}
13+
{{ label|capfirst }}
14+
{% endif %}
15+
</span>
1016
</a>
1117
</li>
1218
{% endfor %}

0 commit comments

Comments
 (0)