Skip to content

Commit 1f0a8aa

Browse files
committed
Add a "Check Vulnerabilities" pipeline #284
Signed-off-by: Thomas Druez <[email protected]>
1 parent e417f53 commit 1f0a8aa

File tree

6 files changed

+352
-1
lines changed

6 files changed

+352
-1
lines changed

docs/built-in-pipelines.rst

+8
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,14 @@ Pipeline Base Class
1515
:members:
1616
:member-order: bysource
1717

18+
.. _pipeline_check_vulnerabilities:
19+
20+
Check Vulnerabilities
21+
---------------------
22+
.. autoclass:: scanpipe.pipelines.check_vulnerabilities.CheckVulnerabilities()
23+
:members:
24+
:member-order: bysource
25+
1826
.. _pipeline_docker:
1927

2028
Docker Image Analysis

scancodeio/settings.py

+7
Original file line numberDiff line numberDiff line change
@@ -311,3 +311,10 @@
311311
REST_FRAMEWORK["DEFAULT_PERMISSION_CLASSES"] = (
312312
"rest_framework.permissions.AllowAny",
313313
)
314+
315+
# VulnerableCode integration
316+
317+
VULNERABLECODE_URL = env.str("VULNERABLECODE_URL", default="")
318+
VULNERABLECODE_USER = env.str("VULNERABLECODE_USER", default="")
319+
VULNERABLECODE_PASSWORD = env.str("VULNERABLECODE_PASSWORD", default="")
320+
VULNERABLECODE_API_KEY = env.str("VULNERABLECODE_API_KEY", default="")
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
# SPDX-License-Identifier: Apache-2.0
2+
#
3+
# http://nexb.com and https://github.com/nexB/scancode.io
4+
# The ScanCode.io software is licensed under the Apache License version 2.0.
5+
# Data generated with ScanCode.io is provided as-is without warranties.
6+
# ScanCode is a trademark of nexB Inc.
7+
#
8+
# You may not use this software except in compliance with the License.
9+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
10+
# Unless required by applicable law or agreed to in writing, software distributed
11+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
13+
# specific language governing permissions and limitations under the License.
14+
#
15+
# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
16+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
17+
# ScanCode.io should be considered or used as legal advice. Consult an Attorney
18+
# for any legal advice.
19+
#
20+
# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
21+
# Visit https://github.com/nexB/scancode.io for support and download.
22+
23+
from scanpipe.pipelines import Pipeline
24+
from scanpipe.pipes import vulnerablecode
25+
26+
27+
class CheckVulnerabilities(Pipeline):
28+
"""
29+
A pipeline to check for discovered packages vulnerabilities in the VulnerableCode
30+
database.
31+
32+
Vulnerability data is stored in the extra_data field of each package.
33+
"""
34+
35+
@classmethod
36+
def steps(cls):
37+
return (
38+
cls.check_vulnerablecode_service_availability,
39+
cls.lookup_vulnerabilities,
40+
)
41+
42+
def check_vulnerablecode_service_availability(self):
43+
"""
44+
Check if the VulnerableCode service if configured and available.
45+
"""
46+
if not vulnerablecode.is_configured():
47+
raise Exception("VulnerableCode is not configured.")
48+
49+
if not vulnerablecode.is_available():
50+
raise Exception("VulnerableCode is not available.")
51+
52+
def lookup_vulnerabilities(self):
53+
"""
54+
Check for vulnerabilities on each of the project's discovered package.
55+
"""
56+
packages = self.project.discoveredpackages.all()
57+
58+
for package in packages:
59+
purl = vulnerablecode.get_base_purl(package.package_url)
60+
vulnerabilities = vulnerablecode.get_vulnerabilities_by_purl(purl)
61+
package.update_extra_data({"discovered_vulnerabilities": vulnerabilities})

scanpipe/pipes/vulnerablecode.py

+274
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,274 @@
1+
# SPDX-License-Identifier: Apache-2.0
2+
#
3+
# http://nexb.com and https://github.com/nexB/scancode.io
4+
# The ScanCode.io software is licensed under the Apache License version 2.0.
5+
# Data generated with ScanCode.io is provided as-is without warranties.
6+
# ScanCode is a trademark of nexB Inc.
7+
#
8+
# You may not use this software except in compliance with the License.
9+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
10+
# Unless required by applicable law or agreed to in writing, software distributed
11+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
13+
# specific language governing permissions and limitations under the License.
14+
#
15+
# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
16+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
17+
# ScanCode.io should be considered or used as legal advice. Consult an Attorney
18+
# for any legal advice.
19+
#
20+
# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
21+
# Visit https://github.com/nexB/scancode.io for support and download.
22+
23+
import logging
24+
25+
from django.conf import settings
26+
27+
import requests
28+
29+
label = "VulnerableCode"
30+
logger = logging.getLogger(__name__)
31+
session = requests.Session()
32+
33+
34+
VULNERABLECODE_API_URL = None
35+
VULNERABLECODE_URL = settings.VULNERABLECODE_URL
36+
if VULNERABLECODE_URL:
37+
VULNERABLECODE_API_URL = f'{VULNERABLECODE_URL.rstrip("/")}/api/'
38+
39+
# Basic Authentication
40+
VULNERABLECODE_USER = settings.VULNERABLECODE_USER
41+
VULNERABLECODE_PASSWORD = settings.VULNERABLECODE_PASSWORD
42+
basic_auth_enabled = VULNERABLECODE_USER and VULNERABLECODE_PASSWORD
43+
if basic_auth_enabled:
44+
session.auth = (VULNERABLECODE_USER, VULNERABLECODE_PASSWORD)
45+
46+
# Authentication with single API key
47+
VULNERABLECODE_API_KEY = settings.VULNERABLECODE_API_KEY
48+
if VULNERABLECODE_API_KEY:
49+
session.headers.update({"Authorization": f"Token {VULNERABLECODE_API_KEY}"})
50+
51+
52+
def is_configured():
53+
"""
54+
Returns True if the required VulnerableCode settings have been set.
55+
"""
56+
if VULNERABLECODE_API_URL:
57+
return True
58+
return False
59+
60+
61+
def is_service_available(label, session, url, raise_exceptions):
62+
"""
63+
Base function that checks if a configured integration service is available.
64+
"""
65+
try:
66+
response = session.head(url)
67+
response.raise_for_status()
68+
except requests.exceptions.RequestException as request_exception:
69+
logger.debug(f"{label} is_available() error: {request_exception}")
70+
if raise_exceptions:
71+
raise
72+
return False
73+
74+
return response.status_code == requests.codes.ok
75+
76+
77+
def is_available(raise_exceptions=False):
78+
"""
79+
Returns True if the configured VulnerableCode server is available.
80+
"""
81+
if not is_configured():
82+
return False
83+
84+
return is_service_available(
85+
label, session, VULNERABLECODE_API_URL, raise_exceptions
86+
)
87+
88+
89+
def request_get(
90+
url,
91+
payload=None,
92+
timeout=None,
93+
):
94+
"""
95+
Wraps the HTTP request calls on the API.
96+
"""
97+
if not url:
98+
return
99+
100+
params = {"format": "json"}
101+
if payload:
102+
params.update(payload)
103+
104+
logger.debug(f"VulnerableCode: url={url} params={params}")
105+
try:
106+
response = session.get(url, params=params, timeout=timeout)
107+
response.raise_for_status()
108+
return response.json()
109+
except (requests.RequestException, ValueError, TypeError) as exception:
110+
logger.debug(f"{label} [Exception] {exception}")
111+
112+
113+
def request_post(
114+
url,
115+
data,
116+
timeout=None,
117+
):
118+
try:
119+
response = session.post(url, json=data, timeout=timeout)
120+
response.raise_for_status()
121+
return response.json()
122+
except (requests.RequestException, ValueError, TypeError) as exception:
123+
logger.debug(f"{label} [Exception] {exception}")
124+
125+
126+
def get_base_purl(purl):
127+
"""
128+
Returns the `purl` without the qualifiers and the subpath.
129+
"""
130+
return purl.split("?")[0]
131+
132+
133+
def _get_vulnerabilities(
134+
url,
135+
field_name,
136+
field_value,
137+
timeout=None,
138+
):
139+
"""
140+
Get the list of vulnerabilities.
141+
"""
142+
payload = {field_name: field_value}
143+
144+
response = request_get(url=url, payload=payload, timeout=timeout)
145+
if response and response.get("count"):
146+
results = response["results"]
147+
return results
148+
149+
150+
def get_vulnerabilities_by_purl(
151+
purl,
152+
timeout=None,
153+
api_url=VULNERABLECODE_API_URL,
154+
):
155+
"""
156+
Get the list of vulnerabilities providing a package `purl`.
157+
"""
158+
return _get_vulnerabilities(
159+
url=f"{api_url}packages/",
160+
field_name="purl",
161+
field_value=get_base_purl(purl),
162+
timeout=timeout,
163+
)
164+
165+
166+
def get_vulnerabilities_by_cpe(
167+
cpe,
168+
timeout=None,
169+
api_url=VULNERABLECODE_API_URL,
170+
):
171+
"""
172+
Get the list of vulnerabilities providing a package or component `cpe`.
173+
"""
174+
return _get_vulnerabilities(
175+
url=f"{api_url}cpes/",
176+
field_name="cpe",
177+
field_value=cpe,
178+
timeout=timeout,
179+
)
180+
181+
182+
def bulk_search_by_purl(
183+
purls,
184+
timeout=None,
185+
api_url=VULNERABLECODE_API_URL,
186+
):
187+
"""
188+
Bulk search of vulnerabilities using the provided list of `purls`.
189+
"""
190+
url = f"{api_url}packages/bulk_search"
191+
192+
data = {
193+
"purls": purls,
194+
}
195+
196+
logger.debug(f"VulnerableCode: url={url} purls_count={len(purls)}")
197+
return request_post(url, data, timeout)
198+
199+
200+
def bulk_search_by_cpes(
201+
cpes,
202+
timeout=None,
203+
api_url=VULNERABLECODE_API_URL,
204+
):
205+
"""
206+
Bulk search of vulnerabilities using the provided list of `cpes`.
207+
"""
208+
url = f"{api_url}cpes/bulk_search"
209+
210+
data = {
211+
"cpes": cpes,
212+
}
213+
214+
logger.debug(f"VulnerableCode: url={url} cpes_count={len(cpes)}")
215+
return request_post(url, data, timeout)
216+
217+
218+
def get_purls(packages):
219+
"""
220+
Returns the PURLs for the given list of `packages`.
221+
List comprehension is not used on purpose to avoid crafting each
222+
PURL twice.
223+
"""
224+
purls = []
225+
for package in packages:
226+
package_url = package.package_url
227+
if package_url:
228+
purls.append(package_url)
229+
return purls
230+
231+
232+
def get_vulnerable_purls(packages):
233+
"""
234+
Returns a list of PURLs for which at least one `affected_by_vulnerabilities`
235+
was found in the VulnerableCodeDB for the given list of `packages`.
236+
"""
237+
purls = get_purls(packages)
238+
239+
if not purls:
240+
return []
241+
242+
search_results = bulk_search_by_purl(purls, timeout=5)
243+
if not search_results:
244+
return []
245+
246+
return [
247+
entry.get("purl")
248+
for entry in search_results
249+
if entry.get("affected_by_vulnerabilities")
250+
]
251+
252+
253+
def get_vulnerable_cpes(components):
254+
"""
255+
Returns a list of vulnerable CPEs found in the VulnerableCodeDB for the given list
256+
of `components`.
257+
"""
258+
cpes = [component.cpe for component in components if component.cpe]
259+
260+
if not cpes:
261+
return []
262+
263+
search_results = bulk_search_by_cpes(cpes, timeout=5)
264+
if not search_results:
265+
return []
266+
267+
vulnerable_cpes = [
268+
reference.get("reference_id")
269+
for entry in search_results
270+
for reference in entry.get("references")
271+
if reference.get("reference_id").startswith("cpe")
272+
]
273+
274+
return list(set(vulnerable_cpes))

scanpipe/templates/scanpipe/tabset/tab_default.html

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
{{ field_data.label }}
55
</dt>
66
<dd class="mb-4">
7-
<pre class="break-all is-small">{{ field_data.value|default:''|default_if_none:''|urlize }}</pre>
7+
<pre class="break-all wrap is-small">{{ field_data.value|default:''|default_if_none:''|urlize }}</pre>
88
</dd>
99
{% endfor %}
1010
</dl>

setup.cfg

+1
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ console_scripts =
104104
scanpipe = scancodeio:command_line
105105

106106
scancodeio_pipelines =
107+
check_vulnerabilities = scanpipe.pipelines.check_vulnerabilities:CheckVulnerabilities
107108
docker = scanpipe.pipelines.docker:Docker
108109
docker_windows = scanpipe.pipelines.docker_windows:DockerWindows
109110
load_inventory = scanpipe.pipelines.load_inventory:LoadInventory

0 commit comments

Comments
 (0)