From 6497e90014ab4a5f8370005b827929a38cb180ed Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Mon, 7 Aug 2023 19:04:39 -0700 Subject: [PATCH] Update affected-fixed package matching #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/models.py | 663 ++++++++++++++++-- .../templates/package_details.html | 140 +++- vulnerabilities/tests/test_models.py | 65 ++ vulnerabilities/views.py | 5 +- vulnerablecode/settings.py | 4 +- vulnerablecode/static/css/custom.css | 102 ++- 6 files changed, 929 insertions(+), 50 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index dde000813..d07a0530f 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -635,100 +635,685 @@ def get_fixed_packages(self, package): packagerelatedvulnerability__fix=True, ).distinct() - def assign_univers_version(self, fixing_pkg): + def get_sibling_packages(self, package): """ - Identify which univers version applies to the two packages to be compared (self and a fixing package), - evaluate whether the fixing_pkg version is > than the target affected package, and + Return a queryset of all packages with the same type, namespace, name, subpath and qualifiers of the `package`, whether or not they fix any vulnerability + """ + return Package.objects.filter( + name=package.name, + namespace=package.namespace, + type=package.type, + qualifiers=package.qualifiers, + subpath=package.subpath, + # packagerelatedvulnerability__fix=True, + ).distinct() + + # def assign_and_compare_univers_versions(self, fixed_pkg): + def assign_univers_version(self, fixed_pkg): + """ + Identify which univers version applies to the two packages to be compared (self and a fixed package), + evaluate whether the fixed_pkg version is > than the target affected package, and return True or False. """ - # TODO: We also need to find which fixing_pkg is closest to the affected version -- we don't do that yet. + # TODO: Instead of return True or False based on evaluating the incoming fixed_pkg type to a univers version and then checking whether the fixed version is greater than the affected (self) version, we'll just use the incoming type to assign and return a univers version -- as command_name -- to be used in get_closest_fixed_package() below to add all greater-than versions to the later_matching_fixed_packages list -- which in turn will be fed to self.sort_by_version(later_matching_fixed_packages) # Many more to be added. match_type_to_univers_version = { "conan": versions.ConanVersion, + "deb": versions.DebianVersion, "maven": versions.MavenVersion, + "openssl": versions.OpensslVersion, + "pypi": versions.PypiVersion, } command_name = "" - type_to_version = match_type_to_univers_version.get(fixing_pkg.type) - if type_to_version: + matched_type_to_version = match_type_to_univers_version.get(fixed_pkg.type) + if matched_type_to_version: print("\t--------------------------") - print("\ttype_to_version = {}\n".format(type_to_version)) - command_name = type_to_version + print("*** matched_type_to_version = {}".format(matched_type_to_version)) + command_name = matched_type_to_version else: - print("\ttype_to_version = NO MATCH\n") - command_name = versions.Version + print("\t--------------------------") + print("*** matched_type_to_version = NO MATCH") + # Using "command_name = versions.Version", the test + # assert versions.Version("0.9") < versions.Version("0.10") + # fails! + # command_name = versions.Version + # Use this as a default fallback instead. + command_name = versions.SemverVersion + + # if command_name(fixed_pkg.version) > command_name(self.version): + # return True + # else: + # return False + + # Instead return command_name for recipient to use as needed for sorting or perhaps other uses + return command_name + + def sort_by_version(self, later_matching_fixed_packages): + # Incoming is a list of + + # ALERT: added this to address server error 500 but related error arose: line 908, in get_closest_fixed_package + # HOT: How is this related to the source of the server error (500)? + if len(later_matching_fixed_packages) == 0: + return + + # Replace find_closest_fixed_by_package()? + print("\nlater_matching_fixed_packages = {}".format(later_matching_fixed_packages)) + print("\nlater_matching_fixed_packages[0] = {}".format(later_matching_fixed_packages[0])) + print( + "\ntype(later_matching_fixed_packages[0]) = {}".format( + type(later_matching_fixed_packages[0]) + ) + ) + # NOTE: This gives us the PURL type but instead we want the PURL itself to pass to assign_univers_version(self, fixed_pkg) and get the command_name in return, which we'll then use in the sort process. + print( + "\nlater_matching_fixed_packages[0].type = {}".format( + later_matching_fixed_packages[0].type + ) + ) + print( + "\ntype(later_matching_fixed_packages[0].type) = {}".format( + type(later_matching_fixed_packages[0].type) + ) + ) - if command_name(fixing_pkg.version) > command_name(self.version): - return True - else: - return False + # Incoming is a list -- later_matching_fixed_packages + + # We'll use assign_univers_version() above to get the univers version as a command_name. + # But what do we pass to it? The [0] index of the incoming list, i.e., later_matching_fixed_packages[0]? + command_name = self.assign_univers_version(later_matching_fixed_packages[0]) + + print("\n>>> command_name = {}\n".format(command_name)) + + # TODO: Maybe we don't need to convert to a PURL, a list of dictionaries etc.?? + print( + "\n+++++++ later_matching_fixed_packages[0].version = {}".format( + later_matching_fixed_packages[0].version + ) + ) + + # sort + test_sort_by_version = [] + test_sort_by_version = sorted( + # later_matching_fixed_packages, key=lambda x: versions.DebianVersion(x["version"]) + later_matching_fixed_packages, + # key=lambda x: versions.MavenVersion(x.version), + key=lambda x: command_name(x.version), + ) + + print("\ntest_sort_by_version = {}\n".format(test_sort_by_version)) + + return test_sort_by_version + + # convert_to_dict_list = [] + + # sorted_later_matching_fixed_packages = [] + + # # TODO: First, convert to a list of dictionaries. + # for pkg in later_matching_fixed_packages: + # # pkg is a + # print("pkg = {}".format(pkg)) + # print("type(pkg) = {}".format(type(pkg))) + + # # pkg_str is a string + # pkg_str = pkg.package_url + # print("pkg_str = {}".format(pkg_str)) + # print("type(pkg_str) = {}".format(type(pkg_str))) + + # # purl is a + # purl = PackageURL.from_string(pkg_str) + # print("purl = {}".format(purl)) + # print("type(purl) = {}".format(type(purl))) + + # purl_dict = purl.to_dict() + # print("purl_dict = {}".format(purl_dict)) + # print("type(purl_dict) = {}".format(type(purl_dict))) + + # convert_to_dict_list.append(purl.to_dict()) + # print("HELLO\n") + + # print("\nconvert_to_dict_list = {}\n".format(convert_to_dict_list)) + + # return convert_to_dict_list + # ========================================================== + # sorted_later_matching_fixed_packages = sorted( + # later_matching_fixed_packages, key=lambda x: versions.MavenVersion(x["version"]) + # ) + # print( + # "\nsorted_later_matching_fixed_packages = {}\n".format( + # sorted_later_matching_fixed_packages + # ) + # ) + # print("\n".join(map(str, sorted_later_matching_fixed_packages))) + + # return what? + + # ========================================================== + # ========================================================== + + # def find_closest_fixed_by_package(self, later_matching_fixed_packages): + # # Maybe use sort_by_version() above instead? + # # take the incoming list later_matching_fixed_packages, convert to list of dictionaries, sort by version using univers.version.[version class], choose the top i.e., index [0] and convert back to PURL and return that PURL. + # print("\nlater_matching_fixed_packages = {}\n".format(later_matching_fixed_packages)) + + # closest_fixed_by_package = "TBD" + + # return closest_fixed_by_package @property - def get_fixing_packages(self): + # def get_fixing_packages(self): + def get_closest_fixed_package(self): """ - This function identifies the closest fixing version that is greater than the affected version and + This function identifies the closest fixed package version that is greater than the affected package version and is the same type, namespace, name, qualifiers and subpath as the affected package. """ print("\nself = {}\n".format(self)) - # This returns all fixing packages that match the target package (type etc.), regardless of fixed vuln. - fixed_packages = self.get_fixed_packages(package=self) + # This returns all fixed packages that match the target package (type etc.), regardless of fixed vuln. + # fixed_packages = self.get_fixed_packages(package=self) + # This is clearer. + matching_fixed_packages = self.get_fixed_packages(package=self) # This returns a list of the vulnerabilities that affect this package (i.e., self). qs = self.vulnerabilities.filter(packagerelatedvulnerability__fix=False) - # This takes the list of vulns affecting the current package, retrieves a list of the fixing packages for each vuln, and assigns the result to a custom attribute, `filtered_fixed_packages`. - # We use this in a for loop below like this -- qs[vuln_count].filtered_fixed_packages -- where `vuln_count` is used to iterate through the list of vulns that affect the current package (i.e., self). + # This takes the list of vulns affecting the current package, retrieves a list of the fixed packages for each vuln, and assigns the result to a custom attribute, `filtered_fixed_packages` (renamed 'matching_fixed_packages'). + # We use this in a for loop below like this -- qs[vuln_count].filtered_fixed_packages (renamed 'matching_fixed_packages') -- where `vuln_count` is used to iterate through the list of vulns that affect the current package (i.e., self). qs = qs.prefetch_related( Prefetch( "packages", - queryset=fixed_packages, - to_attr="filtered_fixed_packages", + # queryset=fixed_packages, + queryset=matching_fixed_packages, + # to_attr="filtered_fixed_packages", + to_attr="matching_fixed_packages", ) ) # Ex: qs[0].filtered_fixed_packages gives us the fixed package(s) for the 1st vuln for this affected package (i.e., self). print("qs = {}\n".format(qs)) - prefetch_fixed_packages = [] + # ************************************************************************ + + later_matching_fixed_packages = [] vuln_count = 0 for vuln in qs: print("vuln = {}\n".format(vuln)) + # print( + # "\tqs[vuln_count].filtered_fixed_packages = {}".format( + # qs[vuln_count].filtered_fixed_packages + # ) + # ) print( - "\tqs[vuln_count].filtered_fixed_packages = {}".format( - qs[vuln_count].filtered_fixed_packages + "\tqs[vuln_count].matching_fixed_packages = {}".format( + qs[vuln_count].matching_fixed_packages ) ) print("") # Check the Prefetch qs. - # TODO: Do we want to check whether the fixing version has any vulnerabilities of its own? - for fixing_pkg in qs[vuln_count].filtered_fixed_packages: - print("\tfixing_pkg = {}".format(fixing_pkg)) - print("\tfixing_pkg.type = {}".format(fixing_pkg.type)) - print("\tfixing_pkg.version = {}".format(fixing_pkg.version)) + # TODO: Do we want to check whether the fixed version has any vulnerabilities of its own? + # for fixed_pkg in qs[vuln_count].filtered_fixed_packages: + for fixed_pkg in qs[vuln_count].matching_fixed_packages: + print("\tfixed_pkg = {}".format(fixed_pkg)) + print("\tfixed_pkg.type = {}".format(fixed_pkg.type)) + print("\tfixed_pkg.version = {}".format(fixed_pkg.version)) print("\t--------------------------") print("\tself.type = {}".format(self.type)) print("\tself.version = {}".format(self.version)) - # Assign univers version and compare: False = fixing_pkg.version < self.version (affected version). - # TODO: We also need to find which fixing_pkg is closest to the affected version -- we don't do that yet. - immediate_fix = self.assign_univers_version(fixing_pkg) - print("\t--------------------------") - print("\timmediate_fix = {}\n".format(immediate_fix)) - - if fixing_pkg in fixed_packages and immediate_fix: - prefetch_fixed_packages.append(fixing_pkg) + # Assign univers version and compare: False = fixed_pkg.version < self.version (affected version). + + # 2023-08-02 Wednesday 16:01:35. atm immediate_fix is True or False. If instead assign_and_compare_univers_versions() returns the univers version, we could get that here and then test with this or similar right here -- enabling use of the univers version function in other places as well, like a sort_by_version function! + # if command_name(fixed_pkg.version) > command_name(self.version): + # return True + # else: + # return False + # ===================================================== + # Replace this with chunk below + # immediate_fix = self.assign_and_compare_univers_versions(fixed_pkg) + # print("\t--------------------------") + # print("\timmediate_fix = {}\n".format(immediate_fix)) + + # if fixed_pkg in fixed_packages and immediate_fix: + # later_matching_fixed_packages.append(fixed_pkg) + # ===================================================== + # command_name = self.assign_and_compare_univers_versions(fixed_pkg) + # renamed + # TODO: Move this up before the for loop -- both for loops if possible -- to reduce calls! + command_name = self.assign_univers_version(fixed_pkg) + print("\nJust requested command_name >>> {}\n".format(command_name)) + # if fixed_pkg in fixed_packages and command_name(fixed_pkg.version) > command_name( + # self.version + # ): + if fixed_pkg in matching_fixed_packages and command_name( + fixed_pkg.version + ) > command_name(self.version): + later_matching_fixed_packages.append(fixed_pkg) vuln_count += 1 - return prefetch_fixed_packages + # find_closest_fixed_by_package -- from the list later_matching_fixed_packages + # closest_fixed_by_package = self.find_closest_fixed_by_package(later_matching_fixed_packages) + + # TODO: or instead use this. This will be a list sorted by univers version class, and here all we need is to grab the [0] index from that list for the closest fixed by package! So we'd return a single closest_fixed_package. + # ALERT: The sort query needs to be done separately for each vulnerability because the list of fixed by packages is likely to be different. As is, we return a single sorted list of all fixed by packages for the affected package and then pass just the [0] package -- not what we want to do! + sort_fixed_by_packages_by_version = self.sort_by_version(later_matching_fixed_packages) + print( + "\nsort_fixed_by_packages_by_version = {}\n".format(sort_fixed_by_packages_by_version) + ) + # ALERT: 2023-08-05 Saturday 23:22:08. Address server error 500? + # ALERT: 2023-08-05 Saturday 23:24:50. This actusally fixed the server error (500) and I can now even see the Packafe details page for pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1 !!! + # HOT: I need to trace back the root cause of the server error (500). I suspect it's something like a record with no fixed by packages or something else that is an empty list but which i try to measure, e.g., for a print statement, or possibly for a real if condition.. + if sort_fixed_by_packages_by_version is None: + return + + closest_fixed_package = sort_fixed_by_packages_by_version[0] + + # print("\n!!! later_matching_fixed_packages = {}\n".format(later_matching_fixed_packages)) + # print( + # "\n!!! sort_fixed_by_packages_by_version = {}\n".format( + # sort_fixed_by_packages_by_version + # ) + # ) + # print("\n!!! closest_fixed_package = {}\n".format(closest_fixed_package)) + + # rebuilt_purl_from_dict = PackageURL( + # closest_fixed_package["type"], + # closest_fixed_package["namespace"], + # closest_fixed_package["name"], + # closest_fixed_package["version"], + # closest_fixed_package["qualifiers"], + # closest_fixed_package["subpath"], + # ) + # print("\n!!! rebuilt_purl_from_dict = {}\n".format(rebuilt_purl_from_dict)) + + # return later_matching_fixed_packages + return sort_fixed_by_packages_by_version + # return [closest_fixed_package] + + # return [rebuilt_purl_from_dict] + + @property + def fixed_package_details(self): + """ + This is a test that might develop into a model-based equivalent of the loops etc. I was doing/trying to do in the Jinja2 template. I'm going to add this as a context so we can see it in the template. + """ + # return "Hello" + + # vcio_dict = { + # [ + # {"VCID-2nyb-8rwu-aaag": "PURL01"}, + # {"VCID-gqhw-ngh8-aaap": "PURL02"}, + # {"some-other-id": "PURL03"}, + # ] + # } + + print("\n==> This is from the test_property_01() property.\n") + + print("\nself = {}\n".format(self)) + + # This returns all fixed packages that match the target package (type etc.), regardless of fixed vuln. + # fixed_packages = self.get_fixed_packages(package=self) + # This is clearer. + matching_fixed_packages = self.get_fixed_packages(package=self) + + # This returns a list of the vulnerabilities that affect this package (i.e., self). + qs = self.vulnerabilities.filter(packagerelatedvulnerability__fix=False) + + # TODO: Can we get all sibling packages so that we can then determine which have 0 vulnerabilities and of these the closest and maybe the most recent as well? + + all_sibling_packages = self.get_sibling_packages(package=self) + print("\nall_sibling_packages = {}\n".format(all_sibling_packages)) + print("\nlen(all_sibling_packages) = {}\n".format(len(all_sibling_packages))) + + non_vuln_sibs = [] + for sib in all_sibling_packages: + if sib.is_vulnerable is False: + non_vuln_sibs.append(sib) + print("\nnon_vuln_sibs = {}\n".format(non_vuln_sibs)) + print("\nlen(non_vuln_sibs) = {}\n".format(len(non_vuln_sibs))) + + # Add just the greater-than versions to a new list + command_name = self.assign_univers_version(self) + print( + "\nOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOO command_name = {}\n".format( + command_name + ) + ) + later_non_vuln_sibs = [] + for non_vuln_sib in non_vuln_sibs: + if command_name(non_vuln_sib.version) > command_name(self.version): + later_non_vuln_sibs.append(non_vuln_sib) + + print("\nlater_non_vuln_sibs = {}\n".format(later_non_vuln_sibs)) + print("\nlen(later_non_vuln_sibs) = {}\n".format(len(later_non_vuln_sibs))) + + # This takes the list of vulns affecting the current package, retrieves a list of the fixed packages for each vuln, and assigns the result to a custom attribute, `filtered_fixed_packages` (renamed 'matching_fixed_packages'). + # We use this in a for loop below like this -- qs[vuln_count].filtered_fixed_packages (renamed 'matching_fixed_packages') -- where `vuln_count` is used to iterate through the list of vulns that affect the current package (i.e., self). + qs = qs.prefetch_related( + Prefetch( + "packages", + # queryset=fixed_packages, + queryset=matching_fixed_packages, + # to_attr="filtered_fixed_packages", + to_attr="matching_fixed_packages", + ) + ) + + # Ex: qs[0].filtered_fixed_packages gives us the fixed package(s) for the 1st vuln for this affected package (i.e., self). + print("\nzzz qs = {}\n".format(qs)) + + purl_dict = {} + + purl_dict["purl"] = self.purl + + purl_dict.update({"vulnerabilities": []}) + + # purl_dict["vulnerabilities"].append({"fruit": "orange"}) + + for vuln in qs: + print("\nzzz vuln = {}\n".format(vuln)) + print("\nzzz type(vuln) = {}\n".format(type(vuln))) + + later_matching_fixed_packages = [] + # purl_dict[vuln.vulnerability_id] = "aaa" + # purl_dict.update({"vulnerability": vuln.vulnerability_id}) + + purl_dict["vulnerabilities"].append({"vulnerability": vuln.vulnerability_id}) + + # TODO:2023-08-05 Saturday 13:12:28. This returns a list of matching fixed packages for this specific vuln! + vuln_matching_fixed_packages = vuln.matching_fixed_packages + print("\nzzz self.purl = {}\n".format(self.purl)) + print("\nzzz vuln = {}\n".format(vuln)) + print("\nzzz vuln_matching_fixed_packages = {}\n".format(vuln_matching_fixed_packages)) + + # TODO: So we need to sort this list by version using the correct univers version and then return the [0] index in that sorted list + # QUESTION: Do we still need to remove lesser-than fixed packages or did we already do that? + + # ============================================================= + # command_name = self.assign_univers_version(fixed_pkg) + command_name = self.assign_univers_version(self) + print("\nzzz command_name = {}\n".format(command_name)) + + # ALERT: What if there are no fixed by packages? The following thows an error because the list 'vuln_matching_fixed_packages' is empty! + # [I fixed this, right? ;-] + + closest_fixed_package = "" + + if len(vuln_matching_fixed_packages) > 0: + + for fixed_pkg in vuln_matching_fixed_packages: + if fixed_pkg in matching_fixed_packages and command_name( + fixed_pkg.version + ) > command_name(self.version): + later_matching_fixed_packages.append(fixed_pkg) + + # print("\nJust requested command_name >>> {}\n".format(command_name)) + # # if fixed_pkg in fixed_packages and command_name(fixed_pkg.version) > command_name( + # # self.version + # # ): + # if fixed_pkg in matching_fixed_packages and command_name( + # fixed_pkg.version + # ) > command_name(self.version): + # later_matching_fixed_packages.append(fixed_pkg) + # ============================================================= + # later_matching_fixed_packages = vuln.matching_fixed_packages + + print( + "\nzzz later_matching_fixed_packages = {}\n".format( + later_matching_fixed_packages + ) + ) + + sort_fixed_by_packages_by_version = self.sort_by_version( + later_matching_fixed_packages + ) + print( + "\nzzz sort_fixed_by_packages_by_version = {}\n".format( + sort_fixed_by_packages_by_version + ) + ) + closest_fixed_package = sort_fixed_by_packages_by_version[0] + # closest_fixed_package = sort_fixed_by_packages_by_version[0].purl + # 2023-08-06 Sunday 11:15:03. This returns a queryset of vulns affecting this package. + # HOT: How do we get the closest fixed by package vuln count and list of vulns? I keep getting errors. ;-) + closest_fixed_package_vulns = closest_fixed_package.affected_by + # closest_fixed_package_vulns_list = list(closest_fixed_package_vulns) + # ALERT: 2023-08-06 Sunday 14:25:49. This did the trick! + + # closest_fixed_package_vulns_list = [ + # i.vulnerability_id for i in closest_fixed_package_vulns + # ] + + # 2023-08-06 Sunday 16:53:27. Try a named tuple to pass the vuln's vulnerability+id and get_absolute_url. + # FixedPackageVuln = namedtuple("FixedPackageVuln", "vuln_id, vuln_get_absolute_url") + # closest_fixed_package_vulns_list = [ + # FixedPackageVuln( + # vuln_id=fixed_pkg_vuln.vulnerability_id, + # vuln_get_absolute_url=fixed_pkg_vuln.get_absolute_url(), + # ) + # for fixed_pkg_vuln in closest_fixed_package_vulns + # ] + # ALERT: Replace the namedtuple with a dict -- this way it can be added to the purl_dict as a nested dict rather than a list of 2 values. + closest_fixed_package_vulns_dict = [ + { + "vuln_id": fixed_pkg_vuln.vulnerability_id, + "vuln_get_absolute_url": fixed_pkg_vuln.get_absolute_url(), + } + for fixed_pkg_vuln in closest_fixed_package_vulns + ] + + # === + # closest_fixed_package_vulns_list = closest_fixed_package_vulns.objects.values_list() + + # closest_fixed_package_vulns_list = [] + # for closest_vuln in closest_fixed_package_vulns: + # closest_fixed_package_vulns_list.append(closest_vuln) + # print( + # "\t\nQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQ vuln = {}\n".format( + # closest_vuln + # ) + # ) + # print("\t\ntype(closest_vuln) = {}".format(type(closest_vuln))) + + # # closest_fixed_package_vuln_count = len(closest_fixed_package.affected_by) + # print( + # "\t\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type(closest_fixed_package_vulns) = {}\n".format( + # type(closest_fixed_package_vulns) + # ) + # ) + + else: + closest_fixed_package = "There are no reported fixed packages." + # Is None the value we want? We do not want to display anything but the count = 0. + # closest_fixed_package_vulns = None + # closest_fixed_package_vuln_count = 0 + + # closest_fixed_package_vulns_list = [] + + print("\nzzz closest_fixed_package = {}".format(closest_fixed_package)) + print("zzz type(closest_fixed_package) = {}\n".format(type(closest_fixed_package))) + + # TODO: How do we add 'closest_fixed_by_purl', 'closest_fixed_by_vulns' and 'non_vulnerable_fix'? + + # # for vuln in purl_dict["vulnerabilities"]: + # # # vuln["closest_fixed_by_purl"] = "?????" + # # vuln["closest_fixed_by_purl"] = closest_fixed_package + # # vuln["closest_fixed_by_url"] = "?????" + # # vuln["closest_fixed_by_vulnerabilities"] = "?????" + # # vuln["non_vulnerable_fix"] = "?????" + # # vuln["non_vulnerable_fix_url"] = "?????" + + for dict_vuln in purl_dict["vulnerabilities"]: + print("\n===================================> vuln = {}\n".format(vuln)) + print("\n===================================> type(vuln) = {}\n".format(type(vuln))) + print("\n===================================> vuln.vcid = {}\n".format(vuln.vcid)) + print( + "\n===================================> dict_vuln['vulnerability'] = {}\n".format( + dict_vuln["vulnerability"] + ) + ) + # TODO: Up above we defined 'non_vuln_sibs' but we still need to remove those with less than version + # ALERT: remove less than versions from 'non_vuln_sibs' + # 2023-08-05 Saturday 20:30:47. Hopefully just wrote the code for that up above, with the new list 'later_non_vuln_sibs'. + + closest_non_vulnerable_fix = "" + # if len(non_vuln_sibs) > 0: + # closest_non_vulnerable_fix = self.sort_by_version(non_vuln_sibs)[0] + if len(later_non_vuln_sibs) > 0: + closest_non_vulnerable_fix = self.sort_by_version(later_non_vuln_sibs)[0] + # else: + # # closest_non_vulnerable_fix = ( + # # "There are no reported non-vulnerable fixed packages." + # # ) + # closest_non_vulnerable_fix = None + + most_recent_non_vulnerable_fix = "" + if len(later_non_vuln_sibs) > 0: + most_recent_non_vulnerable_fix = self.sort_by_version(later_non_vuln_sibs)[-1] + else: + # most_recent_non_vulnerable_fix = ( + # "There are no reported non-vulnerable fixed packages." + # ) + most_recent_non_vulnerable_fix = None + + # if dict_vuln["vulnerability"] == vuln.vulnerability_id: + if dict_vuln["vulnerability"] == str(vuln): + # if dict_vuln["vulnerability"] == vuln.vcid: + # dict_vuln["closest_fixed_by_purl"] = "?????" + dict_vuln["closest_fixed_by_purl"] = str(closest_fixed_package) + dict_vuln["closest_fixed_by_url"] = closest_fixed_package.get_absolute_url() + # dict_vuln["closest_fixed_by_vulnerabilities"] = closest_fixed_package_vuln_count + # dict_vuln["closest_fixed_by_vulnerabilities"] = closest_fixed_package_vulns + # dict_vuln["closest_fixed_by_vulnerabilities"] = ["A", "B"] + + # dict_vuln["closest_fixed_by_vulnerabilities"] = closest_fixed_package_vulns_list + # ALERT: Replace the above list created with a namedtuple with the following dictionary: + dict_vuln["closest_fixed_by_vulnerabilities"] = closest_fixed_package_vulns_dict + # ALERT: Moved these up 1 level in the dict. + # dict_vuln["closest_non_vulnerable_fix"] = str(closest_non_vulnerable_fix) + # dict_vuln[ + # "closest_non_vulnerable_fix_url" + # ] = closest_non_vulnerable_fix.get_absolute_url() + # dict_vuln["most_recent_non_vulnerable_fix"] = str( + # most_recent_non_vulnerable_fix + # ) + # dict_vuln[ + # "most_recent_non_vulnerable_fix_url" + # ] = most_recent_non_vulnerable_fix.get_absolute_url() + + # QUESTION: Can we add the non-vuln data as higher-level key-value pairs rather than children of "vulnerabilities"? + + # purl_dict.update({"fruits": []}) + # purl_dict["fruits"].append({"fruit": "apple"}) + # purl_dict["fruits"].append({"fruit": "banana"}) + + # purl_dict.update( + # {"closest_non_vulnerable_fix": str(closest_non_vulnerable_fix)} + # ) + # purl_dict.update( + # { + # "closest_non_vulnerable_fix_url": closest_non_vulnerable_fix.get_absolute_url() + # } + # ) + # purl_dict.update( + # {"most_recent_non_vulnerable_fix": str(most_recent_non_vulnerable_fix)} + # ) + # purl_dict.update( + # { + # "most_recent_non_vulnerable_fix_url": most_recent_non_vulnerable_fix.get_absolute_url() + # } + # ) + + purl_dict["closest_non_vulnerable_fix"] = str(closest_non_vulnerable_fix) + purl_dict[ + "closest_non_vulnerable_fix_url" + ] = closest_non_vulnerable_fix.get_absolute_url() + purl_dict["most_recent_non_vulnerable_fix"] = str( + most_recent_non_vulnerable_fix + ) + purl_dict[ + "most_recent_non_vulnerable_fix_url" + ] = most_recent_non_vulnerable_fix.get_absolute_url() + + print("\npurl_dict = {}\n".format(purl_dict)) + + print(json.dumps(purl_dict, indent=4, sort_keys=False)) + + # # Print to text file + pretty_purl_dict = json.dumps(purl_dict, indent=4, sort_keys=False) + # logger = logging.getLogger(__name__) + # logger.setLevel(logging.INFO) + # # logger.addHandler(logging.FileHandler("2023-08-07-pretty_purl_dict.txt")) + # # will this overwrite prior writes? weird output + # logger.addHandler(logging.FileHandler("2023-08-07-pretty_purl_dict.txt", mode="w")) + # logger.info(pretty_purl_dict) + + with open("/home/jmh/pretty_purl_dict.txt", "w") as f: + f.write(pretty_purl_dict) + + alternate_dict_01 = { + "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", + "vulnerabilities": [ + { + "vulnerability": "VCID-2nyb-8rwu-aaag", + "closest_fixed_by_purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", + # "get_absolute_url": reverse("package_details", args=[self.purl]), + "closest_fixed_by_url": reverse( + "package_details", + args=["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2"], + ), + "closest_fixed_by_vulns": 2, + "non_vulnerable_fix": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", + "non_vulnerable_fix_url": reverse( + "package_details", + args=["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1"], + ), + }, + { + "vulnerability": "VCID-gqhw-ngh8-aaap", + "closest_fixed_by_purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.4", + # "get_absolute_url": reverse("package_details", args=[self.purl]), + "closest_fixed_by_url": reverse( + "package_details", + args=["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.4"], + ), + "closest_fixed_by_vulns": 1, + "non_vulnerable_fix": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", + "non_vulnerable_fix_url": reverse( + "package_details", + args=["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1"], + ), + }, + { + "vulnerability": "VCID-t7e4-g3fr-aaan", + "closest_fixed_by_purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", + # "get_absolute_url": reverse("package_details", args=[self.purl]), + "closest_fixed_by_url": reverse( + "package_details", + args=["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1"], + ), + "closest_fixed_by_vulns": 0, + "non_vulnerable_fix": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", + "non_vulnerable_fix_url": reverse( + "package_details", + args=["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1"], + ), + }, + ], + } + + # return vcio_dict + + # return alternate_dict_01 + + return purl_dict class PackageRelatedVulnerability(models.Model): diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index cfbfdd706..8d204bccf 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -41,7 +41,6 @@
-
During testing, red = all packages that fix the vulnerability (for pkg in vulnerability.fixed_by_packages) and green = the closest fixing package whose version is greater than the affected package's version (if pkg in get_fixing_packages).
Affected by vulnerabilities ({{ affected_by_vulnerabilities|length }})
@@ -52,7 +51,7 @@ Vulnerability Summary Aliases - Fixing Packages + Fixed by packages @@ -76,22 +75,147 @@ {% endif %} {% endfor %} - + + + + + + + + {% if package.purl in fixed_package_details.purl %} + {% for key, value in fixed_package_details.items %} + {% if key == "vulnerabilities" %} + {% for abc in value %} + {% if abc.vulnerability == vulnerability.vulnerability_id %} +
    +
  • + PURL: {{ fixed_package_details.purl }} + + + +
    • +
  • + Vulnerability: {{ abc.vulnerability }} +
    • +
  • + Closest fixed-by PURL: + {{ abc.closest_fixed_by_purl }} +
    • +
  • + Closest fixed-by vulnerability count: {{ abc.closest_fixed_by_vulnerabilities|length }} + {% if abc.closest_fixed_by_vulnerabilities|length != 0 %} +
    +
    + +
    + +
    + {% endif %} +
    • +
  • + Closest non-vulnerable fix: + + + {{ fixed_package_details.closest_non_vulnerable_fix }} +
    • +
  • + Most recent non-vulnerable fix: + + + {{ fixed_package_details.most_recent_non_vulnerable_fix }} +
    • +
+ {% endif %} + {% endfor %} + + + {% endif %} + {% endfor %} + {% else %} + NO-- {{ package.purl }} + {% endif %} + + + + + +
@@ -109,7 +233,7 @@
- Fixing vulnerabilities ({{ fixing_vulnerabilities|length }}) + Fixed by vulnerabilities ({{ fixing_vulnerabilities|length }})
diff --git a/vulnerabilities/tests/test_models.py b/vulnerabilities/tests/test_models.py index 58b95af80..24a66d2d2 100644 --- a/vulnerabilities/tests/test_models.py +++ b/vulnerabilities/tests/test_models.py @@ -7,12 +7,14 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import urllib.parse from datetime import datetime from unittest import TestCase import pytest from django.db.utils import IntegrityError from freezegun import freeze_time +from univers import versions from vulnerabilities import models @@ -88,3 +90,66 @@ def test_vulnerability_package(self): assert v1.vulnerable_packages.all()[0] == p1 assert v1.patched_packages.all()[0] == p2 + + +@pytest.mark.django_db +class TestPackageModel(TestCase): + def test_univers_version_comparisons(self): + assert versions.PypiVersion("1.2.3") < versions.PypiVersion("1.2.4") + + assert versions.PypiVersion("0.9") < versions.PypiVersion("0.10") + + # pkg:deb/debian/jackson-databind@2.12.1-1%2Bdeb11u1 is a real PURL in the DB + # But I get an error when I try to compare 2 PURLs with the same suffix -- + # univers.versions.InvalidVersion: '2.12.1-1%2Bdeb11u1' is not a valid + # Do we need to replace/delete the "%"? + # assert versions.DebianVersion("2.12.1-1%2Bdeb11u1") < versions.DebianVersion( + # "2.13.1-1%2Bdeb11u1" + # ) + # Test the error + with pytest.raises(versions.InvalidVersion): + assert versions.DebianVersion("2.12.1-1%2Bdeb11u1") < versions.DebianVersion( + "2.13.1-1%2Bdeb11u1" + ) + # Decode the version and test. + assert versions.DebianVersion( + urllib.parse.unquote("2.12.1-1%2Bdeb11u1") + ) < versions.DebianVersion(urllib.parse.unquote("2.13.1-1%2Bdeb11u1")) + + with pytest.raises(TypeError): + assert versions.PypiVersion("0.9") < versions.DebianVersion("0.10") + + # Using versions.Version does not correctly make this comparison! + assert not versions.Version("0.9") < versions.Version("0.10") + # Use SemverVersion instead as a default fallback version for comparisons. + assert versions.SemverVersion("0.9") < versions.SemverVersion("0.10") + + def test_assign_and_compare_univers_versions(self): + deb01 = models.Package.objects.create(type="deb", name="git", version="2.30.1") + deb02 = models.Package.objects.create(type="deb", name="git", version="2.31.1") + + immediate_fix01 = deb01.assign_and_compare_univers_versions(deb02) + print("\nimmediate_fix01 = {}\n".format(immediate_fix01)) + # assert deb01.assign_and_compare_univers_versions(deb02) is True + assert deb01.assign_and_compare_univers_versions(deb02) + + immediate_fix02 = deb02.assign_and_compare_univers_versions(deb01) + print("\nimmediate_fix02 = {}\n".format(immediate_fix02)) + # assert deb02.assign_and_compare_univers_versions(deb01) is False + assert not deb02.assign_and_compare_univers_versions(deb01) + + pypi01 = models.Package.objects.create(type="pypi", name="pyopenssl", version="0.9") + pypi02 = models.Package.objects.create(type="pypi", name="pyopenssl", version="0.10") + + immediate_fix03 = pypi01.assign_and_compare_univers_versions(pypi02) + print("\nimmediate_fix03 = {}\n".format(immediate_fix03)) + # assert pypi01.assign_and_compare_univers_versions(pypi02) is True + assert pypi01.assign_and_compare_univers_versions(pypi02) + + gem01 = models.Package.objects.create(type="gem", name="sidekiq", version="0.9") + gem02 = models.Package.objects.create(type="gem", name="sidekiq", version="0.10") + + immediate_fix04 = gem01.assign_and_compare_univers_versions(gem02) + print("\nimmediate_fix04 = {}\n".format(immediate_fix04)) + # assert gem01.assign_and_compare_univers_versions(gem02) is True + assert gem01.assign_and_compare_univers_versions(gem02) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 6a41620b3..396ac1bc2 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -83,7 +83,10 @@ def get_context_data(self, **kwargs): context["affected_by_vulnerabilities"] = package.affected_by.order_by("vulnerability_id") context["fixing_vulnerabilities"] = package.fixing.order_by("vulnerability_id") context["package_search_form"] = PackageSearchForm(self.request.GET) - context["get_fixing_packages"] = package.get_fixing_packages + # context["get_fixing_packages"] = package.get_fixing_packages + context["get_closest_fixed_package"] = package.get_closest_fixed_package + # context["test_property_01"] = package.test_property_01 + context["fixed_package_details"] = package.fixed_package_details return context diff --git a/vulnerablecode/settings.py b/vulnerablecode/settings.py index 3187b67ec..0071686fb 100644 --- a/vulnerablecode/settings.py +++ b/vulnerablecode/settings.py @@ -39,7 +39,9 @@ CSRF_TRUSTED_ORIGINS = env.list("CSRF_TRUSTED_ORIGINS", default=[]) # SECURITY WARNING: do not run with debug turned on in production -DEBUG = env.bool("VULNERABLECODE_DEBUG", default=False) +# DEBUG = env.bool("VULNERABLECODE_DEBUG", default=False) +# DEBUG = "127.0.0.1" +DEBUG = True # SECURITY WARNING: do not run with debug turned on in production DEBUG_TOOLBAR = env.bool("VULNERABLECODE_DEBUG_TOOLBAR", default=False) diff --git a/vulnerablecode/static/css/custom.css b/vulnerablecode/static/css/custom.css index 6699d8977..96ef5aad3 100644 --- a/vulnerablecode/static/css/custom.css +++ b/vulnerablecode/static/css/custom.css @@ -329,6 +329,14 @@ a.small_page_button { box-shadow: 0px 8px 16px 0px #808080; } +.dropdown-vuln-list-width { + width: 400px; +} + +.dropdown-vuln-dict-width { + max-width: 600px; +} + .width-100-pct { width: 100%; } @@ -368,4 +376,96 @@ a.small_page_button { span.tag.custom { margin: 0px 0px 6px 10px; -} \ No newline at end of file +} + +/* test bulleted list */ + +ul.fixed_by_bullet { + list-style-type: disc; + /*margin-top: 2px; +margin-bottom: 10px;*/ + /*margin-left: -24px;*/ + /*margin-left: -30px;*/ + margin-top: 0.25em; + margin-left: 7px; + margin-bottom: 0.25em; + padding-left: 10px; +} + +ul.fixed_by_bullet ul { + list-style-type: disc; + /*margin-top: 10px;*/ + margin-top: 5px; + margin-top: 0px; + margin-bottom: 0px; + margin-left: 23px; + margin-left: 18px; + padding: 0; + border: none; +} + + + +ul.fixed_by_bullet li { + margin-left: 0px; + font-family: Arial; + font-family: BlinkMacSystemFont, -apple-system, "Segoe UI", "Roboto", "Oxygen", "Ubuntu", "Cantarell", "Fira Sans", "Droid Sans", "Helvetica Neue", "Helvetica", "Arial", sans-serif; + font-size: 13px; + font-weight: normal; + /*margin-bottom: 10px;*/ + margin-bottom: 2px; +} + +ul.fixed_by_bullet li:last-child { + margin-left: 0px; + font-family: Arial; + font-family: BlinkMacSystemFont, -apple-system, "Segoe UI", "Roboto", "Oxygen", "Ubuntu", "Cantarell", "Fira Sans", "Droid Sans", "Helvetica Neue", "Helvetica", "Arial", sans-serif; + font-size: 13px; + font-weight: normal; + /*margin-bottom: 10px;*/ + margin-bottom: 0px; +} + +ul.fixed_by_bullet li li { + margin-left: 0px; + font-family: Arial; + font-family: BlinkMacSystemFont, -apple-system, "Segoe UI", "Roboto", "Oxygen", "Ubuntu", "Cantarell", "Fira Sans", "Droid Sans", "Helvetica Neue", "Helvetica", "Arial", sans-serif; + font-size: 13px; + font-weight: normal; + margin-top: 0px; + color: #000000; +} + +/* 10/10/15 add 3rd-level bullets */ +ul.fixed_by_bullet ul ul { + list-style-type: disc; + margin-top: 0px; + margin-bottom: 0px; + margin-left: 50px; + margin-left: 17px; + padding: 0; + border: none; +} + +ul.fixed_by_bullet li li li { + margin-left: 0px; + font-family: Arial; + font-family: BlinkMacSystemFont, -apple-system, "Segoe UI", "Roboto", "Oxygen", "Ubuntu", "Cantarell", "Fira Sans", "Droid Sans", "Helvetica Neue", "Helvetica", "Arial", sans-serif; + font-size: 13px; + font-weight: normal; + margin-top: 0px; + color: #000000; +} + +/* CSS for dev fixed by headers */ +.dev_fixed_by_headers { + border: solid 1px #cccccc; + border-radius: 3px; + background-color: #f2f2f2; + color: #000000; + font-weight: bold; + font-size: 13px; + padding: 3px; + margin-bottom: 3px; + display: block; +}