From 3f43f6ecd7bb475228d225683138ad8577b54a46 Mon Sep 17 00:00:00 2001 From: Rex P Date: Fri, 7 Feb 2025 15:25:32 +1100 Subject: [PATCH 1/3] fix: Base image dedup --- .../baseimagematcher/baseimagematcher.go | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/internal/clients/clientimpl/baseimagematcher/baseimagematcher.go b/internal/clients/clientimpl/baseimagematcher/baseimagematcher.go index 971be61a56..7755508637 100644 --- a/internal/clients/clientimpl/baseimagematcher/baseimagematcher.go +++ b/internal/clients/clientimpl/baseimagematcher/baseimagematcher.go @@ -12,6 +12,7 @@ import ( "slices" "time" + "github.com/google/go-cmp/cmp" "github.com/google/osv-scanner/v2/pkg/models" "github.com/google/osv-scanner/v2/pkg/reporter" "github.com/opencontainers/go-digest" @@ -35,7 +36,7 @@ type DepsDevBaseImageMatcher struct { } func (matcher *DepsDevBaseImageMatcher) MatchBaseImages(ctx context.Context, layerMetadata []models.LayerMetadata) ([][]models.BaseImageDetails, error) { - baseImagesMap := make([][]models.BaseImageDetails, len(layerMetadata)) + baseImagesToLayerMap := make([][]models.BaseImageDetails, len(layerMetadata)) g, ctx := errgroup.WithContext(ctx) g.SetLimit(maxConcurrentRequests) @@ -60,7 +61,7 @@ func (matcher *DepsDevBaseImageMatcher) MatchBaseImages(ctx context.Context, lay // If we are erroring for one base image even with retry, we probably should stop var err error - baseImagesMap[i], err = matcher.queryBaseImagesForChainID(ctx, chainID) + baseImagesToLayerMap[i], err = matcher.queryBaseImagesForChainID(ctx, chainID) return err }) @@ -70,7 +71,7 @@ func (matcher *DepsDevBaseImageMatcher) MatchBaseImages(ctx context.Context, lay return nil, err } - return buildBaseImageDetails(layerMetadata, baseImagesMap), nil + return buildBaseImageDetails(layerMetadata, baseImagesToLayerMap), nil } // makeRetryRequest will return an error on both network errors, and if the response is not 200 or 404 @@ -187,7 +188,7 @@ func (matcher *DepsDevBaseImageMatcher) queryBaseImagesForChainID(ctx context.Co return baseImagePossibilities, nil } -func buildBaseImageDetails(layerMetadata []models.LayerMetadata, baseImagesMap [][]models.BaseImageDetails) [][]models.BaseImageDetails { +func buildBaseImageDetails(layerMetadata []models.LayerMetadata, baseImagesToLayersMap [][]models.BaseImageDetails) [][]models.BaseImageDetails { allBaseImages := [][]models.BaseImageDetails{ // The base image at index 0 is a placeholder representing your image, so always empty // This is the case even if your image is a base image, in that case no layers point to index 0 @@ -195,13 +196,20 @@ func buildBaseImageDetails(layerMetadata []models.LayerMetadata, baseImagesMap [ } currentBaseImageIndex := 0 - for i, baseImages := range slices.Backward(baseImagesMap) { + for i, baseImages := range slices.Backward(baseImagesToLayersMap) { if len(baseImages) == 0 { layerMetadata[i].BaseImageIndex = currentBaseImageIndex continue } - // This layer is a base image boundary + // Is the current set of baseImages the same as the previous? + if cmp.Equal(baseImages, allBaseImages[len(allBaseImages)-1]) { + // If so, merge them + layerMetadata[i].BaseImageIndex = currentBaseImageIndex + continue + } + + // This layer is a new base image boundary allBaseImages = append(allBaseImages, baseImages) currentBaseImageIndex += 1 layerMetadata[i].BaseImageIndex = currentBaseImageIndex From 056add3a2b33bfe12476b5d054274f5e5678f512 Mon Sep 17 00:00:00 2001 From: Rex P Date: Mon, 10 Feb 2025 12:23:56 +1100 Subject: [PATCH 2/3] Apply deterministic ordering to base images --- .../clientimpl/baseimagematcher/baseimagematcher.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/internal/clients/clientimpl/baseimagematcher/baseimagematcher.go b/internal/clients/clientimpl/baseimagematcher/baseimagematcher.go index 7755508637..886223c9ec 100644 --- a/internal/clients/clientimpl/baseimagematcher/baseimagematcher.go +++ b/internal/clients/clientimpl/baseimagematcher/baseimagematcher.go @@ -10,6 +10,7 @@ import ( "math/rand/v2" "net/http" "slices" + "strings" "time" "github.com/google/go-cmp/cmp" @@ -182,7 +183,13 @@ func (matcher *DepsDevBaseImageMatcher) queryBaseImagesForChainID(ctx context.Co // TODO(v2): Temporary heuristic for what is more popular // Ideally this is done by deps.dev before release slices.SortFunc(baseImagePossibilities, func(a, b models.BaseImageDetails) int { - return len(a.Name) - len(b.Name) + lengthDiff := len(a.Name) - len(b.Name) + if lengthDiff != 0 { + return lengthDiff + } + + // Apply deterministic ordering to same length base images + return strings.Compare(a.Name, b.Name) }) return baseImagePossibilities, nil From ac97b8009acad579563c919ff9e83be8c2708c26 Mon Sep 17 00:00:00 2001 From: Rex P Date: Wed, 12 Feb 2025 09:51:10 +1100 Subject: [PATCH 3/3] Update Snaps --- cmd/osv-scanner/__snapshots__/main_test.snap | 45 +++++++++++--------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/cmd/osv-scanner/__snapshots__/main_test.snap b/cmd/osv-scanner/__snapshots__/main_test.snap index 76bd45f7b4..57600dcbf9 100755 --- a/cmd/osv-scanner/__snapshots__/main_test.snap +++ b/cmd/osv-scanner/__snapshots__/main_test.snap @@ -732,6 +732,7 @@ Filtered 9 local/unscannable package/s from the scan. | https://osv.dev/CVE-2023-6129 | 6.5 | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/CVE-2023-6237 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/CVE-2024-0727 | 5.5 | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | +| https://osv.dev/CVE-2024-12797 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/CVE-2024-13176 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/CVE-2024-2511 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/CVE-2024-4603 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | @@ -2209,6 +2210,7 @@ Loaded OSS-Fuzz local db from /osv-scanner/OSS-Fuzz/all.zip | https://osv.dev/CVE-2023-6129 | 6.5 | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/CVE-2023-6237 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/CVE-2024-0727 | 5.5 | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | +| https://osv.dev/CVE-2024-12797 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/CVE-2024-13176 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/CVE-2024-2511 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/CVE-2024-4603 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | @@ -2419,6 +2421,7 @@ Loaded OSS-Fuzz local db from /osv-scanner/OSS-Fuzz/all.zip | https://osv.dev/CVE-2023-6129 | 6.5 | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/CVE-2023-6237 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/CVE-2024-0727 | 5.5 | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | +| https://osv.dev/CVE-2024-12797 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/CVE-2024-13176 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/CVE-2024-2511 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/CVE-2024-4603 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | @@ -3014,8 +3017,8 @@ failed to load image from tarball with path "./fixtures/oci-image/no-file-here.t Scanning local image tarball "../../internal/image/fixtures/test-java-full.tar" Container Scanning Result (Alpine Linux v3.21): -Total 12 packages affected by 16 vulnerabilities (1 Critical, 5 High, 9 Medium, 0 Low, 1 Unknown) from 2 ecosystems. -15 vulnerabilities have fixes available. +Total 12 packages affected by 17 vulnerabilities (1 Critical, 5 High, 9 Medium, 0 Low, 2 Unknown) from 2 ecosystems. +16 vulnerabilities have fixes available. Maven +-----------------------------------------------------------------------------------------------------------------------------------------+ @@ -3041,7 +3044,7 @@ Alpine:v3.21 | PACKAGE | INSTALLED VERSION | FIX AVAILABLE | VULN COUNT | INTRODUCED LAYER | IN BASE IMAGE | +----------+-------------------+---------------+------------+------------------+-----------------+ | libtasn1 | 4.19.0-r2 | Fix Available | 1 | # 5 Layer | eclipse-temurin | -| openssl | 3.3.2-r4 | Fix Available | 1 | # 0 Layer | alpine | +| openssl | 3.3.2-r4 | Fix Available | 2 | # 0 Layer | alpine | +----------+-------------------+---------------+------------+------------------+-----------------+ For the most comprehensive scan results, we recommend using the HTML output: `osv-scanner scan image --serve `. @@ -3262,8 +3265,8 @@ Warning: `scan` exists as both a subcommand of OSV-Scanner and as a file on the Scanning local image tarball "../../internal/image/fixtures/test-node_modules-npm-empty.tar" Container Scanning Result (Alpine Linux v3.19): -Total 2 packages affected by 10 vulnerabilities (0 Critical, 0 High, 4 Medium, 0 Low, 6 Unknown) from 1 ecosystems. -10 vulnerabilities have fixes available. +Total 2 packages affected by 11 vulnerabilities (0 Critical, 0 High, 4 Medium, 0 Low, 7 Unknown) from 1 ecosystems. +11 vulnerabilities have fixes available. Alpine:v3.19 +---------------------------------------------------------------------------------------------+ @@ -3272,7 +3275,7 @@ Alpine:v3.19 | PACKAGE | INSTALLED VERSION | FIX AVAILABLE | VULN COUNT | INTRODUCED LAYER | IN BASE IMAGE | +---------+-------------------+---------------+------------+------------------+---------------+ | busybox | 1.36.1-r15 | Fix Available | 4 | # 0 Layer | alpine | -| openssl | 3.1.4-r5 | Fix Available | 6 | # 0 Layer | alpine | +| openssl | 3.1.4-r5 | Fix Available | 7 | # 0 Layer | alpine | +---------+-------------------+---------------+------------+------------------+---------------+ For the most comprehensive scan results, we recommend using the HTML output: `osv-scanner scan image --serve `. @@ -3289,8 +3292,8 @@ Warning: `scan` exists as both a subcommand of OSV-Scanner and as a file on the Scanning local image tarball "../../internal/image/fixtures/test-node_modules-npm-full.tar" Container Scanning Result (Alpine Linux v3.19): -Total 4 packages affected by 13 vulnerabilities (2 Critical, 0 High, 5 Medium, 0 Low, 6 Unknown) from 2 ecosystems. -12 vulnerabilities have fixes available. +Total 4 packages affected by 14 vulnerabilities (2 Critical, 0 High, 5 Medium, 0 Low, 7 Unknown) from 2 ecosystems. +13 vulnerabilities have fixes available. npm +-------------------------------------------------------------------------------------------------+ @@ -3308,7 +3311,7 @@ Alpine:v3.19 | PACKAGE | INSTALLED VERSION | FIX AVAILABLE | VULN COUNT | INTRODUCED LAYER | IN BASE IMAGE | +---------+-------------------+---------------+------------+------------------+---------------+ | busybox | 1.36.1-r15 | Fix Available | 4 | # 0 Layer | alpine | -| openssl | 3.1.4-r5 | Fix Available | 6 | # 0 Layer | alpine | +| openssl | 3.1.4-r5 | Fix Available | 7 | # 0 Layer | alpine | +---------+-------------------+---------------+------------+------------------+---------------+ For the most comprehensive scan results, we recommend using the HTML output: `osv-scanner scan image --serve `. @@ -3325,8 +3328,8 @@ Warning: `scan` exists as both a subcommand of OSV-Scanner and as a file on the Scanning local image tarball "../../internal/image/fixtures/test-node_modules-pnpm-empty.tar" Container Scanning Result (Alpine Linux v3.19): -Total 2 packages affected by 10 vulnerabilities (0 Critical, 0 High, 4 Medium, 0 Low, 6 Unknown) from 1 ecosystems. -10 vulnerabilities have fixes available. +Total 2 packages affected by 11 vulnerabilities (0 Critical, 0 High, 4 Medium, 0 Low, 7 Unknown) from 1 ecosystems. +11 vulnerabilities have fixes available. Alpine:v3.19 +---------------------------------------------------------------------------------------------+ @@ -3335,7 +3338,7 @@ Alpine:v3.19 | PACKAGE | INSTALLED VERSION | FIX AVAILABLE | VULN COUNT | INTRODUCED LAYER | IN BASE IMAGE | +---------+-------------------+---------------+------------+------------------+---------------+ | busybox | 1.36.1-r15 | Fix Available | 4 | # 0 Layer | alpine | -| openssl | 3.1.4-r5 | Fix Available | 6 | # 0 Layer | alpine | +| openssl | 3.1.4-r5 | Fix Available | 7 | # 0 Layer | alpine | +---------+-------------------+---------------+------------+------------------+---------------+ For the most comprehensive scan results, we recommend using the HTML output: `osv-scanner scan image --serve `. @@ -3352,8 +3355,8 @@ Warning: `scan` exists as both a subcommand of OSV-Scanner and as a file on the Scanning local image tarball "../../internal/image/fixtures/test-node_modules-pnpm-full.tar" Container Scanning Result (Alpine Linux v3.19): -Total 2 packages affected by 10 vulnerabilities (0 Critical, 0 High, 4 Medium, 0 Low, 6 Unknown) from 1 ecosystems. -10 vulnerabilities have fixes available. +Total 2 packages affected by 11 vulnerabilities (0 Critical, 0 High, 4 Medium, 0 Low, 7 Unknown) from 1 ecosystems. +11 vulnerabilities have fixes available. Alpine:v3.19 +---------------------------------------------------------------------------------------------+ @@ -3362,7 +3365,7 @@ Alpine:v3.19 | PACKAGE | INSTALLED VERSION | FIX AVAILABLE | VULN COUNT | INTRODUCED LAYER | IN BASE IMAGE | +---------+-------------------+---------------+------------+------------------+---------------+ | busybox | 1.36.1-r15 | Fix Available | 4 | # 0 Layer | alpine | -| openssl | 3.1.4-r5 | Fix Available | 6 | # 0 Layer | alpine | +| openssl | 3.1.4-r5 | Fix Available | 7 | # 0 Layer | alpine | +---------+-------------------+---------------+------------+------------------+---------------+ For the most comprehensive scan results, we recommend using the HTML output: `osv-scanner scan image --serve `. @@ -3379,8 +3382,8 @@ Warning: `scan` exists as both a subcommand of OSV-Scanner and as a file on the Scanning local image tarball "../../internal/image/fixtures/test-node_modules-yarn-empty.tar" Container Scanning Result (Alpine Linux v3.19): -Total 2 packages affected by 10 vulnerabilities (0 Critical, 0 High, 4 Medium, 0 Low, 6 Unknown) from 1 ecosystems. -10 vulnerabilities have fixes available. +Total 2 packages affected by 11 vulnerabilities (0 Critical, 0 High, 4 Medium, 0 Low, 7 Unknown) from 1 ecosystems. +11 vulnerabilities have fixes available. Alpine:v3.19 +---------------------------------------------------------------------------------------------+ @@ -3389,7 +3392,7 @@ Alpine:v3.19 | PACKAGE | INSTALLED VERSION | FIX AVAILABLE | VULN COUNT | INTRODUCED LAYER | IN BASE IMAGE | +---------+-------------------+---------------+------------+------------------+---------------+ | busybox | 1.36.1-r15 | Fix Available | 4 | # 0 Layer | alpine | -| openssl | 3.1.4-r5 | Fix Available | 6 | # 0 Layer | alpine | +| openssl | 3.1.4-r5 | Fix Available | 7 | # 0 Layer | alpine | +---------+-------------------+---------------+------------+------------------+---------------+ For the most comprehensive scan results, we recommend using the HTML output: `osv-scanner scan image --serve `. @@ -3406,8 +3409,8 @@ Warning: `scan` exists as both a subcommand of OSV-Scanner and as a file on the Scanning local image tarball "../../internal/image/fixtures/test-node_modules-yarn-full.tar" Container Scanning Result (Alpine Linux v3.19): -Total 2 packages affected by 10 vulnerabilities (0 Critical, 0 High, 4 Medium, 0 Low, 6 Unknown) from 1 ecosystems. -10 vulnerabilities have fixes available. +Total 2 packages affected by 11 vulnerabilities (0 Critical, 0 High, 4 Medium, 0 Low, 7 Unknown) from 1 ecosystems. +11 vulnerabilities have fixes available. Alpine:v3.19 +---------------------------------------------------------------------------------------------+ @@ -3416,7 +3419,7 @@ Alpine:v3.19 | PACKAGE | INSTALLED VERSION | FIX AVAILABLE | VULN COUNT | INTRODUCED LAYER | IN BASE IMAGE | +---------+-------------------+---------------+------------+------------------+---------------+ | busybox | 1.36.1-r15 | Fix Available | 4 | # 0 Layer | alpine | -| openssl | 3.1.4-r5 | Fix Available | 6 | # 0 Layer | alpine | +| openssl | 3.1.4-r5 | Fix Available | 7 | # 0 Layer | alpine | +---------+-------------------+---------------+------------+------------------+---------------+ For the most comprehensive scan results, we recommend using the HTML output: `osv-scanner scan image --serve `.