Skip to content

Commit

Permalink
fix: Base image dedup (#1585)
Browse files Browse the repository at this point in the history 
Deduplicate base images to always pick the latest layer when multiple
layers returns with the same base image result.
  • Loading branch information
@another-rex
another-rex authored Feb 11, 2025
1 parent af1f897 commit 4a16ab6
Show file tree
Hide file tree
Showing 2 changed files with 46 additions and 28 deletions.
45 changes: 24 additions & 21 deletions cmd/osv-scanner/__snapshots__/main_test.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -732,6 +732,7 @@ Filtered 9 local/unscannable package/s from the scan.
| https://osv.dev/CVE-2023-6129 | 6.5 | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
| https://osv.dev/CVE-2023-6237 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
| https://osv.dev/CVE-2024-0727 | 5.5 | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
| https://osv.dev/CVE-2024-12797 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
| https://osv.dev/CVE-2024-13176 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
| https://osv.dev/CVE-2024-2511 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
| https://osv.dev/CVE-2024-4603 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
Expand Down Expand Up @@ -2209,6 +2210,7 @@ Loaded OSS-Fuzz local db from <tempdir>/osv-scanner/OSS-Fuzz/all.zip
| https://osv.dev/CVE-2023-6129 | 6.5 | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
| https://osv.dev/CVE-2023-6237 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
| https://osv.dev/CVE-2024-0727 | 5.5 | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
| https://osv.dev/CVE-2024-12797 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
| https://osv.dev/CVE-2024-13176 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
| https://osv.dev/CVE-2024-2511 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
| https://osv.dev/CVE-2024-4603 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
Expand Down Expand Up @@ -2419,6 +2421,7 @@ Loaded OSS-Fuzz local db from <tempdir>/osv-scanner/OSS-Fuzz/all.zip
| https://osv.dev/CVE-2023-6129 | 6.5 | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
| https://osv.dev/CVE-2023-6237 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
| https://osv.dev/CVE-2024-0727 | 5.5 | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
| https://osv.dev/CVE-2024-12797 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
| https://osv.dev/CVE-2024-13176 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
| https://osv.dev/CVE-2024-2511 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
| https://osv.dev/CVE-2024-4603 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
Expand Down Expand Up @@ -3014,8 +3017,8 @@ failed to load image from tarball with path "./fixtures/oci-image/no-file-here.t
Scanning local image tarball "../../internal/image/fixtures/test-java-full.tar"

Container Scanning Result (Alpine Linux v3.21):
Total 12 packages affected by 16 vulnerabilities (1 Critical, 5 High, 9 Medium, 0 Low, 1 Unknown) from 2 ecosystems.
15 vulnerabilities have fixes available.
Total 12 packages affected by 17 vulnerabilities (1 Critical, 5 High, 9 Medium, 0 Low, 2 Unknown) from 2 ecosystems.
16 vulnerabilities have fixes available.

Maven
+-----------------------------------------------------------------------------------------------------------------------------------------+
Expand All @@ -3041,7 +3044,7 @@ Alpine:v3.21
| PACKAGE | INSTALLED VERSION | FIX AVAILABLE | VULN COUNT | INTRODUCED LAYER | IN BASE IMAGE |
+----------+-------------------+---------------+------------+------------------+-----------------+
| libtasn1 | 4.19.0-r2 | Fix Available | 1 | # 5 Layer | eclipse-temurin |
| openssl | 3.3.2-r4 | Fix Available | 1 | # 0 Layer | alpine |
| openssl | 3.3.2-r4 | Fix Available | 2 | # 0 Layer | alpine |
+----------+-------------------+---------------+------------+------------------+-----------------+

For the most comprehensive scan results, we recommend using the HTML output: `osv-scanner scan image --serve <image_name>`.
Expand Down Expand Up @@ -3262,8 +3265,8 @@ Warning: `scan` exists as both a subcommand of OSV-Scanner and as a file on the
Scanning local image tarball "../../internal/image/fixtures/test-node_modules-npm-empty.tar"

Container Scanning Result (Alpine Linux v3.19):
Total 2 packages affected by 10 vulnerabilities (0 Critical, 0 High, 4 Medium, 0 Low, 6 Unknown) from 1 ecosystems.
10 vulnerabilities have fixes available.
Total 2 packages affected by 11 vulnerabilities (0 Critical, 0 High, 4 Medium, 0 Low, 7 Unknown) from 1 ecosystems.
11 vulnerabilities have fixes available.

Alpine:v3.19
+---------------------------------------------------------------------------------------------+
Expand All @@ -3272,7 +3275,7 @@ Alpine:v3.19
| PACKAGE | INSTALLED VERSION | FIX AVAILABLE | VULN COUNT | INTRODUCED LAYER | IN BASE IMAGE |
+---------+-------------------+---------------+------------+------------------+---------------+
| busybox | 1.36.1-r15 | Fix Available | 4 | # 0 Layer | alpine |
| openssl | 3.1.4-r5 | Fix Available | 6 | # 0 Layer | alpine |
| openssl | 3.1.4-r5 | Fix Available | 7 | # 0 Layer | alpine |
+---------+-------------------+---------------+------------+------------------+---------------+

For the most comprehensive scan results, we recommend using the HTML output: `osv-scanner scan image --serve <image_name>`.
Expand All @@ -3289,8 +3292,8 @@ Warning: `scan` exists as both a subcommand of OSV-Scanner and as a file on the
Scanning local image tarball "../../internal/image/fixtures/test-node_modules-npm-full.tar"

Container Scanning Result (Alpine Linux v3.19):
Total 4 packages affected by 13 vulnerabilities (2 Critical, 0 High, 5 Medium, 0 Low, 6 Unknown) from 2 ecosystems.
12 vulnerabilities have fixes available.
Total 4 packages affected by 14 vulnerabilities (2 Critical, 0 High, 5 Medium, 0 Low, 7 Unknown) from 2 ecosystems.
13 vulnerabilities have fixes available.

npm
+-------------------------------------------------------------------------------------------------+
Expand All @@ -3308,7 +3311,7 @@ Alpine:v3.19
| PACKAGE | INSTALLED VERSION | FIX AVAILABLE | VULN COUNT | INTRODUCED LAYER | IN BASE IMAGE |
+---------+-------------------+---------------+------------+------------------+---------------+
| busybox | 1.36.1-r15 | Fix Available | 4 | # 0 Layer | alpine |
| openssl | 3.1.4-r5 | Fix Available | 6 | # 0 Layer | alpine |
| openssl | 3.1.4-r5 | Fix Available | 7 | # 0 Layer | alpine |
+---------+-------------------+---------------+------------+------------------+---------------+

For the most comprehensive scan results, we recommend using the HTML output: `osv-scanner scan image --serve <image_name>`.
Expand All @@ -3325,8 +3328,8 @@ Warning: `scan` exists as both a subcommand of OSV-Scanner and as a file on the
Scanning local image tarball "../../internal/image/fixtures/test-node_modules-pnpm-empty.tar"

Container Scanning Result (Alpine Linux v3.19):
Total 2 packages affected by 10 vulnerabilities (0 Critical, 0 High, 4 Medium, 0 Low, 6 Unknown) from 1 ecosystems.
10 vulnerabilities have fixes available.
Total 2 packages affected by 11 vulnerabilities (0 Critical, 0 High, 4 Medium, 0 Low, 7 Unknown) from 1 ecosystems.
11 vulnerabilities have fixes available.

Alpine:v3.19
+---------------------------------------------------------------------------------------------+
Expand All @@ -3335,7 +3338,7 @@ Alpine:v3.19
| PACKAGE | INSTALLED VERSION | FIX AVAILABLE | VULN COUNT | INTRODUCED LAYER | IN BASE IMAGE |
+---------+-------------------+---------------+------------+------------------+---------------+
| busybox | 1.36.1-r15 | Fix Available | 4 | # 0 Layer | alpine |
| openssl | 3.1.4-r5 | Fix Available | 6 | # 0 Layer | alpine |
| openssl | 3.1.4-r5 | Fix Available | 7 | # 0 Layer | alpine |
+---------+-------------------+---------------+------------+------------------+---------------+

For the most comprehensive scan results, we recommend using the HTML output: `osv-scanner scan image --serve <image_name>`.
Expand All @@ -3352,8 +3355,8 @@ Warning: `scan` exists as both a subcommand of OSV-Scanner and as a file on the
Scanning local image tarball "../../internal/image/fixtures/test-node_modules-pnpm-full.tar"

Container Scanning Result (Alpine Linux v3.19):
Total 2 packages affected by 10 vulnerabilities (0 Critical, 0 High, 4 Medium, 0 Low, 6 Unknown) from 1 ecosystems.
10 vulnerabilities have fixes available.
Total 2 packages affected by 11 vulnerabilities (0 Critical, 0 High, 4 Medium, 0 Low, 7 Unknown) from 1 ecosystems.
11 vulnerabilities have fixes available.

Alpine:v3.19
+---------------------------------------------------------------------------------------------+
Expand All @@ -3362,7 +3365,7 @@ Alpine:v3.19
| PACKAGE | INSTALLED VERSION | FIX AVAILABLE | VULN COUNT | INTRODUCED LAYER | IN BASE IMAGE |
+---------+-------------------+---------------+------------+------------------+---------------+
| busybox | 1.36.1-r15 | Fix Available | 4 | # 0 Layer | alpine |
| openssl | 3.1.4-r5 | Fix Available | 6 | # 0 Layer | alpine |
| openssl | 3.1.4-r5 | Fix Available | 7 | # 0 Layer | alpine |
+---------+-------------------+---------------+------------+------------------+---------------+

For the most comprehensive scan results, we recommend using the HTML output: `osv-scanner scan image --serve <image_name>`.
Expand All @@ -3379,8 +3382,8 @@ Warning: `scan` exists as both a subcommand of OSV-Scanner and as a file on the
Scanning local image tarball "../../internal/image/fixtures/test-node_modules-yarn-empty.tar"

Container Scanning Result (Alpine Linux v3.19):
Total 2 packages affected by 10 vulnerabilities (0 Critical, 0 High, 4 Medium, 0 Low, 6 Unknown) from 1 ecosystems.
10 vulnerabilities have fixes available.
Total 2 packages affected by 11 vulnerabilities (0 Critical, 0 High, 4 Medium, 0 Low, 7 Unknown) from 1 ecosystems.
11 vulnerabilities have fixes available.

Alpine:v3.19
+---------------------------------------------------------------------------------------------+
Expand All @@ -3389,7 +3392,7 @@ Alpine:v3.19
| PACKAGE | INSTALLED VERSION | FIX AVAILABLE | VULN COUNT | INTRODUCED LAYER | IN BASE IMAGE |
+---------+-------------------+---------------+------------+------------------+---------------+
| busybox | 1.36.1-r15 | Fix Available | 4 | # 0 Layer | alpine |
| openssl | 3.1.4-r5 | Fix Available | 6 | # 0 Layer | alpine |
| openssl | 3.1.4-r5 | Fix Available | 7 | # 0 Layer | alpine |
+---------+-------------------+---------------+------------+------------------+---------------+

For the most comprehensive scan results, we recommend using the HTML output: `osv-scanner scan image --serve <image_name>`.
Expand All @@ -3406,8 +3409,8 @@ Warning: `scan` exists as both a subcommand of OSV-Scanner and as a file on the
Scanning local image tarball "../../internal/image/fixtures/test-node_modules-yarn-full.tar"

Container Scanning Result (Alpine Linux v3.19):
Total 2 packages affected by 10 vulnerabilities (0 Critical, 0 High, 4 Medium, 0 Low, 6 Unknown) from 1 ecosystems.
10 vulnerabilities have fixes available.
Total 2 packages affected by 11 vulnerabilities (0 Critical, 0 High, 4 Medium, 0 Low, 7 Unknown) from 1 ecosystems.
11 vulnerabilities have fixes available.

Alpine:v3.19
+---------------------------------------------------------------------------------------------+
Expand All @@ -3416,7 +3419,7 @@ Alpine:v3.19
| PACKAGE | INSTALLED VERSION | FIX AVAILABLE | VULN COUNT | INTRODUCED LAYER | IN BASE IMAGE |
+---------+-------------------+---------------+------------+------------------+---------------+
| busybox | 1.36.1-r15 | Fix Available | 4 | # 0 Layer | alpine |
| openssl | 3.1.4-r5 | Fix Available | 6 | # 0 Layer | alpine |
| openssl | 3.1.4-r5 | Fix Available | 7 | # 0 Layer | alpine |
+---------+-------------------+---------------+------------+------------------+---------------+

For the most comprehensive scan results, we recommend using the HTML output: `osv-scanner scan image --serve <image_name>`.
Expand Down
29 changes: 22 additions & 7 deletions internal/clients/clientimpl/baseimagematcher/baseimagematcher.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,10 @@ import (
"math/rand/v2"
"net/http"
"slices"
"strings"
"time"

"github.com/google/go-cmp/cmp"
"github.com/google/osv-scanner/v2/pkg/models"
"github.com/google/osv-scanner/v2/pkg/reporter"
"github.com/opencontainers/go-digest"
Expand All @@ -37,7 +39,7 @@ type DepsDevBaseImageMatcher struct {
}

func (matcher *DepsDevBaseImageMatcher) MatchBaseImages(ctx context.Context, layerMetadata []models.LayerMetadata) ([][]models.BaseImageDetails, error) {
baseImagesMap := make([][]models.BaseImageDetails, len(layerMetadata))
baseImagesToLayerMap := make([][]models.BaseImageDetails, len(layerMetadata))
g, ctx := errgroup.WithContext(ctx)
g.SetLimit(maxConcurrentRequests)

Expand All @@ -62,7 +64,7 @@ func (matcher *DepsDevBaseImageMatcher) MatchBaseImages(ctx context.Context, lay

// If we are erroring for one base image even with retry, we probably should stop
var err error
baseImagesMap[i], err = matcher.queryBaseImagesForChainID(ctx, chainID)
baseImagesToLayerMap[i], err = matcher.queryBaseImagesForChainID(ctx, chainID)

return err
})
Expand All @@ -72,7 +74,7 @@ func (matcher *DepsDevBaseImageMatcher) MatchBaseImages(ctx context.Context, lay
return nil, err
}

return buildBaseImageDetails(layerMetadata, baseImagesMap), nil
return buildBaseImageDetails(layerMetadata, baseImagesToLayerMap), nil
}

// makeRetryRequest will return an error on both network errors, and if the response is not 200 or 404
Expand Down Expand Up @@ -183,27 +185,40 @@ func (matcher *DepsDevBaseImageMatcher) queryBaseImagesForChainID(ctx context.Co
// TODO(v2): Temporary heuristic for what is more popular
// Ideally this is done by deps.dev before release
slices.SortFunc(baseImagePossibilities, func(a, b models.BaseImageDetails) int {
return len(a.Name) - len(b.Name)
lengthDiff := len(a.Name) - len(b.Name)
if lengthDiff != 0 {
return lengthDiff
}

// Apply deterministic ordering to same length base images
return strings.Compare(a.Name, b.Name)
})

return baseImagePossibilities, nil
}

func buildBaseImageDetails(layerMetadata []models.LayerMetadata, baseImagesMap [][]models.BaseImageDetails) [][]models.BaseImageDetails {
func buildBaseImageDetails(layerMetadata []models.LayerMetadata, baseImagesToLayersMap [][]models.BaseImageDetails) [][]models.BaseImageDetails {
allBaseImages := [][]models.BaseImageDetails{
// The base image at index 0 is a placeholder representing your image, so always empty
// This is the case even if your image is a base image, in that case no layers point to index 0
{},
}

currentBaseImageIndex := 0
for i, baseImages := range slices.Backward(baseImagesMap) {
for i, baseImages := range slices.Backward(baseImagesToLayersMap) {
if len(baseImages) == 0 {
layerMetadata[i].BaseImageIndex = currentBaseImageIndex
continue
}

// This layer is a base image boundary
// Is the current set of baseImages the same as the previous?
if cmp.Equal(baseImages, allBaseImages[len(allBaseImages)-1]) {
// If so, merge them
layerMetadata[i].BaseImageIndex = currentBaseImageIndex
continue
}

// This layer is a new base image boundary
allBaseImages = append(allBaseImages, baseImages)
currentBaseImageIndex += 1
layerMetadata[i].BaseImageIndex = currentBaseImageIndex
Expand Down

0 comments on commit 4a16ab6

Please sign in to comment.