Skip to content

Commit 4a16ab6

Browse files
another-rexanother-rex
authored
fix: Base image dedup (#1585)
Deduplicate base images to always pick the latest layer when multiple layers returns with the same base image result.
1 parent af1f897 commit 4a16ab6

File tree

2 files changed

+46
-28
lines changed

2 files changed

+46
-28
lines changed

‎cmd/osv-scanner/__snapshots__/main_test.snap

Lines changed: 24 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -732,6 +732,7 @@ Filtered 9 local/unscannable package/s from the scan.
732732
| https://osv.dev/CVE-2023-6129 | 6.5 | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
733733
| https://osv.dev/CVE-2023-6237 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
734734
| https://osv.dev/CVE-2024-0727 | 5.5 | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
735+
| https://osv.dev/CVE-2024-12797 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
735736
| https://osv.dev/CVE-2024-13176 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
736737
| https://osv.dev/CVE-2024-2511 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
737738
| https://osv.dev/CVE-2024-4603 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
@@ -2209,6 +2210,7 @@ Loaded OSS-Fuzz local db from <tempdir>/osv-scanner/OSS-Fuzz/all.zip
22092210
| https://osv.dev/CVE-2023-6129 | 6.5 | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
22102211
| https://osv.dev/CVE-2023-6237 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
22112212
| https://osv.dev/CVE-2024-0727 | 5.5 | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
2213+
| https://osv.dev/CVE-2024-12797 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
22122214
| https://osv.dev/CVE-2024-13176 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
22132215
| https://osv.dev/CVE-2024-2511 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
22142216
| https://osv.dev/CVE-2024-4603 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
@@ -2419,6 +2421,7 @@ Loaded OSS-Fuzz local db from <tempdir>/osv-scanner/OSS-Fuzz/all.zip
24192421
| https://osv.dev/CVE-2023-6129 | 6.5 | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
24202422
| https://osv.dev/CVE-2023-6237 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
24212423
| https://osv.dev/CVE-2024-0727 | 5.5 | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
2424+
| https://osv.dev/CVE-2024-12797 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
24222425
| https://osv.dev/CVE-2024-13176 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
24232426
| https://osv.dev/CVE-2024-2511 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
24242427
| https://osv.dev/CVE-2024-4603 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
@@ -3014,8 +3017,8 @@ failed to load image from tarball with path "./fixtures/oci-image/no-file-here.t
30143017
Scanning local image tarball "../../internal/image/fixtures/test-java-full.tar"
30153018

30163019
Container Scanning Result (Alpine Linux v3.21):
3017-
Total 12 packages affected by 16 vulnerabilities (1 Critical, 5 High, 9 Medium, 0 Low, 1 Unknown) from 2 ecosystems.
3018-
15 vulnerabilities have fixes available.
3020+
Total 12 packages affected by 17 vulnerabilities (1 Critical, 5 High, 9 Medium, 0 Low, 2 Unknown) from 2 ecosystems.
3021+
16 vulnerabilities have fixes available.
30193022

30203023
Maven
30213024
+-----------------------------------------------------------------------------------------------------------------------------------------+
@@ -3041,7 +3044,7 @@ Alpine:v3.21
30413044
| PACKAGE | INSTALLED VERSION | FIX AVAILABLE | VULN COUNT | INTRODUCED LAYER | IN BASE IMAGE |
30423045
+----------+-------------------+---------------+------------+------------------+-----------------+
30433046
| libtasn1 | 4.19.0-r2 | Fix Available | 1 | # 5 Layer | eclipse-temurin |
3044-
| openssl | 3.3.2-r4 | Fix Available | 1 | # 0 Layer | alpine |
3047+
| openssl | 3.3.2-r4 | Fix Available | 2 | # 0 Layer | alpine |
30453048
+----------+-------------------+---------------+------------+------------------+-----------------+
30463049

30473050
For the most comprehensive scan results, we recommend using the HTML output: `osv-scanner scan image --serve <image_name>`.
@@ -3262,8 +3265,8 @@ Warning: `scan` exists as both a subcommand of OSV-Scanner and as a file on the
32623265
Scanning local image tarball "../../internal/image/fixtures/test-node_modules-npm-empty.tar"
32633266

32643267
Container Scanning Result (Alpine Linux v3.19):
3265-
Total 2 packages affected by 10 vulnerabilities (0 Critical, 0 High, 4 Medium, 0 Low, 6 Unknown) from 1 ecosystems.
3266-
10 vulnerabilities have fixes available.
3268+
Total 2 packages affected by 11 vulnerabilities (0 Critical, 0 High, 4 Medium, 0 Low, 7 Unknown) from 1 ecosystems.
3269+
11 vulnerabilities have fixes available.
32673270

32683271
Alpine:v3.19
32693272
+---------------------------------------------------------------------------------------------+
@@ -3272,7 +3275,7 @@ Alpine:v3.19
32723275
| PACKAGE | INSTALLED VERSION | FIX AVAILABLE | VULN COUNT | INTRODUCED LAYER | IN BASE IMAGE |
32733276
+---------+-------------------+---------------+------------+------------------+---------------+
32743277
| busybox | 1.36.1-r15 | Fix Available | 4 | # 0 Layer | alpine |
3275-
| openssl | 3.1.4-r5 | Fix Available | 6 | # 0 Layer | alpine |
3278+
| openssl | 3.1.4-r5 | Fix Available | 7 | # 0 Layer | alpine |
32763279
+---------+-------------------+---------------+------------+------------------+---------------+
32773280

32783281
For the most comprehensive scan results, we recommend using the HTML output: `osv-scanner scan image --serve <image_name>`.
@@ -3289,8 +3292,8 @@ Warning: `scan` exists as both a subcommand of OSV-Scanner and as a file on the
32893292
Scanning local image tarball "../../internal/image/fixtures/test-node_modules-npm-full.tar"
32903293

32913294
Container Scanning Result (Alpine Linux v3.19):
3292-
Total 4 packages affected by 13 vulnerabilities (2 Critical, 0 High, 5 Medium, 0 Low, 6 Unknown) from 2 ecosystems.
3293-
12 vulnerabilities have fixes available.
3295+
Total 4 packages affected by 14 vulnerabilities (2 Critical, 0 High, 5 Medium, 0 Low, 7 Unknown) from 2 ecosystems.
3296+
13 vulnerabilities have fixes available.
32943297

32953298
npm
32963299
+-------------------------------------------------------------------------------------------------+
@@ -3308,7 +3311,7 @@ Alpine:v3.19
33083311
| PACKAGE | INSTALLED VERSION | FIX AVAILABLE | VULN COUNT | INTRODUCED LAYER | IN BASE IMAGE |
33093312
+---------+-------------------+---------------+------------+------------------+---------------+
33103313
| busybox | 1.36.1-r15 | Fix Available | 4 | # 0 Layer | alpine |
3311-
| openssl | 3.1.4-r5 | Fix Available | 6 | # 0 Layer | alpine |
3314+
| openssl | 3.1.4-r5 | Fix Available | 7 | # 0 Layer | alpine |
33123315
+---------+-------------------+---------------+------------+------------------+---------------+
33133316

33143317
For the most comprehensive scan results, we recommend using the HTML output: `osv-scanner scan image --serve <image_name>`.
@@ -3325,8 +3328,8 @@ Warning: `scan` exists as both a subcommand of OSV-Scanner and as a file on the
33253328
Scanning local image tarball "../../internal/image/fixtures/test-node_modules-pnpm-empty.tar"
33263329

33273330
Container Scanning Result (Alpine Linux v3.19):
3328-
Total 2 packages affected by 10 vulnerabilities (0 Critical, 0 High, 4 Medium, 0 Low, 6 Unknown) from 1 ecosystems.
3329-
10 vulnerabilities have fixes available.
3331+
Total 2 packages affected by 11 vulnerabilities (0 Critical, 0 High, 4 Medium, 0 Low, 7 Unknown) from 1 ecosystems.
3332+
11 vulnerabilities have fixes available.
33303333

33313334
Alpine:v3.19
33323335
+---------------------------------------------------------------------------------------------+
@@ -3335,7 +3338,7 @@ Alpine:v3.19
33353338
| PACKAGE | INSTALLED VERSION | FIX AVAILABLE | VULN COUNT | INTRODUCED LAYER | IN BASE IMAGE |
33363339
+---------+-------------------+---------------+------------+------------------+---------------+
33373340
| busybox | 1.36.1-r15 | Fix Available | 4 | # 0 Layer | alpine |
3338-
| openssl | 3.1.4-r5 | Fix Available | 6 | # 0 Layer | alpine |
3341+
| openssl | 3.1.4-r5 | Fix Available | 7 | # 0 Layer | alpine |
33393342
+---------+-------------------+---------------+------------+------------------+---------------+
33403343

33413344
For the most comprehensive scan results, we recommend using the HTML output: `osv-scanner scan image --serve <image_name>`.
@@ -3352,8 +3355,8 @@ Warning: `scan` exists as both a subcommand of OSV-Scanner and as a file on the
33523355
Scanning local image tarball "../../internal/image/fixtures/test-node_modules-pnpm-full.tar"
33533356

33543357
Container Scanning Result (Alpine Linux v3.19):
3355-
Total 2 packages affected by 10 vulnerabilities (0 Critical, 0 High, 4 Medium, 0 Low, 6 Unknown) from 1 ecosystems.
3356-
10 vulnerabilities have fixes available.
3358+
Total 2 packages affected by 11 vulnerabilities (0 Critical, 0 High, 4 Medium, 0 Low, 7 Unknown) from 1 ecosystems.
3359+
11 vulnerabilities have fixes available.
33573360

33583361
Alpine:v3.19
33593362
+---------------------------------------------------------------------------------------------+
@@ -3362,7 +3365,7 @@ Alpine:v3.19
33623365
| PACKAGE | INSTALLED VERSION | FIX AVAILABLE | VULN COUNT | INTRODUCED LAYER | IN BASE IMAGE |
33633366
+---------+-------------------+---------------+------------+------------------+---------------+
33643367
| busybox | 1.36.1-r15 | Fix Available | 4 | # 0 Layer | alpine |
3365-
| openssl | 3.1.4-r5 | Fix Available | 6 | # 0 Layer | alpine |
3368+
| openssl | 3.1.4-r5 | Fix Available | 7 | # 0 Layer | alpine |
33663369
+---------+-------------------+---------------+------------+------------------+---------------+
33673370

33683371
For the most comprehensive scan results, we recommend using the HTML output: `osv-scanner scan image --serve <image_name>`.
@@ -3379,8 +3382,8 @@ Warning: `scan` exists as both a subcommand of OSV-Scanner and as a file on the
33793382
Scanning local image tarball "../../internal/image/fixtures/test-node_modules-yarn-empty.tar"
33803383

33813384
Container Scanning Result (Alpine Linux v3.19):
3382-
Total 2 packages affected by 10 vulnerabilities (0 Critical, 0 High, 4 Medium, 0 Low, 6 Unknown) from 1 ecosystems.
3383-
10 vulnerabilities have fixes available.
3385+
Total 2 packages affected by 11 vulnerabilities (0 Critical, 0 High, 4 Medium, 0 Low, 7 Unknown) from 1 ecosystems.
3386+
11 vulnerabilities have fixes available.
33843387

33853388
Alpine:v3.19
33863389
+---------------------------------------------------------------------------------------------+
@@ -3389,7 +3392,7 @@ Alpine:v3.19
33893392
| PACKAGE | INSTALLED VERSION | FIX AVAILABLE | VULN COUNT | INTRODUCED LAYER | IN BASE IMAGE |
33903393
+---------+-------------------+---------------+------------+------------------+---------------+
33913394
| busybox | 1.36.1-r15 | Fix Available | 4 | # 0 Layer | alpine |
3392-
| openssl | 3.1.4-r5 | Fix Available | 6 | # 0 Layer | alpine |
3395+
| openssl | 3.1.4-r5 | Fix Available | 7 | # 0 Layer | alpine |
33933396
+---------+-------------------+---------------+------------+------------------+---------------+
33943397

33953398
For the most comprehensive scan results, we recommend using the HTML output: `osv-scanner scan image --serve <image_name>`.
@@ -3406,8 +3409,8 @@ Warning: `scan` exists as both a subcommand of OSV-Scanner and as a file on the
34063409
Scanning local image tarball "../../internal/image/fixtures/test-node_modules-yarn-full.tar"
34073410

34083411
Container Scanning Result (Alpine Linux v3.19):
3409-
Total 2 packages affected by 10 vulnerabilities (0 Critical, 0 High, 4 Medium, 0 Low, 6 Unknown) from 1 ecosystems.
3410-
10 vulnerabilities have fixes available.
3412+
Total 2 packages affected by 11 vulnerabilities (0 Critical, 0 High, 4 Medium, 0 Low, 7 Unknown) from 1 ecosystems.
3413+
11 vulnerabilities have fixes available.
34113414

34123415
Alpine:v3.19
34133416
+---------------------------------------------------------------------------------------------+
@@ -3416,7 +3419,7 @@ Alpine:v3.19
34163419
| PACKAGE | INSTALLED VERSION | FIX AVAILABLE | VULN COUNT | INTRODUCED LAYER | IN BASE IMAGE |
34173420
+---------+-------------------+---------------+------------+------------------+---------------+
34183421
| busybox | 1.36.1-r15 | Fix Available | 4 | # 0 Layer | alpine |
3419-
| openssl | 3.1.4-r5 | Fix Available | 6 | # 0 Layer | alpine |
3422+
| openssl | 3.1.4-r5 | Fix Available | 7 | # 0 Layer | alpine |
34203423
+---------+-------------------+---------------+------------+------------------+---------------+
34213424

34223425
For the most comprehensive scan results, we recommend using the HTML output: `osv-scanner scan image --serve <image_name>`.

‎internal/clients/clientimpl/baseimagematcher/baseimagematcher.go

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,10 @@ import (
1010
"math/rand/v2"
1111
"net/http"
1212
"slices"
13+
"strings"
1314
"time"
1415

16+
"github.com/google/go-cmp/cmp"
1517
"github.com/google/osv-scanner/v2/pkg/models"
1618
"github.com/google/osv-scanner/v2/pkg/reporter"
1719
"github.com/opencontainers/go-digest"
@@ -37,7 +39,7 @@ type DepsDevBaseImageMatcher struct {
3739
}
3840

3941
func (matcher *DepsDevBaseImageMatcher) MatchBaseImages(ctx context.Context, layerMetadata []models.LayerMetadata) ([][]models.BaseImageDetails, error) {
40-
baseImagesMap := make([][]models.BaseImageDetails, len(layerMetadata))
42+
baseImagesToLayerMap := make([][]models.BaseImageDetails, len(layerMetadata))
4143
g, ctx := errgroup.WithContext(ctx)
4244
g.SetLimit(maxConcurrentRequests)
4345

@@ -62,7 +64,7 @@ func (matcher *DepsDevBaseImageMatcher) MatchBaseImages(ctx context.Context, lay
6264

6365
// If we are erroring for one base image even with retry, we probably should stop
6466
var err error
65-
baseImagesMap[i], err = matcher.queryBaseImagesForChainID(ctx, chainID)
67+
baseImagesToLayerMap[i], err = matcher.queryBaseImagesForChainID(ctx, chainID)
6668

6769
return err
6870
})
@@ -72,7 +74,7 @@ func (matcher *DepsDevBaseImageMatcher) MatchBaseImages(ctx context.Context, lay
7274
return nil, err
7375
}
7476

75-
return buildBaseImageDetails(layerMetadata, baseImagesMap), nil
77+
return buildBaseImageDetails(layerMetadata, baseImagesToLayerMap), nil
7678
}
7779

7880
// makeRetryRequest will return an error on both network errors, and if the response is not 200 or 404
@@ -183,27 +185,40 @@ func (matcher *DepsDevBaseImageMatcher) queryBaseImagesForChainID(ctx context.Co
183185
// TODO(v2): Temporary heuristic for what is more popular
184186
// Ideally this is done by deps.dev before release
185187
slices.SortFunc(baseImagePossibilities, func(a, b models.BaseImageDetails) int {
186-
return len(a.Name) - len(b.Name)
188+
lengthDiff := len(a.Name) - len(b.Name)
189+
if lengthDiff != 0 {
190+
return lengthDiff
191+
}
192+
193+
// Apply deterministic ordering to same length base images
194+
return strings.Compare(a.Name, b.Name)
187195
})
188196

189197
return baseImagePossibilities, nil
190198
}
191199

192-
func buildBaseImageDetails(layerMetadata []models.LayerMetadata, baseImagesMap [][]models.BaseImageDetails) [][]models.BaseImageDetails {
200+
func buildBaseImageDetails(layerMetadata []models.LayerMetadata, baseImagesToLayersMap [][]models.BaseImageDetails) [][]models.BaseImageDetails {
193201
allBaseImages := [][]models.BaseImageDetails{
194202
// The base image at index 0 is a placeholder representing your image, so always empty
195203
// This is the case even if your image is a base image, in that case no layers point to index 0
196204
{},
197205
}
198206

199207
currentBaseImageIndex := 0
200-
for i, baseImages := range slices.Backward(baseImagesMap) {
208+
for i, baseImages := range slices.Backward(baseImagesToLayersMap) {
201209
if len(baseImages) == 0 {
202210
layerMetadata[i].BaseImageIndex = currentBaseImageIndex
203211
continue
204212
}
205213

206-
// This layer is a base image boundary
214+
// Is the current set of baseImages the same as the previous?
215+
if cmp.Equal(baseImages, allBaseImages[len(allBaseImages)-1]) {
216+
// If so, merge them
217+
layerMetadata[i].BaseImageIndex = currentBaseImageIndex
218+
continue
219+
}
220+
221+
// This layer is a new base image boundary
207222
allBaseImages = append(allBaseImages, baseImages)
208223
currentBaseImageIndex += 1
209224
layerMetadata[i].BaseImageIndex = currentBaseImageIndex

0 commit comments

Comments
 (0)