Skip to content

Commit ebcb244

Browse files
tatianabtatianab
committed
cmd/vulnreport: check if packages exist in vulnreport fix
Add a simple check for package existence in vulnreport fix, which pings pkg.go.dev to determine if a package exists. This is more likely to succeed (and faster) than the package/symbol check which downloads the whole package. We now skip this symbol-check when there are no symbols listed. There are still some cases in which this fails incorrectly (e.g. if pkgsite for some reason couldn't cache the given package/version), so the check can be bypassed. Change-Id: I922eae0dec9a376210f0f0fd1d70a67da934ffaa Reviewed-on: https://go-review.googlesource.com/c/vulndb/+/599180 Reviewed-by: Damien Neil <[email protected]> LUCI-TryBot-Result: Go LUCI <[email protected]>
1 parent 0550a0e commit ebcb244

File tree

29 files changed

+331
-68
lines changed

29 files changed

+331
-68
lines changed

‎cmd/vulnreport/creator.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,7 @@ func (c *creator) metaToSource(ctx context.Context, meta *reportMeta) report.Sou
161161
}
162162

163163
func (c *creator) rawReport(ctx context.Context, meta *reportMeta) *report.Report {
164-
return report.New(c.metaToSource(ctx, meta), c.pc,
164+
return report.New(c.metaToSource(ctx, meta), c.pxc,
165165
report.WithGoID(meta.id),
166166
report.WithModulePath(meta.modulePath),
167167
report.WithAliases(meta.aliases),
@@ -172,7 +172,7 @@ func (c *creator) rawReport(ctx context.Context, meta *reportMeta) *report.Repor
172172

173173
func (c *creator) reportFromMeta(ctx context.Context, meta *reportMeta) (*yamlReport, error) {
174174
// Find the underlying module if the "module" provided is actually a package path.
175-
if module, err := c.pc.FindModule(meta.modulePath); err == nil { // no error
175+
if module, err := c.pxc.FindModule(meta.modulePath); err == nil { // no error
176176
meta.modulePath = module
177177
}
178178
meta.aliases = c.allAliases(ctx, meta.aliases)

‎cmd/vulnreport/environment.go

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,16 @@ import (
1515
"golang.org/x/vulndb/internal/ghsa"
1616
"golang.org/x/vulndb/internal/gitrepo"
1717
"golang.org/x/vulndb/internal/issues"
18+
"golang.org/x/vulndb/internal/pkgsite"
1819
"golang.org/x/vulndb/internal/proxy"
1920
)
2021

2122
// environment stores fakes/mocks of external dependencies for testing.
2223
type environment struct {
2324
reportRepo *git.Repository
2425
reportFS fs.FS
25-
pc *proxy.Client
26+
pxc *proxy.Client
27+
pkc *pkgsite.Client
2628
wfs wfs
2729
ic issueClient
2830
gc ghsaClient
@@ -50,13 +52,21 @@ func (e *environment) ReportFS() fs.FS {
5052
}
5153

5254
func (e *environment) ProxyClient() *proxy.Client {
53-
if v := e.pc; v != nil {
55+
if v := e.pxc; v != nil {
5456
return v
5557
}
5658

5759
return proxy.NewDefaultClient()
5860
}
5961

62+
func (e *environment) PkgsiteClient() *pkgsite.Client {
63+
if v := e.pkc; v != nil {
64+
return v
65+
}
66+
67+
return pkgsite.Default()
68+
}
69+
6070
func (e *environment) WFS() wfs {
6171
if v := e.wfs; v != nil {
6272
return v

‎cmd/vulnreport/fix.go

Lines changed: 47 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,18 @@ import (
1717
"golang.org/x/exp/slices"
1818
"golang.org/x/vulndb/cmd/vulnreport/log"
1919
"golang.org/x/vulndb/internal/osvutils"
20+
"golang.org/x/vulndb/internal/pkgsite"
2021
"golang.org/x/vulndb/internal/report"
2122
"golang.org/x/vulndb/internal/symbols"
2223
)
2324

2425
var (
25-
force = flag.Bool("f", false, "for fix, force Fix to run even if there are no lint errors")
26-
skipChecks = flag.Bool("skip-checks", false, "for fix, skip all checks except lint")
27-
skipAlias = flag.Bool("skip-alias", false, "for fix, skip adding new GHSAs and CVEs")
28-
skipSymbols = flag.Bool("skip-symbols", false, "for lint and fix, don't load package for symbols checks")
26+
force = flag.Bool("f", false, "for fix, force Fix to run even if there are no lint errors")
27+
skipChecks = flag.Bool("skip-checks", false, "for fix, skip all checks except lint")
28+
skipAlias = flag.Bool("skip-alias", false, "for fix, skip adding new GHSAs and CVEs")
29+
skipSymbols = flag.Bool("skip-symbols", false, "for fix, don't load package for symbols checks")
30+
skipPackages = flag.Bool("skip-packages", false, "for fix, don't check if packages exist")
31+
skipRefs = flag.Bool("skip-refs", false, "for fix, don't check if references exist")
2932
)
3033

3134
type fix struct {
@@ -58,9 +61,12 @@ type fixer struct {
5861
*linter
5962
*aliasFinder
6063
*fileWriter
64+
65+
pkc *pkgsite.Client
6166
}
6267

6368
func (f *fixer) setup(ctx context.Context, env environment) error {
69+
f.pkc = env.PkgsiteClient()
6470
f.linter = new(linter)
6571
f.aliasFinder = new(aliasFinder)
6672
f.fileWriter = new(fileWriter)
@@ -85,8 +91,8 @@ func (f *fixer) fixAndWriteAll(ctx context.Context, r *yamlReport, addNotes bool
8591
func (f *fixer) fix(ctx context.Context, r *yamlReport, addNotes bool) (fixed bool) {
8692
fixed = true
8793

88-
if lints := r.Lint(f.pc); *force || len(lints) > 0 {
89-
r.Fix(f.pc)
94+
if lints := r.Lint(f.pxc); *force || len(lints) > 0 {
95+
r.Fix(f.pxc)
9096
}
9197

9298
if !*skipChecks {
@@ -97,12 +103,12 @@ func (f *fixer) fix(ctx context.Context, r *yamlReport, addNotes bool) (fixed bo
97103

98104
// Check for remaining lint errors.
99105
if addNotes {
100-
if r.LintAsNotes(f.pc) {
106+
if r.LintAsNotes(f.pxc) {
101107
log.Warnf("%s: still has lint errors after fix", r.ID)
102108
fixed = false
103109
}
104110
} else {
105-
if lints := r.Lint(f.pc); len(lints) > 0 {
111+
if lints := r.Lint(f.pxc); len(lints) > 0 {
106112
log.Warnf("%s: still has lint errors after fix:\n\t- %s", r.ID, strings.Join(lints, "\n\t- "))
107113
fixed = false
108114
}
@@ -124,10 +130,17 @@ func (f *fixer) allChecks(ctx context.Context, r *yamlReport, addNotes bool) (ok
124130
ok = false
125131
}
126132

133+
if !*skipPackages {
134+
log.Infof("%s: checking that all packages exist", r.ID)
135+
if err := r.CheckPackages(ctx, f.pkc); err != nil {
136+
fixErr("package error: %s", err)
137+
}
138+
}
139+
127140
if !*skipSymbols {
128-
log.Infof("%s: checking packages and symbols (use -skip-symbols to skip this)", r.ID)
141+
log.Infof("%s: checking symbols (use -skip-symbols to skip this)", r.ID)
129142
if err := r.checkSymbols(); err != nil {
130-
fixErr("package or symbol error: %s", err)
143+
fixErr("symbol error: %s", err)
131144
}
132145
}
133146

@@ -138,9 +151,11 @@ func (f *fixer) allChecks(ctx context.Context, r *yamlReport, addNotes bool) (ok
138151
}
139152
}
140153

141-
// For now, this is a fix check instead of a lint.
142-
log.Infof("%s: checking that all references are reachable", r.ID)
143-
checkRefs(r.References, fixErr)
154+
if !*skipRefs {
155+
// For now, this is a fix check instead of a lint.
156+
log.Infof("%s: checking that all references are reachable", r.ID)
157+
checkRefs(r.References, fixErr)
158+
}
144159

145160
return ok
146161
}
@@ -171,7 +186,16 @@ func (r *yamlReport) checkSymbols() error {
171186
log.Infof("%s: excluded, skipping symbol checks", r.ID)
172187
return nil
173188
}
189+
if len(r.Modules) == 0 {
190+
log.Infof("%s: no modules, skipping symbol checks", r.ID)
191+
return nil
192+
}
174193
for _, m := range r.Modules {
194+
if len(m.Packages) == 0 {
195+
log.Infof("%s: module %s has no packages, skipping symbol checks", r.ID, m.Module)
196+
return nil
197+
}
198+
175199
if m.IsFirstParty() {
176200
gover := runtime.Version()
177201
ver := semverForGoVersion(gover)
@@ -196,8 +220,16 @@ func (r *yamlReport) checkSymbols() error {
196220
}
197221

198222
for _, p := range m.Packages {
199-
if p.SkipFix != "" {
200-
log.Infof("%s: skipping symbol checks for package %s (reason: %q)", r.ID, p.Package, p.SkipFix)
223+
if len(p.AllSymbols()) == 0 && p.SkipFixSymbols != "" {
224+
log.Warnf("%s: skip_fix not needed", r.Filename)
225+
continue
226+
}
227+
if len(p.AllSymbols()) == 0 {
228+
log.Infof("%s: skipping symbol checks for package %s (no symbols)", r.ID, p.Package)
229+
continue
230+
}
231+
if p.SkipFixSymbols != "" {
232+
log.Infof("%s: skipping symbol checks for package %s (reason: %q)", r.ID, p.Package, p.SkipFixSymbols)
201233
continue
202234
}
203235
syms, err := symbols.Exported(m, p)

‎cmd/vulnreport/lint.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -39,26 +39,26 @@ func (l *lint) run(_ context.Context, input any) error {
3939
}
4040

4141
type linter struct {
42-
pc *proxy.Client
42+
pxc *proxy.Client
4343
}
4444

4545
func (l *linter) setup(_ context.Context, env environment) error {
46-
l.pc = env.ProxyClient()
46+
l.pxc = env.ProxyClient()
4747
return nil
4848
}
4949

5050
func (l *linter) lint(r *yamlReport) error {
51-
if lints := r.Lint(l.pc); len(lints) > 0 {
51+
if lints := r.Lint(l.pxc); len(lints) > 0 {
5252
return fmt.Errorf("%v has %d lint warnings:%s%s", r.ID, len(lints), listItem, strings.Join(lints, listItem))
5353
}
5454
return nil
5555
}
5656

5757
func (l *linter) canonicalModule(mp string) string {
58-
if module, err := l.pc.FindModule(mp); err == nil { // no error
58+
if module, err := l.pxc.FindModule(mp); err == nil { // no error
5959
mp = module
6060
}
61-
if module, err := l.pc.CanonicalAtLatest(mp); err == nil { // no error
61+
if module, err := l.pxc.CanonicalAtLatest(mp); err == nil { // no error
6262
mp = module
6363
}
6464
return mp

‎cmd/vulnreport/run_test.go

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,16 @@ import (
2121
"golang.org/x/vulndb/cmd/vulnreport/log"
2222
"golang.org/x/vulndb/cmd/vulnreport/priority"
2323
"golang.org/x/vulndb/internal/gitrepo"
24+
"golang.org/x/vulndb/internal/pkgsite"
2425
"golang.org/x/vulndb/internal/proxy"
2526
"golang.org/x/vulndb/internal/test"
2627
)
2728

28-
// go test ./cmd/vulnreport -update-test -proxy
29+
// go test ./cmd/vulnreport -update-test -proxy -pkgsite
2930
var (
3031
testUpdate = flag.Bool("update-test", false, "(for test) whether to update test files")
3132
realProxy = flag.Bool("proxy", false, "(for test) whether to use real proxy")
33+
usePkgsite = flag.Bool("pkgsite", false, "(for test) whether to use real pkgsite")
3234
)
3335

3436
type testCase struct {
@@ -91,7 +93,12 @@ func newDefaultTestEnv(t *testing.T) (*environment, error) {
9193
return nil, err
9294
}
9395

94-
pc, err := proxy.NewTestClient(t, *realProxy)
96+
pxc, err := proxy.NewTestClient(t, *realProxy)
97+
if err != nil {
98+
return nil, err
99+
}
100+
101+
pkc, err := pkgsite.TestClient(t, *usePkgsite)
95102
if err != nil {
96103
return nil, err
97104
}
@@ -113,7 +120,8 @@ func newDefaultTestEnv(t *testing.T) (*environment, error) {
113120
return &environment{
114121
reportRepo: repo,
115122
reportFS: fsys,
116-
pc: pc,
123+
pxc: pxc,
124+
pkc: pkc,
117125
wfs: newInMemoryWFS(),
118126
ic: ic,
119127
gc: gc,

‎cmd/vulnreport/testdata/TestFix/no_change.txtar

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,19 @@ data/osv/GO-9999-0001.json
1111
-- logs --
1212
info: fix: operating on 1 report(s)
1313
info: fix data/reports/GO-9999-0001.yaml
14-
info: GO-9999-0001: checking packages and symbols (use -skip-symbols to skip this)
14+
info: GO-9999-0001: checking that all packages exist
15+
info: GO-9999-0001: checking symbols (use -skip-symbols to skip this)
16+
info: GO-9999-0001: skipping symbol checks for package golang.org/x/vulndb/cmd/vulnreport (no symbols)
1517
info: GO-9999-0001: checking for missing GHSAs and CVEs (use -skip-alias to skip this)
1618
info: GO-9999-0001: checking that all references are reachable
1719
info: fix: processed 1 report(s) (success=1; skip=0; error=0)
1820
-- data/reports/GO-9999-0001.yaml --
1921
id: GO-9999-0001
2022
modules:
2123
- module: golang.org/x/vulndb
24+
vulnerable_at: 0.0.0-20240716161253-dd7900b89e20
25+
packages:
26+
- package: golang.org/x/vulndb/cmd/vulnreport
2227
summary: A problem with golang.org/x/vulndb
2328
description: A description of the issue
2429
review_status: REVIEWED
@@ -46,7 +51,13 @@ review_status: REVIEWED
4651
]
4752
}
4853
],
49-
"ecosystem_specific": {}
54+
"ecosystem_specific": {
55+
"imports": [
56+
{
57+
"path": "golang.org/x/vulndb/cmd/vulnreport"
58+
}
59+
]
60+
}
5061
}
5162
],
5263
"database_specific": {

‎cmd/vulnreport/testdata/TestOSV/ok.txtar

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,13 @@ info: osv: processed 1 report(s) (success=1; skip=0; error=0)
3535
]
3636
}
3737
],
38-
"ecosystem_specific": {}
38+
"ecosystem_specific": {
39+
"imports": [
40+
{
41+
"path": "golang.org/x/vulndb/cmd/vulnreport"
42+
}
43+
]
44+
}
3945
}
4046
],
4147
"database_specific": {

‎cmd/vulnreport/testdata/pkgsite/TestCVE/err.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{}

‎cmd/vulnreport/testdata/pkgsite/TestCVE/ok.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{}

‎cmd/vulnreport/testdata/pkgsite/TestFix/no_change.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"/mod/golang.org/x/vulndb/cmd/vulnreport": true
3+
}

‎cmd/vulnreport/testdata/pkgsite/TestLint/found_lints.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{}

‎cmd/vulnreport/testdata/pkgsite/TestLint/no_lints.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{}

‎cmd/vulnreport/testdata/pkgsite/TestOSV/err.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{}

‎cmd/vulnreport/testdata/pkgsite/TestOSV/ok.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{}

‎cmd/vulnreport/testdata/pkgsite/TestTriage/all.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{}

‎cmd/vulnreport/testdata/pkgsite/TestXref/found_xrefs.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{}

‎cmd/vulnreport/testdata/pkgsite/TestXref/no_xrefs.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{}

‎cmd/vulnreport/testdata/repo.txtar

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@
66
id: GO-9999-0001
77
modules:
88
- module: golang.org/x/vulndb
9+
vulnerable_at: 0.0.0-20240716161253-dd7900b89e20
10+
packages:
11+
- package: golang.org/x/vulndb/cmd/vulnreport
912
summary: A problem with golang.org/x/vulndb
1013
description: A description of the issue
1114
review_status: REVIEWED

‎internal/cveutils/triage_test.go

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,7 @@ var usePkgsite = flag.Bool("pkgsite", false, "use pkg.go.dev for tests")
2424

2525
func TestTriageV4CVE(t *testing.T) {
2626
ctx := context.Background()
27-
cf, err := pkgsite.CacheFile(t)
28-
if err != nil {
29-
t.Fatal(err)
30-
}
31-
pc, err := pkgsite.TestClient(t, *usePkgsite, cf)
27+
pc, err := pkgsite.TestClient(t, *usePkgsite)
3228
if err != nil {
3329
t.Fatal(err)
3430
}
@@ -181,11 +177,7 @@ func TestTriageV4CVE(t *testing.T) {
181177

182178
func TestTriageV5CVE(t *testing.T) {
183179
ctx := context.Background()
184-
cf, err := pkgsite.CacheFile(t)
185-
if err != nil {
186-
t.Fatal(err)
187-
}
188-
pc, err := pkgsite.TestClient(t, *usePkgsite, cf)
180+
pc, err := pkgsite.TestClient(t, *usePkgsite)
189181
if err != nil {
190182
t.Fatal(err)
191183
}

0 commit comments

Comments
 (0)