feat: fetch Maven metadata from specified repositories

cmd/osv-scanner/__snapshots__/main_test.snap

@@ -2289,6 +2289,42 @@ No issues found
 
 ---
 
+[TestRun_MavenTransitive/resolve_transitive_dependencies_with_native_datda_source - 1]
+Scanned <rootdir>/fixtures/maven-transitive/registry.xml file as a pom.xml and found 13 packages
++-------------------------------------+------+-----------+-------------------------------------+---------+----------------------------------------+
+| OSV URL                             | CVSS | ECOSYSTEM | PACKAGE                             | VERSION | SOURCE                                 |
++-------------------------------------+------+-----------+-------------------------------------+---------+----------------------------------------+
+| https://osv.dev/GHSA-735f-pc8j-v9w8 | 8.7  | Maven     | com.google.protobuf:protobuf-java   | 4.28.0  | fixtures/maven-transitive/registry.xml |
+| https://osv.dev/GHSA-7rjr-3q55-vv33 | 9.0  | Maven     | org.apache.logging.log4j:log4j-core | 2.14.1  | fixtures/maven-transitive/registry.xml |
+| https://osv.dev/GHSA-8489-44mv-ggj8 | 6.6  | Maven     | org.apache.logging.log4j:log4j-core | 2.14.1  | fixtures/maven-transitive/registry.xml |
+| https://osv.dev/GHSA-jfh8-c2jp-5v3q | 10.0 | Maven     | org.apache.logging.log4j:log4j-core | 2.14.1  | fixtures/maven-transitive/registry.xml |
+| https://osv.dev/GHSA-p6xc-xr62-6r2g | 8.6  | Maven     | org.apache.logging.log4j:log4j-core | 2.14.1  | fixtures/maven-transitive/registry.xml |
++-------------------------------------+------+-----------+-------------------------------------+---------+----------------------------------------+
+
+---
+
+[TestRun_MavenTransitive/resolve_transitive_dependencies_with_native_datda_source - 2]
+
+---
+
+[TestRun_MavenTransitive/scans_dependencies_from_multiple_registries - 1]
+Scanned <rootdir>/fixtures/maven-transitive/registry.xml file as a pom.xml and found 13 packages
++-------------------------------------+------+-----------+-------------------------------------+---------+----------------------------------------+
+| OSV URL                             | CVSS | ECOSYSTEM | PACKAGE                             | VERSION | SOURCE                                 |
++-------------------------------------+------+-----------+-------------------------------------+---------+----------------------------------------+
+| https://osv.dev/GHSA-735f-pc8j-v9w8 | 8.7  | Maven     | com.google.protobuf:protobuf-java   | 4.28.0  | fixtures/maven-transitive/registry.xml |
+| https://osv.dev/GHSA-7rjr-3q55-vv33 | 9.0  | Maven     | org.apache.logging.log4j:log4j-core | 2.14.1  | fixtures/maven-transitive/registry.xml |
+| https://osv.dev/GHSA-8489-44mv-ggj8 | 6.6  | Maven     | org.apache.logging.log4j:log4j-core | 2.14.1  | fixtures/maven-transitive/registry.xml |
+| https://osv.dev/GHSA-jfh8-c2jp-5v3q | 10.0 | Maven     | org.apache.logging.log4j:log4j-core | 2.14.1  | fixtures/maven-transitive/registry.xml |
+| https://osv.dev/GHSA-p6xc-xr62-6r2g | 8.6  | Maven     | org.apache.logging.log4j:log4j-core | 2.14.1  | fixtures/maven-transitive/registry.xml |
++-------------------------------------+------+-----------+-------------------------------------+---------+----------------------------------------+
+
+---
+
+[TestRun_MavenTransitive/scans_dependencies_from_multiple_registries - 2]
+
+---
+
 [TestRun_MavenTransitive/scans_transitive_dependencies_by_specifying_pom.xml - 1]
 Scanned <rootdir>/fixtures/maven-transitive/abc.xml file as a pom.xml and found 3 packages
 +-------------------------------------+------+-----------+-------------------------------------+---------+-----------------------------------+

cmd/osv-scanner/fix/main.go

@@ -13,7 +13,6 @@ import (
 	"github.com/google/osv-scanner/internal/remediation/upgrade"
 	"github.com/google/osv-scanner/internal/resolution"
 	"github.com/google/osv-scanner/internal/resolution/client"
-	"github.com/google/osv-scanner/internal/resolution/datasource"
 	"github.com/google/osv-scanner/internal/resolution/lockfile"
 	"github.com/google/osv-scanner/internal/resolution/manifest"
 	"github.com/google/osv-scanner/pkg/depsdev"
@@ -68,6 +67,10 @@ func Command(stdout, stderr io.Writer, r *reporter.Reporter) *cli.Command {
 					return nil
 				},
 			},
+			&cli.StringFlag{
+				Name:  "maven-registry",
+				Usage: "URL of the default Maven registry to fetch metadata",
+			},
 			&cli.StringFlag{
 				Name:  "relock-cmd",
 				Usage: "command to run to regenerate lockfile on disk after changing the manifest",
@@ -269,7 +272,7 @@ func action(ctx *cli.Context, stdout, stderr io.Writer) (reporter.Reporter, erro
 	}
 
 	if opts.Manifest != "" {
-		rw, err := manifest.GetReadWriter(opts.Manifest)
+		rw, err := manifest.GetReadWriter(opts.Manifest, ctx.String("maven-registry"))
 		if err != nil {
 			return nil, err
 		}
@@ -302,7 +305,7 @@ func action(ctx *cli.Context, stdout, stderr io.Writer) (reporter.Reporter, erro
 			}
 			opts.Client.DependencyClient = cl
 		case resolve.Maven:
-			cl, err := client.NewMavenRegistryClient(datasource.MavenCentral)
+			cl, err := client.NewMavenRegistryClient(ctx.String("maven-registry"))
 			if err != nil {
 				return nil, err
 			}

cmd/osv-scanner/fix/noninteractive.go

@@ -295,6 +295,17 @@ func autoOverride(ctx context.Context, r reporter.Reporter, opts osvFixOptions,
 		return err
 	}
 
+	if opts.ManifestRW.System() == resolve.Maven {
+		// Update Maven registries based on the pom.xml
+		specific, ok := manif.EcosystemSpecific.(manifest.MavenManifestSpecific)
+		if ok {
+			registries := make([]string, len(specific.Repositories))
+			for i, repo := range specific.Repositories {
+				registries[i] = string(repo.URL)
+			}
+			opts.Client.DependencyClient.UpdateRegistries(registries)
+		}
+	}
 	client.PreFetch(ctx, opts.Client, manif.Requirements, manif.FilePath)
 	res, err := resolution.Resolve(ctx, opts.Client, manif, opts.ResolveOpts)
 	if err != nil {

cmd/osv-scanner/fixtures/maven-transitive/parent.xml

@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>com.mycompany.app</groupId>
+  <artifactId>parent</artifactId>
+  <version>1.0.0</version>
+
+  <name>my-app</name>
+
+  <packaging>pom</packaging>
+
+  <dependencies>
+    <dependency>
+      <groupId>com.google.protobuf</groupId>
+      <artifactId>protobuf-java-util</artifactId>
+      <version>4.28.0</version>
+    </dependency>
+  </dependencies>
+
+  <repositories>
+    <repository>
+      <id>google-andriod</id>
+      <url>https://dl.google.com/dl/android/maven2</url>
+    </repository>
+  </repositories>
+
+</project>

cmd/osv-scanner/fixtures/maven-transitive/registry.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>com.mycompany.app</groupId>
+  <artifactId>my-app</artifactId>
+  <version>1.0.0</version>
+
+  <name>my-app</name>
+
+  <parent>
+    <groupId>com.mycompany.app</groupId>
+    <artifactId>parent</artifactId>
+    <version>1.0.0</version>
+    <relativePath>./parent.xml</relativePath>
+  </parent>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-web</artifactId>
+      <version>2.14.1</version>
+    </dependency>
+  </dependencies>
+
+</project>

cmd/osv-scanner/main_test.go

@@ -911,6 +911,16 @@ func TestRun_MavenTransitive(t *testing.T) {
 			args: []string{"", "--config=./fixtures/osv-scanner-empty-config.toml", "--experimental-offline", "--experimental-download-offline-databases", "./fixtures/maven-transitive/pom.xml"},
 			exit: 0,
 		},
+		{
+			name: "scans dependencies from multiple registries",
+			args: []string{"", "--config=./fixtures/osv-scanner-empty-config.toml", "-L", "pom.xml:./fixtures/maven-transitive/registry.xml"},
+			exit: 1,
+		},
+		{
+			name: "resolve transitive dependencies with native datda source",
+			args: []string{"", "--config=./fixtures/osv-scanner-empty-config.toml", "--experimental-resolution-data-source=native", "-L", "pom.xml:./fixtures/maven-transitive/registry.xml"},
+			exit: 1,
+		},
 	}
 
 	for _, tt := range tests {

cmd/osv-scanner/scan/main.go

@@ -132,6 +132,22 @@ func Command(stdout, stderr io.Writer, r *reporter.Reporter) *cli.Command {
 				TakesFile: true,
 				Hidden:    true,
 			},
+			&cli.StringFlag{
+				Name:  "experimental-resolution-data-source",
+				Usage: "source to fetch package information from; value can be: deps.dev, native",
+				Value: "deps.dev",
+				Action: func(_ *cli.Context, s string) error {
+					if s != "deps.dev" && s != "native" {
+						return fmt.Errorf("unsupported data-source \"%s\" - must be one of: deps.dev, native", s)
+					}
+
+					return nil
+				},
+			},
+			&cli.StringFlag{
+				Name:  "experimental-maven-registry",
+				Usage: "URL of the default registry to fetch Maven metadata",
+			},
 		},
 		ArgsUsage: "[directory1 directory2...]",
 		Action: func(c *cli.Context) error {
@@ -222,6 +238,10 @@ func action(context *cli.Context, stdout, stderr io.Writer) (reporter.Reporter,
 			ScanLicensesSummary:   context.Bool("experimental-licenses-summary"),
 			ScanLicensesAllowlist: context.StringSlice("experimental-licenses"),
 			ScanOCIImage:          context.String("experimental-oci-image"),
+			TransitiveScanningActions: osvscanner.TransitiveScanningActions{
+				NativeDataSource: context.String("experimental-resolution-data-source") == "native",
+				MavenRegistry:    context.String("experimental-maven-registry"),
+			},
 		},
 	}, r)
 

cmd/osv-scanner/update/main.go

@@ -78,7 +78,7 @@ func action(ctx *cli.Context, stdout, stderr io.Writer) (reporter.Reporter, erro
 	if err != nil {
 		return nil, err
 	}
-	options.ManifestRW, err = manifest.GetReadWriter(options.Manifest)
+	options.ManifestRW, err = manifest.GetReadWriter(options.Manifest, "")
 	if err != nil {
 		return nil, err
 	}

internal/manifest/maven.go

@@ -32,11 +32,18 @@ func (e MavenResolverExtractor) Extract(f lockfile.DepFile) ([]lockfile.PackageD
 
 	var project maven.Project
 	if err := xml.NewDecoder(f).Decode(&project); err != nil {
-		return []lockfile.PackageDetails{}, fmt.Errorf("could not extract from %s: %w", f.Path(), err)
+		return nil, fmt.Errorf("could not extract from %s: %w", f.Path(), err)
+	}
+	// Empty JDK and ActivationOS indicates merging the default profiles.
+	if err := project.MergeProfiles("", maven.ActivationOS{}); err != nil {
+		return nil, fmt.Errorf("failed to merge profiles: %w", err)
+	}
+	for _, repo := range project.Repositories {
+		e.MavenRegistryAPIClient.Add(string(repo.URL))
 	}
 	// Merging parents data by parsing local parent pom.xml or fetching from upstream.
 	if err := mavenutil.MergeParents(ctx, e.MavenRegistryAPIClient, &project, project.Parent, 1, f.Path(), true); err != nil {
-		return []lockfile.PackageDetails{}, fmt.Errorf("failed to merge parents: %w", err)
+		return nil, fmt.Errorf("failed to merge parents: %w", err)
 	}
 	// Process the dependencies:
 	//  - dedupe dependencies and dependency management
@@ -52,6 +59,7 @@ func (e MavenResolverExtractor) Extract(f lockfile.DepFile) ([]lockfile.PackageD
 		return result.DependencyManagement, nil
 	})
 
+	e.DependencyClient.UpdateRegistries(e.MavenRegistryAPIClient.GetRegistries())
 	overrideClient := client.NewOverrideClient(e.DependencyClient)
 	resolver := mavenresolve.NewResolver(overrideClient)
 
@@ -96,7 +104,7 @@ func (e MavenResolverExtractor) Extract(f lockfile.DepFile) ([]lockfile.PackageD
 
 	g, err := resolver.Resolve(ctx, root.VersionKey)
 	if err != nil {
-		return []lockfile.PackageDetails{}, fmt.Errorf("failed resolving %v: %w", root, err)
+		return nil, fmt.Errorf("failed resolving %v: %w", root, err)
 	}
 	for i, e := range g.Edges {
 		e.Type = dep.Type{}
@@ -128,7 +136,7 @@ func (e MavenResolverExtractor) Extract(f lockfile.DepFile) ([]lockfile.PackageD
 func ParseMavenWithResolver(depClient client.DependencyClient, mavenClient *datasource.MavenRegistryAPIClient, pathToLockfile string) ([]lockfile.PackageDetails, error) {
 	f, err := lockfile.OpenLocalDepFile(pathToLockfile)
 	if err != nil {
-		return []lockfile.PackageDetails{}, err
+		return nil, err
 	}
 	defer f.Close()
 

internal/manifest/maven_test.go

@@ -92,7 +92,8 @@ func TestParseMavenWithResolver_InvalidSyntax(t *testing.T) {
 func TestParseMavenWithResolver_NoPackages(t *testing.T) {
 	t.Parallel()
 
-	packages, err := manifest.ParseMavenWithResolver(nil, nil, "fixtures/maven/empty.xml")
+	resolutionClient := clienttest.NewMockResolutionClient(t, "fixtures/universe/basic-universe.yaml")
+	packages, err := manifest.ParseMavenWithResolver(resolutionClient, &datasource.MavenRegistryAPIClient{}, "fixtures/maven/empty.xml")
 	if err != nil {
 		t.Errorf("Got unexpected error: %v", err)
 	}
@@ -104,7 +105,7 @@ func TestParseMavenWithResolver_OnePackage(t *testing.T) {
 	t.Parallel()
 
 	resolutionClient := clienttest.NewMockResolutionClient(t, "fixtures/universe/basic-universe.yaml")
-	packages, err := manifest.ParseMavenWithResolver(resolutionClient, nil, "fixtures/maven/one-package.xml")
+	packages, err := manifest.ParseMavenWithResolver(resolutionClient, &datasource.MavenRegistryAPIClient{}, "fixtures/maven/one-package.xml")
 	if err != nil {
 		t.Errorf("Got unexpected error: %v", err)
 	}
@@ -123,7 +124,7 @@ func TestParseMavenWithResolver_TwoPackages(t *testing.T) {
 	t.Parallel()
 
 	resolutionClient := clienttest.NewMockResolutionClient(t, "fixtures/universe/basic-universe.yaml")
-	packages, err := manifest.ParseMavenWithResolver(resolutionClient, nil, "fixtures/maven/two-packages.xml")
+	packages, err := manifest.ParseMavenWithResolver(resolutionClient, &datasource.MavenRegistryAPIClient{}, "fixtures/maven/two-packages.xml")
 	if err != nil {
 		t.Errorf("Got unexpected error: %v", err)
 	}
@@ -148,7 +149,7 @@ func TestParseMavenWithResolver_WithDependencyManagement(t *testing.T) {
 	t.Parallel()
 
 	resolutionClient := clienttest.NewMockResolutionClient(t, "fixtures/universe/basic-universe.yaml")
-	packages, err := manifest.ParseMavenWithResolver(resolutionClient, nil, "fixtures/maven/with-dependency-management.xml")
+	packages, err := manifest.ParseMavenWithResolver(resolutionClient, &datasource.MavenRegistryAPIClient{}, "fixtures/maven/with-dependency-management.xml")
 	if err != nil {
 		t.Errorf("Got unexpected error: %v", err)
 	}
@@ -173,7 +174,7 @@ func TestParseMavenWithResolver_Interpolation(t *testing.T) {
 	t.Parallel()
 
 	resolutionClient := clienttest.NewMockResolutionClient(t, "fixtures/universe/basic-universe.yaml")
-	packages, err := manifest.ParseMavenWithResolver(resolutionClient, nil, "fixtures/maven/interpolation.xml")
+	packages, err := manifest.ParseMavenWithResolver(resolutionClient, &datasource.MavenRegistryAPIClient{}, "fixtures/maven/interpolation.xml")
 	if err != nil {
 		t.Errorf("Got unexpected error: %v", err)
 	}
@@ -204,7 +205,7 @@ func TestParseMavenWithResolver_WithScope(t *testing.T) {
 	t.Parallel()
 
 	resolutionClient := clienttest.NewMockResolutionClient(t, "fixtures/universe/basic-universe.yaml")
-	packages, err := manifest.ParseMavenWithResolver(resolutionClient, nil, "fixtures/maven/with-scope.xml")
+	packages, err := manifest.ParseMavenWithResolver(resolutionClient, &datasource.MavenRegistryAPIClient{}, "fixtures/maven/with-scope.xml")
 	if err != nil {
 		t.Errorf("Got unexpected error: %v", err)
 	}
@@ -258,7 +259,8 @@ func TestParseMavenWithResolver_WithParent(t *testing.T) {
 	`))
 
 	resolutionClient := clienttest.NewMockResolutionClient(t, "fixtures/universe/basic-universe.yaml")
-	packages, err := manifest.ParseMavenWithResolver(resolutionClient, datasource.NewMavenRegistryAPIClient(srv.URL), "fixtures/maven/with-parent.xml")
+	client, _ := datasource.NewMavenRegistryAPIClient(srv.URL)
+	packages, err := manifest.ParseMavenWithResolver(resolutionClient, client, "fixtures/maven/with-parent.xml")
 	if err != nil {
 		t.Errorf("Got unexpected error: %v", err)
 	}
@@ -307,7 +309,7 @@ func TestParseMavenWithResolver_Transitive(t *testing.T) {
 	t.Parallel()
 
 	resolutionClient := clienttest.NewMockResolutionClient(t, "fixtures/universe/basic-universe.yaml")
-	packages, err := manifest.ParseMavenWithResolver(resolutionClient, nil, "fixtures/maven/transitive.xml")
+	packages, err := manifest.ParseMavenWithResolver(resolutionClient, &datasource.MavenRegistryAPIClient{}, "fixtures/maven/transitive.xml")
 	if err != nil {
 		t.Errorf("Got unexpected error: %v", err)
 	}

internal/remediation/testhelpers_test.go

@@ -19,7 +19,7 @@ import (
 func parseRemediationFixture(t *testing.T, universePath, manifestPath string, opts resolution.ResolveOpts) (*resolution.Result, client.ResolutionClient) {
 	t.Helper()
 
-	rw, err := manifest.GetReadWriter(manifestPath)
+	rw, err := manifest.GetReadWriter(manifestPath, "")
 	if err != nil {
 		t.Fatalf("Failed to get ReadWriter: %v", err)
 	}

internal/resolution/client/client.go

@@ -30,6 +30,8 @@ type DependencyClient interface {
 	WriteCache(filepath string) error
 	// LoadCache loads a manifest-specific resolution cache.
 	LoadCache(filepath string) error
+	// UpdateRegistries updates the registries to fetch data.
+	UpdateRegistries(registries []string)
 }
 
 // PreFetch loads cache, then makes and caches likely queries needed for resolving a package with a list of requirements

internal/resolution/client/depsdev_client.go

@@ -25,6 +25,8 @@ func NewDepsDevClient(addr string) (*DepsDevClient, error) {
 	return &DepsDevClient{APIClient: *resolve.NewAPIClient(c), c: c}, nil
 }
 
+func (d *DepsDevClient) UpdateRegistries(_ []string) {}
+
 func (d *DepsDevClient) WriteCache(path string) error {
 	f, err := os.Create(path + depsDevCacheExt)
 	if err != nil {