cmd/api: omit outside dependencies when listing the packages in "std"

As of CL 251159, when 'go list -deps std' is run within GOROOT/src, it
treats the vendored external dependencies as real module dependencies,
not standard-library "vendor/" packages (which still exist in that
case, but are treated as distinct packages outside the "std" module).

Fixes #41358
Updates #30241

Change-Id: Ic23eae9829d90e74a340d49ca9052e9191597410
Reviewed-on: https://go-review.googlesource.com/c/go/+/254738
Run-TryBot: Bryan C. Mills <[email protected]>
Trust: Bryan C. Mills <[email protected]>
TryBot-Result: Go Bot <[email protected]>
Reviewed-by: Michael Matloob <[email protected]>
Reviewed-by: Jay Conrod <[email protected]>

Author: Bryan C. Mills

src/cmd/api/goapi.go

@@ -87,7 +87,10 @@ var contexts = []*build.Context{
 func contextName(c *build.Context) string {
 	s := c.GOOS + "-" + c.GOARCH
 	if c.CgoEnabled {
-		return s + "-cgo"
+		s += "-cgo"
+	}
+	if c.Dir != "" {
+		s += fmt.Sprintf(" [%s]", c.Dir)
 	}
 	return s
 }
@@ -478,6 +481,9 @@ func (w *Walker) loadImports() {
 
 		cmd := exec.Command(goCmd(), "list", "-e", "-deps", "-json", "std")
 		cmd.Env = listEnv(w.context)
+		if w.context.Dir != "" {
+			cmd.Dir = w.context.Dir
+		}
 		out, err := cmd.CombinedOutput()
 		if err != nil {
 			log.Fatalf("loading imports: %v\n%s", err, out)
@@ -491,6 +497,7 @@ func (w *Walker) loadImports() {
 			var pkg struct {
 				ImportPath, Dir string
 				ImportMap       map[string]string
+				Standard        bool
 			}
 			err := dec.Decode(&pkg)
 			if err == io.EOF {
@@ -503,11 +510,13 @@ func (w *Walker) loadImports() {
 			// - Package "unsafe" contains special signatures requiring
 			//   extra care when printing them - ignore since it is not
 			//   going to change w/o a language change.
-			// - internal and vendored packages do not contribute to our
-			//   API surface.
+			// - Internal and vendored packages do not contribute to our
+			//   API surface. (If we are running within the "std" module,
+			//   vendored dependencies appear as themselves instead of
+			//   their "vendor/" standard-library copies.)
 			// - 'go list std' does not include commands, which cannot be
 			//   imported anyway.
-			if ip := pkg.ImportPath; ip != "unsafe" && !strings.HasPrefix(ip, "vendor/") && !internalPkg.MatchString(ip) {
+			if ip := pkg.ImportPath; pkg.Standard && ip != "unsafe" && !strings.HasPrefix(ip, "vendor/") && !internalPkg.MatchString(ip) {
 				stdPackages = append(stdPackages, ip)
 			}
 			importDir[pkg.ImportPath] = pkg.Dir

src/cmd/api/goapi_test.go

@@ -216,3 +216,16 @@ func TestIssue29837(t *testing.T) {
 		}
 	}
 }
+
+func TestIssue41358(t *testing.T) {
+	context := new(build.Context)
+	*context = build.Default
+	context.Dir = filepath.Join(context.GOROOT, "src")
+
+	w := NewWalker(context, context.Dir)
+	for _, pkg := range w.stdPackages {
+		if strings.HasPrefix(pkg, "vendor/") || strings.HasPrefix(pkg, "golang.org/x/") {
+			t.Fatalf("stdPackages contains unexpected package %s", pkg)
+		}
+	}
+}

src/cmd/go/internal/modload/load.go

@@ -1106,8 +1106,12 @@ func (ld *loader) stdVendor(parentPath, path string) string {
 		// Do the same for importers beginning with the prefix 'vendor/' even if we
 		// are *inside* of the 'std' module: the 'vendor/' packages that resolve
 		// globally from GOROOT/src/vendor (and are listed as part of 'go list std')
-		// are distinct from the real module dependencies, and cannot import internal
-		// packages from the real module.
+		// are distinct from the real module dependencies, and cannot import
+		// internal packages from the real module.
+		//
+		// (Note that although the 'vendor/' packages match the 'std' *package*
+		// pattern, they are not part of the std *module*, and do not affect
+		// 'go mod tidy' and similar module commands when working within std.)
 		vendorPath := pathpkg.Join("vendor", path)
 		if _, err := os.Stat(filepath.Join(cfg.GOROOTsrc, filepath.FromSlash(vendorPath))); err == nil {
 			return vendorPath