internal/postgres: exclude internal packages when doing shortcut resolution

pkgsite allows users to use a shortcut when looking up std packages. For
example, pkg.go.dev/http will resolve to pkg.go.dev/net/http.

Currently, this shortcut resolution also works for internal packages
(e.g. pkg.go.dev/boring will resolve to
pkg.go.dev/crypto/internal/boring), even though Go users would likely
not care about internal packages.

An unfortunate side-effect of this is that our shortcut resolution does
not work for packages which has a similarly-named internal package:
pkg.go.dev/synctest does not resolve to pkg.go.dev/testing/synctest as
the existence of pkg.go.dev/internal/synctest makes the resolution
ambiguous.

This change updates our shortcut resolution logic to exclude all
internal packages.

For golang/go#76136

Change-Id: I83fcf32861079d051bc8dc3be29092480d53f667
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/717840
LUCI-TryBot-Result: Go LUCI <[email protected]>
Reviewed-by: Jonathan Amsterdam <[email protected]>
kokoro-CI: kokoro <[email protected]>
Reviewed-by: Nicholas Husin <[email protected]>

Author: nicholashusin

internal/frontend/fetchserver/server_test.go

@@ -161,6 +161,14 @@ var testModules = []testModule{
 				name:   "http",
 				suffix: "net/http",
 			},
+			{
+				name:   "boring",
+				suffix: "crypto/internal/boring",
+			},
+			{
+				name:   "asan",
+				suffix: "internal/asan",
+			},
 		},
 	},
 	{
@@ -686,6 +694,16 @@ func serverTestCases() []serverTestCase {
 			wantStatusCode: http.StatusMovedPermanently,
 			wantLocation:   "/[email protected]",
 		},
+		{
+			name:           "stdlib top-level internal shortcut (internal/asan)",
+			urlPath:        "/asan",
+			wantStatusCode: http.StatusNotFound,
+		},
+		{
+			name:           "stdlib internal shortcut (crypto/internal/boring)",
+			urlPath:        "/boring",
+			wantStatusCode: http.StatusNotFound,
+		},
 		{
 			name:           "package page with trailiing slash",
 			urlPath:        "/github.com/my/module/",

internal/postgres/stdlib.go

@@ -17,7 +17,8 @@ import (
 // the path must end with "/" + suffix.
 //
 // We are only interested in actual standard library packages: not commands, which we happen to include
-// in the stdlib module, and not directories (paths that do not contain a package).
+// in the stdlib module; not directories (paths that do not contain a package); and not internal
+// packages.
 func (db *DB) GetStdlibPathsWithSuffix(ctx context.Context, suffix string) (paths []string, err error) {
 	defer derrors.WrapStack(&err, "DB.GetStdlibPaths(ctx, %q)", suffix)
 
@@ -37,6 +38,8 @@ func (db *DB) GetStdlibPathsWithSuffix(ctx context.Context, suffix string) (path
 			LIMIT 1)
 			AND u.name != ''
 			AND p.path NOT LIKE 'cmd/%'
+			AND p.path NOT LIKE 'internal/%'
+			AND p.path NOT LIKE '%/internal/%'
 			AND p.path LIKE '%/' || $2
 		ORDER BY p.path
 	`