Skip to content

Commit b317742

Browse files
julieqiujulieqiu
committed
cmd/pkgsite: add command
Functionality for running the pkgsite frontend locally is moved from cmd/frontend to cmd/pkgsite, since cmd/frontend is currently overloaded with flag options and running locally does not need all the dependencies for running cmd/frontend. Additional functionality will be added to cmd/pkgsite in future CLs. For golang/go#40371 Change-Id: I4230aa9539c94e01a68eda33cc6492ae377debff Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/290134 Reviewed-by: Jamal Carvalho <[email protected]> Trust: Julie Qiu <[email protected]>
1 parent e10f26f commit b317742

File tree

3 files changed

+138
-73
lines changed

3 files changed

+138
-73
lines changed

‎cmd/frontend/main.go

+27-65
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ import (
1010
"flag"
1111
"net/http"
1212
"os"
13-
"path/filepath"
1413
"time"
1514

1615
"cloud.google.com/go/profiler"
@@ -21,7 +20,6 @@ import (
2120
"golang.org/x/pkgsite/internal/config"
2221
"golang.org/x/pkgsite/internal/dcensus"
2322
"golang.org/x/pkgsite/internal/frontend"
24-
"golang.org/x/pkgsite/internal/localdatasource"
2523
"golang.org/x/pkgsite/internal/log"
2624
"golang.org/x/pkgsite/internal/middleware"
2725
"golang.org/x/pkgsite/internal/postgres"
@@ -42,8 +40,6 @@ var (
4240
"for direct proxy mode and frontend fetches")
4341
directProxy = flag.Bool("direct_proxy", false, "if set to true, uses the module proxy referred to by this URL "+
4442
"as a direct backend, bypassing the database")
45-
localPaths = flag.String("local", "", "run locally, accepts a GOPATH-like collection of local paths for modules to load to memory")
46-
gopathMode = flag.Bool("gopath_mode", false, "assume that local modules' paths are relative to GOPATH/src, used only with -local")
4743
bypassLicenseCheck = flag.Bool("bypass_license_check", false, "display all information, even for non-redistributable paths")
4844
)
4945

@@ -75,41 +71,36 @@ func main() {
7571
expg := cmdconfig.ExperimentGetter(ctx, cfg)
7672
log.Infof(ctx, "cmd/frontend: initialized cmdconfig.ExperimentGetter")
7773

78-
if *localPaths != "" {
79-
lds := localdatasource.New()
80-
dsg = func(context.Context) internal.DataSource { return lds }
74+
proxyClient, err := proxy.New(*proxyURL)
75+
if err != nil {
76+
log.Fatal(ctx, err)
77+
}
78+
79+
if *directProxy {
80+
var pds *proxydatasource.DataSource
81+
if *bypassLicenseCheck {
82+
pds = proxydatasource.NewBypassingLicenseCheck(proxyClient)
83+
} else {
84+
pds = proxydatasource.New(proxyClient)
85+
}
86+
dsg = func(context.Context) internal.DataSource { return pds }
8187
} else {
82-
proxyClient, err := proxy.New(*proxyURL)
88+
db, err := cmdconfig.OpenDB(ctx, cfg, *bypassLicenseCheck)
8389
if err != nil {
84-
log.Fatal(ctx, err)
90+
log.Fatalf(ctx, "%v", err)
8591
}
86-
87-
if *directProxy {
88-
var pds *proxydatasource.DataSource
89-
if *bypassLicenseCheck {
90-
pds = proxydatasource.NewBypassingLicenseCheck(proxyClient)
91-
} else {
92-
pds = proxydatasource.New(proxyClient)
93-
}
94-
dsg = func(context.Context) internal.DataSource { return pds }
95-
} else {
96-
db, err := cmdconfig.OpenDB(ctx, cfg, *bypassLicenseCheck)
97-
if err != nil {
98-
log.Fatalf(ctx, "%v", err)
99-
}
100-
defer db.Close()
101-
dsg = func(context.Context) internal.DataSource { return db }
102-
sourceClient := source.NewClient(config.SourceTimeout)
103-
// The closure passed to queue.New is only used for testing and local
104-
// execution, not in production. So it's okay that it doesn't use a
105-
// per-request connection.
106-
fetchQueue, err = queue.New(ctx, cfg, queueName, *workers, expg,
107-
func(ctx context.Context, modulePath, version string) (int, error) {
108-
return frontend.FetchAndUpdateState(ctx, modulePath, version, proxyClient, sourceClient, db)
109-
})
110-
if err != nil {
111-
log.Fatalf(ctx, "queue.New: %v", err)
112-
}
92+
defer db.Close()
93+
dsg = func(context.Context) internal.DataSource { return db }
94+
sourceClient := source.NewClient(config.SourceTimeout)
95+
// The closure passed to queue.New is only used for testing and local
96+
// execution, not in production. So it's okay that it doesn't use a
97+
// per-request connection.
98+
fetchQueue, err = queue.New(ctx, cfg, queueName, *workers, expg,
99+
func(ctx context.Context, modulePath, version string) (int, error) {
100+
return frontend.FetchAndUpdateState(ctx, modulePath, version, proxyClient, sourceClient, db)
101+
})
102+
if err != nil {
103+
log.Fatalf(ctx, "queue.New: %v", err)
113104
}
114105
}
115106

@@ -135,13 +126,6 @@ func main() {
135126
log.Fatalf(ctx, "frontend.NewServer: %v", err)
136127
}
137128

138-
if *localPaths != "" {
139-
lds, ok := dsg(ctx).(*localdatasource.DataSource)
140-
if ok {
141-
load(ctx, lds, *localPaths)
142-
}
143-
}
144-
145129
router := dcensus.NewRouter(frontend.TagRoute)
146130
var cacheClient *redis.Client
147131
if cfg.RedisCacheHost != "" {
@@ -204,25 +188,3 @@ func main() {
204188
log.Infof(ctx, "Listening on addr %s", addr)
205189
log.Fatal(ctx, http.ListenAndServe(addr, mw(router)))
206190
}
207-
208-
// load loads local modules from pathList.
209-
func load(ctx context.Context, ds *localdatasource.DataSource, pathList string) {
210-
paths := filepath.SplitList(pathList)
211-
loaded := len(paths)
212-
for _, path := range paths {
213-
var err error
214-
if *gopathMode {
215-
err = ds.LoadFromGOPATH(ctx, path)
216-
} else {
217-
err = ds.Load(ctx, path)
218-
}
219-
if err != nil {
220-
log.Error(ctx, err)
221-
loaded--
222-
}
223-
}
224-
225-
if loaded == 0 {
226-
log.Fatalf(ctx, "failed to load module(s) at %s", pathList)
227-
}
228-
}

‎cmd/pkgsite/main.go

+99
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
// Copyright 2021 The Go Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
// This is a work in progress.
6+
//
7+
// Pkgsite extracts and generates documentation for Go programs.
8+
// It runs as a web server and presents the documentation as a
9+
// web page.
10+
// Usage:
11+
//
12+
// pkgsite [flag]
13+
//
14+
// The flags are:
15+
//
16+
// -local=path1,path2
17+
// Accepts a GOPATH-like collection of local paths for modules to load to memory
18+
// -gopath_mode=false
19+
// Assume that local modules' paths are relative to GOPATH/src
20+
package main
21+
22+
import (
23+
"context"
24+
"flag"
25+
"net/http"
26+
"path/filepath"
27+
"time"
28+
29+
"github.com/google/safehtml/template"
30+
"golang.org/x/pkgsite/internal"
31+
"golang.org/x/pkgsite/internal/dcensus"
32+
"golang.org/x/pkgsite/internal/frontend"
33+
"golang.org/x/pkgsite/internal/localdatasource"
34+
"golang.org/x/pkgsite/internal/log"
35+
"golang.org/x/pkgsite/internal/middleware"
36+
)
37+
38+
var (
39+
_ = flag.String("static", "content/static", "path to folder containing static files served")
40+
localPaths = flag.String("local", "", "run locally, accepts a GOPATH-like collection of local paths for modules to load to memory")
41+
gopathMode = flag.Bool("gopath_mode", false, "assume that local modules' paths are relative to GOPATH/src, used only with -local")
42+
)
43+
44+
func main() {
45+
flag.Parse()
46+
ctx := context.Background()
47+
var dsg func(context.Context) internal.DataSource
48+
if *localPaths == "" {
49+
log.Fatalf(ctx, "-local is not set")
50+
}
51+
52+
lds := localdatasource.New()
53+
dsg = func(context.Context) internal.DataSource { return lds }
54+
server, err := frontend.NewServer(frontend.ServerConfig{
55+
DataSourceGetter: dsg,
56+
StaticPath: template.TrustedSourceFromFlag(flag.Lookup("static").Value),
57+
})
58+
if err != nil {
59+
log.Fatalf(ctx, "frontend.NewServer: %v", err)
60+
}
61+
lds, ok := dsg(ctx).(*localdatasource.DataSource)
62+
if ok {
63+
load(ctx, lds, *localPaths)
64+
}
65+
66+
router := dcensus.NewRouter(frontend.TagRoute)
67+
server.Install(router.Handle, nil, nil)
68+
69+
mw := middleware.Chain(
70+
middleware.RedirectedFrom(),
71+
middleware.LatestVersions(server.GetLatestInfo), // must come before caching for version badge to work
72+
middleware.Timeout(54*time.Second),
73+
)
74+
addr := "localhost:6060"
75+
log.Infof(ctx, "Listening on addr %s", addr)
76+
log.Fatal(ctx, http.ListenAndServe(addr, mw(router)))
77+
}
78+
79+
// load loads local modules from pathList.
80+
func load(ctx context.Context, ds *localdatasource.DataSource, pathList string) {
81+
paths := filepath.SplitList(pathList)
82+
loaded := len(paths)
83+
for _, path := range paths {
84+
var err error
85+
if *gopathMode {
86+
err = ds.LoadFromGOPATH(ctx, path)
87+
} else {
88+
err = ds.Load(ctx, path)
89+
}
90+
if err != nil {
91+
log.Error(ctx, err)
92+
loaded--
93+
}
94+
}
95+
96+
if loaded == 0 {
97+
log.Fatalf(ctx, "failed to load module(s) at %s", pathList)
98+
}
99+
}

‎doc/frontend.md

+12-8
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,14 @@ See [experiment.md](experiment.md) for instructions how to enable experiments.
1616

1717
You can run the frontend locally like so:
1818

19-
go run ./cmd/frontend [-dev] [-direct_proxy] [-local .]
19+
go run ./cmd/frontend [-dev] [-direct_proxy]
2020

2121
- The `-dev` flag reloads templates on each page load.
2222

2323
The frontend can use one of three datasources:
2424

2525
- Postgres database
2626
- Proxy service
27-
- Local filesystem
2827

2928
The `Datasource` interface implementation is available at internal/datasource.go.
3029

@@ -39,16 +38,21 @@ your local database with packages of your choice.
3938

4039
You can then run the frontend with: `go run ./cmd/frontend`
4140

42-
You can also use `-local` flag to run the frontend with an in-memory datasource
43-
populated with modules loaded from your local filesystem. This allows you to run
44-
the frontend without setting up a database and to view documentation of local
45-
modules without requiring a proxy. `-local` accepts a GOPATH-like string containing
46-
paths of modules to load into memory.
47-
4841
If you add, change or remove any inline scripts in templates, run
4942
`devtools/cmd/csphash` to update the hashes. Running `all.bash`
5043
will do that as well.
5144

45+
### Local mode
46+
47+
You can also use run the frontend locally with an in-memory datasource
48+
populated with modules loaded from your local filesystem.
49+
50+
go run ./cmd/pkgsite [-local .]
51+
52+
This allows you to run the frontend without setting up a database and to view
53+
documentation of local modules without requiring a proxy. `-local` accepts a
54+
GOPATH-like string containing paths of modules to load into memory.
55+
5256
### Testing
5357

5458
In addition to tests inside internal/frontend and internal/testing/integration,

0 commit comments

Comments
 (0)