Skip to content

Commit 818e48f

Browse files
committed
many: cmd/pkgsite embeds static assets
The `pkgsite` command now embeds all the static assets it needs, so it need not be told the location of the static/ directory with a flag. The binary is self-contained and can be placed and invoked from anywhere. This required embedding the static/ and third_party/ directories. Since //go:embed cannot reference files outside the containing package's tree, we had to add trivial Go packages in static/ and third_party/ to construct `embed.FS`s for those directories. Also, the frontend needed to accept `fs.FS` values where it previously took paths, and `template.TrustedFS` values where it previously used `template.TrustedSources`. We ended up clumsily requiring four separate config values: - A TrustedFS to load templates. - Two fs.FSs, one for static and one for third_party, to load other assets. - The path to the static directory as a string, solely to support dynamic loading in dev mode. For golang/go#48410 Change-Id: I9eeb351b1c6f23444b9e65b60f2a1d3905d59ef9 Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/359395 Trust: Jonathan Amsterdam <[email protected]> Run-TryBot: Jonathan Amsterdam <[email protected]> Reviewed-by: Julie Qiu <[email protected]> Reviewed-by: Jamal Carvalho <[email protected]>
1 parent 8256ffb commit 818e48f

23 files changed

+215
-125
lines changed

cmd/frontend/main.go

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ import (
3434
var (
3535
queueName = config.GetEnv("GO_DISCOVERY_FRONTEND_TASK_QUEUE", "")
3636
workers = flag.Int("workers", 10, "number of concurrent requests to the fetch service, when running locally")
37-
_ = flag.String("static", "static", "path to folder containing static files served")
37+
staticFlag = flag.String("static", "static", "path to folder containing static files served")
3838
thirdPartyPath = flag.String("third_party", "third_party", "path to folder containing third-party libraries")
3939
devMode = flag.Bool("dev", false, "enable developer mode (reload templates on each page load, serve non-minified JS/CSS, etc.)")
4040
disableCSP = flag.Bool("nocsp", false, "disable Content Security Policy")
@@ -111,12 +111,15 @@ func main() {
111111
if err != nil {
112112
log.Fatalf(ctx, "vulndbc.NewClient: %v", err)
113113
}
114+
staticSource := template.TrustedSourceFromFlag(flag.Lookup("static").Value)
114115
server, err := frontend.NewServer(frontend.ServerConfig{
115116
DataSourceGetter: dsg,
116117
Queue: fetchQueue,
117118
TaskIDChangeInterval: config.TaskIDChangeIntervalFrontend,
118-
StaticPath: template.TrustedSourceFromFlag(flag.Lookup("static").Value),
119-
ThirdPartyPath: *thirdPartyPath,
119+
TemplateFS: template.TrustedFSFromTrustedSource(staticSource),
120+
StaticFS: os.DirFS(*staticFlag),
121+
StaticPath: *staticFlag,
122+
ThirdPartyFS: os.DirFS(*thirdPartyPath),
120123
DevMode: *devMode,
121124
AppVersionLabel: cfg.AppVersionLabel(),
122125
GoogleTagManagerID: cfg.GoogleTagManagerID,

cmd/pkgsite/main.go

Lines changed: 20 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,29 +6,28 @@
66
// It runs as a web server and presents the documentation as a
77
// web page.
88
//
9-
// After running `go install ./cmd/pkgsite` from the pkgsite repo root, you can
10-
// run `pkgsite` from anywhere, but if you don't run it from the pkgsite repo
11-
// root you must specify the location of the static assets with -static.
9+
// To install, run `go install ./cmd/pkgsite` from the pkgsite repo root.
1210
//
13-
// With just -static, pkgsite will serve docs for the module in the current
11+
// With no arguments, pkgsite will serve docs for the module in the current
1412
// directory, which must have a go.mod file:
1513
//
16-
// cd ~/repos/cue && pkgsite -static ~/repos/pkgsite/static
14+
// cd ~/repos/cue && pkgsite
1715
//
1816
// You can also serve docs from your module cache, directly from the proxy
1917
// (it uses the GOPROXY environment variable), or both:
2018
//
21-
// pkgsite -static ~/repos/pkgsite/static -cache -proxy
19+
// pkgsite -cache -proxy
2220
//
23-
// With either -cache or -proxy, it won't look for a module in the current directory.
24-
// You can still provide modules on the local filesystem by listing their paths:
21+
// With either -cache or -proxy, pkgsite won't look for a module in the current
22+
// directory. You can still provide modules on the local filesystem by listing
23+
// their paths:
2524
//
26-
// pkgsite -static ~/repos/pkgsite/static -cache -proxy ~/repos/cue some/other/module
25+
// pkgsite -cache -proxy ~/repos/cue some/other/module
2726
//
2827
// Although standard library packages will work by default, the docs can take a
2928
// while to appear the first time because the Go repo must be cloned and
3029
// processed. If you clone the repo yourself (https://go.googlesource.com/go),
31-
// provide its location with the -gorepo flag to save a little time.
30+
// you can provide its location with the -gorepo flag to save a little time.
3231
package main
3332

3433
import (
@@ -51,12 +50,14 @@ import (
5150
"golang.org/x/pkgsite/internal/proxy"
5251
"golang.org/x/pkgsite/internal/source"
5352
"golang.org/x/pkgsite/internal/stdlib"
53+
"golang.org/x/pkgsite/static"
54+
thirdparty "golang.org/x/pkgsite/third_party"
5455
)
5556

5657
const defaultAddr = "localhost:8080" // default webserver address
5758

5859
var (
59-
_ = flag.String("static", "static", "path to folder containing static files served")
60+
staticFlag = flag.String("static", "", "OBSOLETE - DO NOT USE")
6061
gopathMode = flag.Bool("gopath_mode", false, "assume that local modules' paths are relative to GOPATH/src")
6162
httpAddr = flag.String("http", defaultAddr, "HTTP service address to listen for incoming requests on")
6263
useCache = flag.Bool("cache", false, "fetch from the module cache")
@@ -76,6 +77,10 @@ func main() {
7677
flag.Parse()
7778
ctx := context.Background()
7879

80+
if *staticFlag != "" {
81+
fmt.Fprintf(os.Stderr, "-static is ignored. It is obsolete and may be removed in a future version.\n")
82+
}
83+
7984
paths := collectPaths(flag.Args())
8085
if len(paths) == 0 && !*useCache && !*useProxy {
8186
paths = []string{"."}
@@ -118,7 +123,7 @@ func main() {
118123

119124
server, err := newServer(ctx, paths, *gopathMode, modCacheDir, prox)
120125
if err != nil {
121-
die("%s\nMaybe you need to provide the location of static assets with -static.", err)
126+
die("%s", err)
122127
}
123128
router := http.NewServeMux()
124129
server.Install(router.Handle, nil, nil)
@@ -160,7 +165,9 @@ func newServer(ctx context.Context, paths []string, gopathMode bool, downloadDir
160165
}.New()
161166
server, err := frontend.NewServer(frontend.ServerConfig{
162167
DataSourceGetter: func(context.Context) internal.DataSource { return lds },
163-
StaticPath: template.TrustedSourceFromFlag(flag.Lookup("static").Value),
168+
TemplateFS: template.TrustedFSFromEmbed(static.FS),
169+
StaticFS: static.FS,
170+
ThirdPartyFS: thirdparty.FS,
164171
})
165172
if err != nil {
166173
return nil, err

cmd/pkgsite/main_test.go

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ package main
66

77
import (
88
"context"
9-
"flag"
109
"net/http"
1110
"net/http/httptest"
1211
"os"
@@ -45,7 +44,6 @@ func Test(t *testing.T) {
4544

4645
localModule := repoPath("internal/fetch/testdata/has_go_mod")
4746
cacheDir := repoPath("internal/fetch/testdata/modcache")
48-
flag.Set("static", repoPath("static"))
4947
testModules := proxytest.LoadTestModules(repoPath("internal/proxy/testdata"))
5048
prox, teardown := proxytest.SetupTestClient(t, testModules)
5149
defer teardown()

go.sum

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -288,7 +288,6 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf
288288
github.com/google/pprof v0.0.0-20200905233945-acf8798be1f7 h1:k+KkMRk8mGOu1xG38StS7dQ+Z6oW1i9n3dgrAVU9Q/E=
289289
github.com/google/pprof v0.0.0-20200905233945-acf8798be1f7/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
290290
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
291-
github.com/google/safehtml v0.0.2 h1:ZOt2VXg4x24bW0m2jtzAOkhoXV0iM8vNKc0paByCZqM=
292291
github.com/google/safehtml v0.0.2/go.mod h1:L4KWwDsUJdECRAEpZoBn3O64bQaywRscowZjJAzjHnU=
293292
github.com/google/safehtml v0.0.3-0.20211026203422-d6f0e11a5516 h1:pSEdbeokt55L2hwtWo6A2k7u5SG08rmw0LhWEyrdWgk=
294293
github.com/google/safehtml v0.0.3-0.20211026203422-d6f0e11a5516/go.mod h1:L4KWwDsUJdECRAEpZoBn3O64bQaywRscowZjJAzjHnU=

internal/fetch/fetch_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,14 +34,14 @@ import (
3434
var testTimeout = 30 * time.Second
3535

3636
var (
37-
templateSource = template.TrustedSourceFromConstant("../../static/doc")
38-
testModules []*proxytest.Module
37+
templateFS = template.TrustedFSFromTrustedSource(template.TrustedSourceFromConstant("../../static"))
38+
testModules []*proxytest.Module
3939
)
4040

4141
type fetchFunc func(t *testing.T, withLicenseDetector bool, ctx context.Context, mod *proxytest.Module, fetchVersion string) (*FetchResult, *licenses.Detector)
4242

4343
func TestMain(m *testing.M) {
44-
dochtml.LoadTemplates(templateSource)
44+
dochtml.LoadTemplates(templateFS)
4545
testModules = proxytest.LoadTestModules("../proxy/testdata")
4646
licenses.OmitExceptions = true
4747
os.Exit(m.Run())

internal/fetchdatasource/fetchdatasource_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ var (
3636
)
3737

3838
func TestMain(m *testing.M) {
39-
dochtml.LoadTemplates(template.TrustedSourceFromConstant("../../static/doc"))
39+
dochtml.LoadTemplates(template.TrustedFSFromTrustedSource(template.TrustedSourceFromConstant("../../static")))
4040
defaultTestModules = proxytest.LoadTestModules("../proxy/testdata")
4141
var cleanup func()
4242
localGetters, cleanup = buildLocalGetters()

internal/frontend/badge.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
package frontend
66

77
import (
8-
"fmt"
98
"net/http"
109
"strings"
1110
)
@@ -23,7 +22,7 @@ type badgePage struct {
2322
func (s *Server) badgeHandler(w http.ResponseWriter, r *http.Request) {
2423
path := strings.TrimPrefix(r.URL.Path, "/badge/")
2524
if path != "" {
26-
http.ServeFile(w, r, fmt.Sprintf("%s/frontend/badge/badge.svg", s.staticPath))
25+
serveFileFS(w, r, s.staticFS, "frontend/badge/badge.svg")
2726
return
2827
}
2928

internal/frontend/server.go

Lines changed: 58 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"io/fs"
1515
"net/http"
1616
"net/url"
17+
"path"
1718
"strings"
1819
"sync"
1920
"time"
@@ -42,10 +43,11 @@ type Server struct {
4243
getDataSource func(context.Context) internal.DataSource
4344
queue queue.Queue
4445
taskIDChangeInterval time.Duration
45-
staticPath template.TrustedSource
46-
thirdPartyPath string
47-
templateDir template.TrustedSource
46+
templateFS template.TrustedFS
47+
staticFS fs.FS
48+
thirdPartyFS fs.FS
4849
devMode bool
50+
staticPath string // used only for dynamic loading in dev mode
4951
errorPage []byte
5052
appVersionLabel string
5153
googleTagManagerID string
@@ -65,9 +67,11 @@ type ServerConfig struct {
6567
DataSourceGetter func(context.Context) internal.DataSource
6668
Queue queue.Queue
6769
TaskIDChangeInterval time.Duration
68-
StaticPath template.TrustedSource
69-
ThirdPartyPath string
70+
TemplateFS template.TrustedFS // for loading templates safely
71+
StaticFS fs.FS // for static/ directory
72+
ThirdPartyFS fs.FS // for third_party/ directory
7073
DevMode bool
74+
StaticPath string // used only for dynamic loading in dev mode
7175
AppVersionLabel string
7276
GoogleTagManagerID string
7377
ServeStats bool
@@ -78,20 +82,19 @@ type ServerConfig struct {
7882
// NewServer creates a new Server for the given database and template directory.
7983
func NewServer(scfg ServerConfig) (_ *Server, err error) {
8084
defer derrors.Wrap(&err, "NewServer(...)")
81-
templateDir := template.TrustedSourceJoin(scfg.StaticPath)
82-
ts, err := parsePageTemplates(templateDir)
85+
ts, err := parsePageTemplates(scfg.TemplateFS)
8386
if err != nil {
8487
return nil, fmt.Errorf("error parsing templates: %v", err)
8588
}
86-
docTemplateDir := template.TrustedSourceJoin(templateDir, template.TrustedSourceFromConstant("doc"))
87-
dochtml.LoadTemplates(docTemplateDir)
89+
dochtml.LoadTemplates(scfg.TemplateFS)
8890
s := &Server{
8991
getDataSource: scfg.DataSourceGetter,
9092
queue: scfg.Queue,
91-
staticPath: scfg.StaticPath,
92-
thirdPartyPath: scfg.ThirdPartyPath,
93-
templateDir: templateDir,
93+
templateFS: scfg.TemplateFS,
94+
staticFS: scfg.StaticFS,
95+
thirdPartyFS: scfg.ThirdPartyFS,
9496
devMode: scfg.DevMode,
97+
staticPath: scfg.StaticPath,
9598
templates: ts,
9699
taskIDChangeInterval: scfg.TaskIDChangeInterval,
97100
appVersionLabel: scfg.AppVersionLabel,
@@ -134,10 +137,11 @@ func (s *Server) Install(handle func(string, http.Handler), redisClient *redis.C
134137
log.Infof(r.Context(), "Request made to %q", r.URL.Path)
135138
}))
136139
handle("/static/", s.staticHandler())
137-
handle("/third_party/", http.StripPrefix("/third_party", http.FileServer(http.Dir(s.thirdPartyPath))))
140+
handle("/third_party/", http.StripPrefix("/third_party", http.FileServer(http.FS(s.thirdPartyFS))))
138141
handle("/favicon.ico", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
139-
http.ServeFile(w, r, fmt.Sprintf("%s/shared/icon/favicon.ico", http.Dir(s.staticPath.String())))
142+
serveFileFS(w, r, s.staticFS, "shared/icon/favicon.ico")
140143
}))
144+
141145
handle("/sitemap/", http.StripPrefix("/sitemap/", http.FileServer(http.Dir("private/sitemap"))))
142146
handle("/mod/", http.HandlerFunc(s.handleModuleDetailsRedirect))
143147
handle("/pkg/", http.HandlerFunc(s.handlePackageDetailsRedirect))
@@ -539,7 +543,7 @@ func (s *Server) findTemplate(templateName string) (*template.Template, error) {
539543
s.mu.Lock()
540544
defer s.mu.Unlock()
541545
var err error
542-
s.templates, err = parsePageTemplates(s.templateDir)
546+
s.templates, err = parsePageTemplates(s.templateFS)
543547
if err != nil {
544548
return nil, fmt.Errorf("error parsing templates: %v", err)
545549
}
@@ -584,69 +588,73 @@ func stripScheme(url string) string {
584588
return url
585589
}
586590

587-
// parsePageTemplates parses html templates contained in the given base
588-
// directory in order to generate a map of Name->*template.Template.
591+
// parsePageTemplates parses html templates contained in the given filesystem in
592+
// order to generate a map of Name->*template.Template.
589593
//
590594
// Separate templates are used so that certain contextual functions (e.g.
591595
// templateName) can be bound independently for each page.
592596
//
593597
// Templates in directories prefixed with an underscore are considered helper
594598
// templates and parsed together with the files in each base directory.
595-
func parsePageTemplates(base template.TrustedSource) (map[string]*template.Template, error) {
596-
tsc := template.TrustedSourceFromConstant
597-
join := template.TrustedSourceJoin
598-
599+
func parsePageTemplates(fsys template.TrustedFS) (map[string]*template.Template, error) {
599600
templates := make(map[string]*template.Template)
600-
htmlSets := [][]template.TrustedSource{
601-
{tsc("badge")},
602-
{tsc("error")},
603-
{tsc("fetch")},
604-
{tsc("homepage")},
605-
{tsc("legacy_search")},
606-
{tsc("license-policy")},
607-
{tsc("search")},
608-
{tsc("search-help")},
609-
{tsc("styleguide")},
610-
{tsc("subrepo")},
611-
{tsc("unit/importedby"), tsc("unit")},
612-
{tsc("unit/imports"), tsc("unit")},
613-
{tsc("unit/licenses"), tsc("unit")},
614-
{tsc("unit/main"), tsc("unit")},
615-
{tsc("unit/versions"), tsc("unit")},
601+
htmlSets := [][]string{
602+
{"badge"},
603+
{"error"},
604+
{"fetch"},
605+
{"homepage"},
606+
{"legacy_search"},
607+
{"license-policy"},
608+
{"search"},
609+
{"search-help"},
610+
{"styleguide"},
611+
{"subrepo"},
612+
{"unit/importedby", "unit"},
613+
{"unit/imports", "unit"},
614+
{"unit/licenses", "unit"},
615+
{"unit/main", "unit"},
616+
{"unit/versions", "unit"},
616617
}
617618

618619
for _, set := range htmlSets {
619-
t, err := template.New("frontend.tmpl").Funcs(templateFuncs).ParseGlobFromTrustedSource(join(base, tsc("frontend/*.tmpl")))
620+
t, err := template.New("frontend.tmpl").Funcs(templateFuncs).ParseFS(fsys, "frontend/*.tmpl")
620621
if err != nil {
621-
return nil, fmt.Errorf("ParseFilesFromTrustedSources: %v", err)
622+
return nil, fmt.Errorf("ParseFS: %v", err)
622623
}
623-
helperGlob := join(base, tsc("shared/*/*.tmpl"))
624-
if _, err := t.ParseGlobFromTrustedSource(helperGlob); err != nil {
625-
return nil, fmt.Errorf("ParseGlobFromTrustedSource(%q): %v", helperGlob, err)
624+
helperGlob := "shared/*/*.tmpl"
625+
if _, err := t.ParseFS(fsys, helperGlob); err != nil {
626+
return nil, fmt.Errorf("ParseFS(%q): %v", helperGlob, err)
626627
}
627-
var files []template.TrustedSource
628628
for _, f := range set {
629-
if _, err := t.ParseGlobFromTrustedSource(join(base, tsc("frontend"), f, tsc("*.tmpl"))); err != nil {
630-
return nil, fmt.Errorf("ParseGlobFromTrustedSource(%v): %v", files, err)
629+
if _, err := t.ParseFS(fsys, path.Join("frontend", f, "*.tmpl")); err != nil {
630+
return nil, fmt.Errorf("ParseFS(%v): %v", f, err)
631631
}
632632
}
633-
templates[set[0].String()] = t
633+
templates[set[0]] = t
634634
}
635635

636636
return templates, nil
637637
}
638638

639639
func (s *Server) staticHandler() http.Handler {
640-
staticPath := s.staticPath.String()
641-
642640
// In dev mode compile TypeScript files into minified JavaScript files
643641
// and rebuild them on file changes.
644642
if s.devMode {
643+
if s.staticPath == "" {
644+
panic("staticPath is empty in dev mode; cannot rebuild static files")
645+
}
645646
ctx := context.Background()
646-
_, err := static.Build(static.Config{EntryPoint: staticPath + "/frontend", Watch: true, Bundle: true})
647+
_, err := static.Build(static.Config{EntryPoint: s.staticPath + "/frontend", Watch: true, Bundle: true})
647648
if err != nil {
648649
log.Error(ctx, err)
649650
}
650651
}
651-
return http.StripPrefix("/static/", http.FileServer(http.Dir(staticPath)))
652+
return http.StripPrefix("/static/", http.FileServer(http.FS(s.staticFS)))
653+
}
654+
655+
// serveFileFS serves a file from the given filesystem.
656+
func serveFileFS(w http.ResponseWriter, r *http.Request, fsys fs.FS, name string) {
657+
fs := http.FileServer(http.FS(fsys))
658+
r.URL.Path = name
659+
fs.ServeHTTP(w, r)
652660
}

0 commit comments

Comments
 (0)