Skip to content

Commit a9ff35d

Browse files
committed
internal/source: match old templates to find new ones
Previously, when we encountered a repo whose URL doesn't match an existing pattern, we did not generate any URL templates for it, meaning we could not render source links in the documentation. This CL uses the templates in the go-source meta tag to guess the version-aware templates that are likely to work for the repo. For golang/go#40477 Change-Id: I2d1978da5a6de1af19284233dbab9ac1ae2cb582 Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/285312 Trust: Jonathan Amsterdam <[email protected]> Run-TryBot: Jonathan Amsterdam <[email protected]> Reviewed-by: Julie Qiu <[email protected]>
1 parent 7099521 commit a9ff35d

File tree

3 files changed

+2509
-1111
lines changed

3 files changed

+2509
-1111
lines changed

internal/source/source.go

Lines changed: 93 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -329,7 +329,7 @@ func newStdlibInfo(version string) (_ *Info, err error) {
329329
// example.com/a/b.git/c
330330
// then repo="example.com/a/b" and relativeModulePath="c"; the ".git" is omitted, since it is neither
331331
// part of the repo nor part of the relative path to the module within the repo.
332-
func matchStatic(moduleOrRepoPath string) (repo, relativeModulePath string, _ urlTemplates, transformCommit func(string, bool) string, _ error) {
332+
func matchStatic(moduleOrRepoPath string) (repo, relativeModulePath string, _ urlTemplates, transformCommit transformCommitFunc, _ error) {
333333
for _, pat := range patterns {
334334
matches := pat.re.FindStringSubmatch(moduleOrRepoPath)
335335
if matches == nil {
@@ -395,7 +395,12 @@ func moduleInfoDynamic(ctx context.Context, client *Client, modulePath, version
395395
var repo string
396396
repo, _, templates, transformCommit, _ = matchStatic(removeHTTPScheme(sourceMeta.dirTemplate))
397397
if templates == (urlTemplates{}) {
398-
log.Infof(ctx, "no templates for repo URL %q from meta tag: err=%v", sourceMeta.repoURL, err)
398+
if err == nil {
399+
templates, transformCommit = matchLegacyTemplates(ctx, sourceMeta)
400+
repoURL = strings.TrimSuffix(repoURL, ".git")
401+
} else {
402+
log.Infof(ctx, "no templates for repo URL %q from meta tag: err=%v", sourceMeta.repoURL, err)
403+
}
399404
} else {
400405
// Use the repo from the template, not the original one.
401406
repoURL = "https://" + repo
@@ -414,6 +419,63 @@ func moduleInfoDynamic(ctx context.Context, client *Client, modulePath, version
414419
}, nil
415420
}
416421

422+
// List of template regexps and their corresponding likely templates,
423+
// used by matchLegacyTemplates below.
424+
var legacyTemplateMatches = []struct {
425+
fileRegexp *regexp.Regexp
426+
templates urlTemplates
427+
transformCommit transformCommitFunc
428+
}{
429+
{
430+
regexp.MustCompile(`/src/branch/\w+\{/dir\}/\{file\}#L\{line\}$`),
431+
giteaURLTemplates, giteaTransformCommit,
432+
},
433+
{
434+
regexp.MustCompile(`/src/\w+\{/dir\}/\{file\}#L\{line\}$`),
435+
giteaURLTemplates, nil,
436+
},
437+
{
438+
regexp.MustCompile(`/-/blob/\w+\{/dir\}/\{file\}#L\{line\}$`),
439+
gitlab2URLTemplates, nil,
440+
},
441+
{
442+
regexp.MustCompile(`/tree\{/dir\}/\{file\}#n\{line\}$`),
443+
fdioURLTemplates, fdioTransformCommit,
444+
},
445+
}
446+
447+
// matchLegacyTemplates matches the templates from the go-source meta tag
448+
// against some known patterns to guess the version-aware URL templates. If it
449+
// can't find a match, it falls back using the go-source templates with some
450+
// small replacements. These will not be version-aware but will still serve
451+
// source at a fixed commit, which is better than nothing.
452+
func matchLegacyTemplates(ctx context.Context, sm *sourceMeta) (_ urlTemplates, transformCommit transformCommitFunc) {
453+
if sm.fileTemplate == "" {
454+
return urlTemplates{}, nil
455+
}
456+
for _, ltm := range legacyTemplateMatches {
457+
if ltm.fileRegexp.MatchString(sm.fileTemplate) {
458+
return ltm.templates, ltm.transformCommit
459+
}
460+
}
461+
log.Infof(ctx, "matchLegacyTemplates: no matches for repo URL %q; replacing", sm.repoURL)
462+
rep := strings.NewReplacer(
463+
"{/dir}/{file}", "/{file}",
464+
"{dir}/{file}", "{file}",
465+
"{/dir}", "/{dir}")
466+
line := rep.Replace(sm.fileTemplate)
467+
file := line
468+
if i := strings.LastIndexByte(line, '#'); i > 0 {
469+
file = line[:i]
470+
}
471+
return urlTemplates{
472+
Repo: sm.repoURL,
473+
Directory: rep.Replace(sm.dirTemplate),
474+
File: file,
475+
Line: line,
476+
}, nil
477+
}
478+
417479
// adjustVersionedModuleDirectory changes info.moduleDir if necessary to
418480
// correctly reflect the repo structure. info.moduleDir will be wrong if it has
419481
// a suffix "/vN" for N > 1, and the repo uses the "major branch" convention,
@@ -463,6 +525,8 @@ func removeVersionSuffix(s string) string {
463525
return strings.TrimSuffix(dir, "/")
464526
}
465527

528+
type transformCommitFunc func(commit string, isHash bool) string
529+
466530
// Patterns for determining repo and URL templates from module paths or repo
467531
// URLs. Each regexp must match a prefix of the target string, and must have a
468532
// group named "repo".
@@ -471,7 +535,7 @@ var patterns = []struct {
471535
templates urlTemplates
472536
re *regexp.Regexp
473537
// transformCommit may alter the commit before substitution
474-
transformCommit func(commit string, isHash bool) string
538+
transformCommit transformCommitFunc
475539
}{
476540
{
477541
pattern: `^(?P<repo>github\.com/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)`,
@@ -504,30 +568,13 @@ var patterns = []struct {
504568
},
505569
},
506570
{
507-
pattern: `^(?P<repo>git\.fd\.io/[a-z0-9A-Z_.\-]+)`,
508-
templates: urlTemplates{
509-
Directory: "{repo}/tree/{dir}?{commit}",
510-
File: "{repo}/tree/{file}?{commit}",
511-
Line: "{repo}/tree/{file}?{commit}#n{line}",
512-
Raw: "{repo}/plain/{file}?{commit}",
513-
},
514-
transformCommit: func(commit string, isHash bool) string {
515-
// hashes use "?id=", tags use "?h="
516-
p := "h"
517-
if isHash {
518-
p = "id"
519-
}
520-
return fmt.Sprintf("%s=%s", p, commit)
521-
},
571+
pattern: `^(?P<repo>git\.fd\.io/[a-z0-9A-Z_.\-]+)`,
572+
templates: fdioURLTemplates,
573+
transformCommit: fdioTransformCommit,
522574
},
523575
{
524-
pattern: `^(?P<repo>git\.pirl\.io/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)`,
525-
templates: urlTemplates{
526-
Directory: "{repo}/-/tree/{commit}/{dir}",
527-
File: "{repo}/-/blob/{commit}/{file}",
528-
Line: "{repo}/-/blob/{commit}/{file}#L{line}",
529-
Raw: "{repo}/-/raw/{commit}/{file}",
530-
},
576+
pattern: `^(?P<repo>git\.pirl\.io/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)`,
577+
templates: gitlab2URLTemplates,
531578
},
532579
{
533580
pattern: `^(?P<repo>gitea\.com/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)(\.git|$)`,
@@ -617,6 +664,15 @@ func giteaTransformCommit(commit string, isHash bool) string {
617664
return "tag/" + commit
618665
}
619666

667+
func fdioTransformCommit(commit string, isHash bool) string {
668+
// hashes use "?id=", tags use "?h="
669+
p := "h"
670+
if isHash {
671+
p = "id"
672+
}
673+
return fmt.Sprintf("%s=%s", p, commit)
674+
}
675+
620676
// urlTemplates describes how to build URLs from bits of source information.
621677
// The fields are exported for JSON encoding.
622678
//
@@ -664,6 +720,18 @@ var (
664720
Line: "{repo}/+/{commit}/{file}#{line}",
665721
// Gitiles has no support for serving raw content at this time.
666722
}
723+
gitlab2URLTemplates = urlTemplates{
724+
Directory: "{repo}/-/tree/{commit}/{dir}",
725+
File: "{repo}/-/blob/{commit}/{file}",
726+
Line: "{repo}/-/blob/{commit}/{file}#L{line}",
727+
Raw: "{repo}/-/raw/{commit}/{file}",
728+
}
729+
fdioURLTemplates = urlTemplates{
730+
Directory: "{repo}/tree/{dir}?{commit}",
731+
File: "{repo}/tree/{file}?{commit}",
732+
Line: "{repo}/tree/{file}?{commit}#n{line}",
733+
Raw: "{repo}/plain/{file}?{commit}",
734+
}
667735
)
668736

669737
// commitFromVersion returns a string that refers to a commit corresponding to version.

internal/source/source_test.go

Lines changed: 97 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -353,6 +353,42 @@ func TestModuleInfo(t *testing.T) {
353353
"https://gotools.org/dmitri.shuralyov.com/gpu/mtl/example/movingtriangle/internal/coreanim?rev=d42048ed14fd#coreanim.go-L1",
354354
"",
355355
},
356+
{
357+
"go-source templates match gitea with transform",
358+
"opendev.org/airship/airshipctl", "v2.0.0-beta.1", "pkg/cluster/command.go",
359+
"https://opendev.org/airship/airshipctl",
360+
"https://opendev.org/airship/airshipctl/src/tag/v2.0.0-beta.1",
361+
"https://opendev.org/airship/airshipctl/src/tag/v2.0.0-beta.1/pkg/cluster/command.go",
362+
"https://opendev.org/airship/airshipctl/src/tag/v2.0.0-beta.1/pkg/cluster/command.go#L1",
363+
"",
364+
},
365+
{
366+
"go-source templates match gitea without transform",
367+
"git.borago.de/Marco/gqltest", "v0.0.18", "go.mod",
368+
"https://git.borago.de/Marco/gqltest",
369+
"https://git.borago.de/Marco/gqltest/src/v0.0.18",
370+
"https://git.borago.de/Marco/gqltest/src/v0.0.18/go.mod",
371+
"https://git.borago.de/Marco/gqltest/src/v0.0.18/go.mod#L1",
372+
"https://git.borago.de/Marco/gqltest/raw/v0.0.18/go.mod",
373+
},
374+
{
375+
"go-source templates match gitlab2",
376+
"git.pluggableideas.com/destrealm/3rdparty/go-yaml", "v2.2.6", "go.mod",
377+
"https://git.pluggableideas.com/destrealm/3rdparty/go-yaml",
378+
"https://git.pluggableideas.com/destrealm/3rdparty/go-yaml/-/tree/v2.2.6",
379+
"https://git.pluggableideas.com/destrealm/3rdparty/go-yaml/-/blob/v2.2.6/go.mod",
380+
"https://git.pluggableideas.com/destrealm/3rdparty/go-yaml/-/blob/v2.2.6/go.mod#L1",
381+
"https://git.pluggableideas.com/destrealm/3rdparty/go-yaml/-/raw/v2.2.6/go.mod",
382+
},
383+
{
384+
"go-source templates match fdio",
385+
"golang.zx2c4.com/wireguard/windows", "v0.3.4", "go.mod",
386+
"https://git.zx2c4.com/wireguard-windows",
387+
"https://git.zx2c4.com/wireguard-windows/tree/?h=v0.3.4",
388+
"https://git.zx2c4.com/wireguard-windows/tree/go.mod?h=v0.3.4",
389+
"https://git.zx2c4.com/wireguard-windows/tree/go.mod?h=v0.3.4#n1",
390+
"https://git.zx2c4.com/wireguard-windows/plain/go.mod?h=v0.3.4",
391+
},
356392
} {
357393
t.Run(test.desc, func(t *testing.T) {
358394
info, err := ModuleInfo(context.Background(), &Client{client}, test.modulePath, test.version)
@@ -484,12 +520,17 @@ func TestModuleInfoDynamic(t *testing.T) {
484520
},
485521
{
486522
"alice.org/pkg/source",
487-
// Has a go-source tag, but we can't use the templates.
523+
// Has a go-source tag; we try to use the templates.
488524
&Info{
489525
repoURL: "http://alice.org/pkg",
490526
moduleDir: "source",
491527
commit: "source/v1.2.3",
492-
// empty templates
528+
templates: urlTemplates{
529+
Repo: "http://alice.org/pkg",
530+
Directory: "http://alice.org/pkg/{dir}",
531+
File: "http://alice.org/pkg/{dir}?f={file}",
532+
Line: "http://alice.org/pkg/{dir}?f={file}#Line{line}",
533+
},
493534
},
494535
},
495536

@@ -500,7 +541,12 @@ func TestModuleInfoDynamic(t *testing.T) {
500541
repoURL: "http://alice.org/pkg",
501542
moduleDir: "ignore",
502543
commit: "ignore/v1.2.3",
503-
// empty templates
544+
templates: urlTemplates{
545+
Repo: "http://alice.org/pkg",
546+
Directory: "http://alice.org/pkg/{dir}",
547+
File: "http://alice.org/pkg/{dir}?f={file}",
548+
Line: "http://alice.org/pkg/{dir}?f={file}#Line{line}",
549+
},
504550
},
505551
},
506552
{"alice.org/pkg/multiple", nil},
@@ -510,7 +556,7 @@ func TestModuleInfoDynamic(t *testing.T) {
510556
&Info{
511557
// The go-import tag's repo root ends in ".git", but according to the spec
512558
// there should not be a .vcs suffix, so we include the ".git" in the repo URL.
513-
repoURL: "https://vcs.net/bob/pkg.git",
559+
repoURL: "https://vcs.net/bob/pkg",
514560
moduleDir: "",
515561
commit: "v1.2.3",
516562
// empty templates
@@ -519,7 +565,7 @@ func TestModuleInfoDynamic(t *testing.T) {
519565
{
520566
"bob.com/pkg/sub",
521567
&Info{
522-
repoURL: "https://vcs.net/bob/pkg.git",
568+
repoURL: "https://vcs.net/bob/pkg",
523569
moduleDir: "sub",
524570
commit: "sub/v1.2.3",
525571
// empty templates
@@ -907,3 +953,49 @@ func TestURLTemplates(t *testing.T) {
907953
check(p.templates.Raw, "commit", "file")
908954
}
909955
}
956+
957+
func TestMatchLegacyTemplates(t *testing.T) {
958+
for _, test := range []struct {
959+
sm sourceMeta
960+
wantTemplates urlTemplates
961+
wantTransformCommitNil bool
962+
}{
963+
{
964+
sm: sourceMeta{"", "", "", "https://git.blindage.org/21h/hcloud-dns/src/branch/master{/dir}/{file}#L{line}"},
965+
wantTemplates: giteaURLTemplates,
966+
wantTransformCommitNil: false,
967+
},
968+
{
969+
sm: sourceMeta{"", "", "", "https://git.lastassault.de/sup/networkoverlap/-/blob/master{/dir}/{file}#L{line}"},
970+
wantTemplates: gitlab2URLTemplates,
971+
wantTransformCommitNil: true,
972+
},
973+
{
974+
sm: sourceMeta{"", "", "", "https://git.borago.de/Marco/gqltest/src/master{/dir}/{file}#L{line}"},
975+
wantTemplates: giteaURLTemplates,
976+
wantTransformCommitNil: true,
977+
},
978+
{
979+
sm: sourceMeta{"", "", "", "https://git.zx2c4.com/wireguard-windows/tree{/dir}/{file}#n{line}"},
980+
wantTemplates: fdioURLTemplates,
981+
wantTransformCommitNil: false,
982+
},
983+
{
984+
sm: sourceMeta{"", "", "unknown{/dir}", "unknown{/dir}/{file}#L{line}"},
985+
wantTemplates: urlTemplates{
986+
Repo: "",
987+
Directory: "unknown/{dir}",
988+
File: "unknown/{file}",
989+
Line: "unknown/{file}#L{line}",
990+
},
991+
wantTransformCommitNil: true,
992+
},
993+
} {
994+
gotTemplates, gotTransformCommit := matchLegacyTemplates(context.Background(), &test.sm)
995+
gotNil := gotTransformCommit == nil
996+
if gotTemplates != test.wantTemplates || gotNil != test.wantTransformCommitNil {
997+
t.Errorf("%+v:\ngot (%+v, %t)\nwant (%+v, %t)",
998+
test.sm, gotTemplates, gotNil, test.wantTemplates, test.wantTransformCommitNil)
999+
}
1000+
}
1001+
}

0 commit comments

Comments
 (0)