From 671592bdcff920c1c64ca339921740f419ee6aa5 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 21 Mar 2022 10:12:29 +0800 Subject: [PATCH 01/13] Use a new vars template implementation --- models/migrations/migrations_test.go | 3 +- modules/cache/cache_redis.go | 8 +- modules/context/csrf.go | 3 +- modules/context/repo.go | 12 ++- modules/markup/html.go | 11 ++- modules/repository/init.go | 9 ++- modules/sync/unique_queue.go | 10 +-- modules/templates/vars/vars.go | 106 +++++++++++++++++++++++++++ modules/templates/vars/vars_test.go | 70 ++++++++++++++++++ modules/util/util.go | 7 ++ routers/web/goget.go | 22 +++++- routers/web/repo/issue.go | 10 ++- 12 files changed, 243 insertions(+), 28 deletions(-) create mode 100644 modules/templates/vars/vars.go create mode 100644 modules/templates/vars/vars_test.go diff --git a/models/migrations/migrations_test.go b/models/migrations/migrations_test.go index f798d501179f7..a17eba54e82be 100644 --- a/models/migrations/migrations_test.go +++ b/models/migrations/migrations_test.go @@ -24,7 +24,6 @@ import ( "code.gitea.io/gitea/modules/util" "github.com/stretchr/testify/assert" - "github.com/unknwon/com" "xorm.io/xorm" "xorm.io/xorm/names" ) @@ -204,7 +203,7 @@ func prepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En deferFn := PrintCurrentTest(t, ourSkip) assert.NoError(t, os.RemoveAll(setting.RepoRootPath)) - assert.NoError(t, com.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), + assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath)) ownerDirs, err := os.ReadDir(setting.RepoRootPath) if err != nil { diff --git a/modules/cache/cache_redis.go b/modules/cache/cache_redis.go index 148725ae66015..e4b9a70f63f3a 100644 --- a/modules/cache/cache_redis.go +++ b/modules/cache/cache_redis.go @@ -10,10 +10,10 @@ import ( "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/nosql" + "code.gitea.io/gitea/modules/util" "gitea.com/go-chi/cache" "github.com/go-redis/redis/v8" - "github.com/unknwon/com" ) // RedisCacher represents a redis cache adapter implementation. @@ -29,15 +29,15 @@ type RedisCacher struct { func (c *RedisCacher) Put(key string, val interface{}, expire int64) error { key = c.prefix + key if expire == 0 { - if err := c.c.Set(graceful.GetManager().HammerContext(), key, com.ToStr(val), 0).Err(); err != nil { + if err := c.c.Set(graceful.GetManager().HammerContext(), key, util.ToStr(val), 0).Err(); err != nil { return err } } else { - dur, err := time.ParseDuration(com.ToStr(expire) + "s") + dur, err := time.ParseDuration(util.ToStr(expire) + "s") if err != nil { return err } - if err = c.c.Set(graceful.GetManager().HammerContext(), key, com.ToStr(val), dur).Err(); err != nil { + if err = c.c.Set(graceful.GetManager().HammerContext(), key, util.ToStr(val), dur).Err(); err != nil { return err } } diff --git a/modules/context/csrf.go b/modules/context/csrf.go index 99c223c884da3..f7c9205b72c5e 100644 --- a/modules/context/csrf.go +++ b/modules/context/csrf.go @@ -23,6 +23,7 @@ import ( "time" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web/middleware" "github.com/unknwon/com" @@ -211,7 +212,7 @@ func Csrfer(opt CsrfOptions, ctx *Context) CSRF { x.ID = "0" uid := ctx.Session.Get(opt.SessionKey) if uid != nil { - x.ID = com.ToStr(uid) + x.ID = util.ToStr(uid) } needsNew := false diff --git a/modules/context/repo.go b/modules/context/repo.go index e55c13f49cce5..24f0dc0fa6a25 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -26,11 +26,11 @@ import ( "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/templates/vars" "code.gitea.io/gitea/modules/util" asymkey_service "code.gitea.io/gitea/services/asymkey" "github.com/editorconfig/editorconfig-core-go/v2" - "github.com/unknwon/com" ) // IssueTemplateDirCandidates issue templates directory @@ -309,11 +309,17 @@ func EarlyResponseForGoGetMeta(ctx *Context) { ctx.PlainText(http.StatusBadRequest, "invalid repository path") return } - ctx.PlainText(http.StatusOK, com.Expand(``, + res, err := vars.Expand(``, map[string]string{ "GoGetImport": ComposeGoGetImport(username, reponame), "CloneLink": repo_model.ComposeHTTPSCloneURL(username, reponame), - })) + }) + if err != nil { + log.Error(err.Error()) + ctx.PlainText(http.StatusInternalServerError, "expand vars failed") + return + } + ctx.PlainText(http.StatusOK, res) } // RedirectToRepo redirect to a differently-named repository diff --git a/modules/markup/html.go b/modules/markup/html.go index 87d05dcabbe2a..1769f76b2a26f 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -21,9 +21,9 @@ import ( "code.gitea.io/gitea/modules/markup/common" "code.gitea.io/gitea/modules/references" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/templates/vars" "code.gitea.io/gitea/modules/util" - "github.com/unknwon/com" "golang.org/x/net/html" "golang.org/x/net/html/atom" "mvdan.cc/xurls/v2" @@ -837,7 +837,14 @@ func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) { reftext := node.Data[ref.RefLocation.Start:ref.RefLocation.End] if exttrack && !ref.IsPull { ctx.Metas["index"] = ref.Issue - link = createLink(com.Expand(ctx.Metas["format"], ctx.Metas), reftext, "ref-issue ref-external-issue") + + res, err := vars.Expand(ctx.Metas["format"], ctx.Metas) + if err != nil { + log.Error(err.Error()) + return + } + + link = createLink(res, reftext, "ref-issue ref-external-issue") } else { // Path determines the type of link that will be rendered. It's unknown at this point whether // the linked item is actually a PR or an issue. Luckily it's of no real consequence because diff --git a/modules/repository/init.go b/modules/repository/init.go index 66d464ef137e3..abf4b9317283b 100644 --- a/modules/repository/init.go +++ b/modules/repository/init.go @@ -19,10 +19,9 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/templates/vars" "code.gitea.io/gitea/modules/util" asymkey_service "code.gitea.io/gitea/services/asymkey" - - "github.com/unknwon/com" ) func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir, repoPath string, opts models.CreateRepoOptions) error { @@ -61,8 +60,12 @@ func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir, "CloneURL.HTTPS": cloneLink.HTTPS, "OwnerName": repo.OwnerName, } + res, err := vars.Expand(string(data), match) + if err != nil { + return fmt.Errorf("expand README.md: %v", err) + } if err = os.WriteFile(filepath.Join(tmpDir, "README.md"), - []byte(com.Expand(string(data), match)), 0o644); err != nil { + []byte(res), 0o644); err != nil { return fmt.Errorf("write README.md: %v", err) } diff --git a/modules/sync/unique_queue.go b/modules/sync/unique_queue.go index d41726b5af930..414cc50f39a76 100644 --- a/modules/sync/unique_queue.go +++ b/modules/sync/unique_queue.go @@ -5,9 +5,7 @@ package sync -import ( - "github.com/unknwon/com" -) +import "code.gitea.io/gitea/modules/util" // UniqueQueue is a queue which guarantees only one instance of same // identity is in the line. Instances with same identity will be @@ -73,13 +71,13 @@ func (q *UniqueQueue) Queue() <-chan string { // Exist returns true if there is an instance with given identity // exists in the queue. func (q *UniqueQueue) Exist(id interface{}) bool { - return q.table.IsRunning(com.ToStr(id)) + return q.table.IsRunning(util.ToStr(id)) } // AddFunc adds new instance to the queue with a custom runnable function, // the queue is blocked until the function exits. func (q *UniqueQueue) AddFunc(id interface{}, fn func()) { - idStr := com.ToStr(id) + idStr := util.ToStr(id) q.table.lock.Lock() if _, ok := q.table.pool[idStr]; ok { q.table.lock.Unlock() @@ -105,5 +103,5 @@ func (q *UniqueQueue) Add(id interface{}) { // Remove removes instance from the queue. func (q *UniqueQueue) Remove(id interface{}) { - q.table.Stop(com.ToStr(id)) + q.table.Stop(util.ToStr(id)) } diff --git a/modules/templates/vars/vars.go b/modules/templates/vars/vars.go new file mode 100644 index 0000000000000..152819c32e553 --- /dev/null +++ b/modules/templates/vars/vars.go @@ -0,0 +1,106 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package vars + +import ( + "fmt" + "strings" +) + +// ErrWrongSyntax represents a wrong syntax with a tempate +type ErrWrongSyntax struct { + Template string +} + +func (err ErrWrongSyntax) Error() string { + return fmt.Sprintf("Wrong syntax found in %s", err.Template) +} + +// IsErrWrongSyntax returns true if the error is ErrWrongSyntax +func IsErrWrongSyntax(err error) bool { + _, ok := err.(ErrWrongSyntax) + return ok +} + +// ErrNoMatchedVar represents an error that no matched vars +type ErrNoMatchedVar struct { + Template string + Var string +} + +func (err ErrNoMatchedVar) Error() string { + return fmt.Sprintf("No matched variable %s found for %s", err.Var, err.Template) +} + +// IsErrNoMatchedVar returns true if the error is ErrNoMatchedVar +func IsErrNoMatchedVar(err error) bool { + _, ok := err.(ErrNoMatchedVar) + return ok +} + +// Expand replaces all variables like {var} to match +func Expand(template string, match map[string]string, subs ...string) (string, error) { + var ( + buf strings.Builder + key strings.Builder + enter bool + ) + for _, c := range template { + switch { + case c == '{': + if enter { + return "", ErrWrongSyntax{ + Template: template, + } + } + enter = true + case c == '}': + if !enter { + return "", ErrWrongSyntax{ + Template: template, + } + } + if key.Len() == 0 { + return "", ErrWrongSyntax{ + Template: template, + } + } + + if len(match) == 0 { + return "", ErrNoMatchedVar{ + Template: template, + Var: key.String(), + } + } + + v, ok := match[key.String()] + if !ok { + if len(subs) == 0 { + return "", ErrNoMatchedVar{ + Template: template, + Var: key.String(), + } + } + v = subs[0] + } + + if _, err := buf.WriteString(v); err != nil { + return "", err + } + key.Reset() + + enter = false + case enter: + if _, err := key.WriteRune(c); err != nil { + return "", err + } + default: + if _, err := buf.WriteRune(c); err != nil { + return "", err + } + } + } + return buf.String(), nil +} diff --git a/modules/templates/vars/vars_test.go b/modules/templates/vars/vars_test.go new file mode 100644 index 0000000000000..1e80ac2b2ff7b --- /dev/null +++ b/modules/templates/vars/vars_test.go @@ -0,0 +1,70 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package vars + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestExpandVars(t *testing.T) { + kases := []struct { + template string + maps map[string]string + expected string + fail bool + }{ + { + template: "{a}", + maps: map[string]string{ + "a": "1", + }, + expected: "1", + }, + { + template: "expand {a}, {b} and {c}", + maps: map[string]string{ + "a": "1", + "b": "2", + "c": "3", + }, + expected: "expand 1, 2 and 3", + }, + { + template: "中文内容 {一}, {二} 和 {三} 中文结尾", + maps: map[string]string{ + "一": "11", + "二": "22", + "三": "33", + }, + expected: "中文内容 11, 22 和 33 中文结尾", + }, + { + template: "expand {{a}, {b} and {c}", + fail: true, + }, + { + template: "expand {}, {b} and {c}", + fail: true, + }, + { + template: "expand }, {b} and {c}", + fail: true, + }, + } + + for _, kase := range kases { + t.Run(kase.template, func(t *testing.T) { + res, err := Expand(kase.template, kase.maps) + if kase.fail { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + assert.EqualValues(t, kase.expected, res) + }) + } +} diff --git a/modules/util/util.go b/modules/util/util.go index af6581f7cdbe3..eceb72d57b459 100644 --- a/modules/util/util.go +++ b/modules/util/util.go @@ -11,6 +11,8 @@ import ( "math/big" "strconv" "strings" + + "github.com/unknwon/com" ) // OptionalBool a boolean that can be "null" @@ -181,3 +183,8 @@ func ToUpperASCII(s string) string { } return string(b) } + +// ToStr should be replaced +func ToStr(value interface{}, args ...int) string { + return com.ToStr(value, args...) +} diff --git a/routers/web/goget.go b/routers/web/goget.go index 2843a96c30bd3..0685730919fcc 100644 --- a/routers/web/goget.go +++ b/routers/web/goget.go @@ -12,10 +12,10 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/templates/vars" "code.gitea.io/gitea/modules/util" - - "github.com/unknwon/com" ) func goGet(ctx *context.Context) { @@ -67,7 +67,7 @@ func goGet(ctx *context.Context) { } ctx.RespHeader().Set("Content-Type", "text/html") ctx.Status(http.StatusOK) - _, _ = ctx.Write([]byte(com.Expand(` + res, err := vars.Expand(` @@ -83,5 +83,19 @@ func goGet(ctx *context.Context) { "GoDocDirectory": prefix + "{/dir}", "GoDocFile": prefix + "{/dir}/{file}#L{line}", "Insecure": insecure, - }))) + }) + if err != nil { + log.Error(err.Error()) + _, _ = ctx.Write([]byte(` + + + invalid import path + + +`)) + ctx.Status(400) + return + } + + _, _ = ctx.Write([]byte(res)) } diff --git a/routers/web/repo/issue.go b/routers/web/repo/issue.go index a81b1f19626c6..cf5aeb14fd348 100644 --- a/routers/web/repo/issue.go +++ b/routers/web/repo/issue.go @@ -32,6 +32,7 @@ import ( "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/templates/vars" "code.gitea.io/gitea/modules/upload" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web" @@ -40,8 +41,6 @@ import ( "code.gitea.io/gitea/services/forms" issue_service "code.gitea.io/gitea/services/issue" pull_service "code.gitea.io/gitea/services/pull" - - "github.com/unknwon/com" ) const ( @@ -1103,7 +1102,12 @@ func ViewIssue(ctx *context.Context) { if extIssueUnit.ExternalTrackerConfig().ExternalTrackerStyle == markup.IssueNameStyleNumeric || extIssueUnit.ExternalTrackerConfig().ExternalTrackerStyle == "" { metas := ctx.Repo.Repository.ComposeMetas() metas["index"] = ctx.Params(":index") - ctx.Redirect(com.Expand(extIssueUnit.ExternalTrackerConfig().ExternalTrackerFormat, metas)) + res, err := vars.Expand(extIssueUnit.ExternalTrackerConfig().ExternalTrackerFormat, metas) + if err != nil { + ctx.ServerError("Expand", err) + return + } + ctx.Redirect(res) return } } else if err != nil && !repo_model.IsErrUnitTypeNotExist(err) { From 6b8b5a777911c48ab768cadbc60b6e03ec7d927d Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Mon, 21 Mar 2022 14:26:28 +0800 Subject: [PATCH 02/13] Remove most calls to `com`, improve `vars.Expand` --- .golangci.yml | 14 +++- modules/context/context.go | 6 +- modules/context/csrf.go | 11 ++- modules/json/json.go | 2 +- modules/markup/html.go | 4 +- modules/repository/init.go | 3 +- modules/setting/setting.go | 3 +- modules/templates/vars/vars.go | 108 ++++++++++++++-------------- modules/templates/vars/vars_test.go | 16 +++-- modules/util/copy.go | 2 +- modules/util/path.go | 25 ++++++- modules/util/util.go | 19 ++++- modules/web/middleware/binding.go | 6 +- routers/web/repo/issue.go | 1 + 14 files changed, 137 insertions(+), 83 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 143359bcf0935..8fcb57fb11cc9 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -18,6 +18,7 @@ linters: - ineffassign - revive - gofumpt + - depguard enable-all: false disable-all: true fast: false @@ -64,7 +65,15 @@ linters-settings: - name: modifies-value-receiver gofumpt: extra-rules: true - lang-version: 1.18 + lang-version: "1.18" + depguard: + # TODO: use depguard to replace import checks in gitea-vet + list-type: denylist + # Check the list against standard lib. + include-go-root: true + packages-with-error-message: + - encoding/json: "use gitea's modules/json instead of encoding/json" + - github.com/unknwon/com: "use gitea's util and replacements" issues: exclude-rules: @@ -152,5 +161,6 @@ issues: - path: models/user/openid.go linters: - golint - - linters: staticcheck + - linters: + - staticcheck text: "strings.Title is deprecated: The rule Title uses for word boundaries does not handle Unicode punctuation properly. Use golang.org/x/text/cases instead." diff --git a/modules/context/context.go b/modules/context/context.go index 8e50e154a14d5..918e70b2b1ec3 100644 --- a/modules/context/context.go +++ b/modules/context/context.go @@ -31,13 +31,13 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/templates" "code.gitea.io/gitea/modules/translation" + "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web/middleware" "code.gitea.io/gitea/services/auth" "gitea.com/go-chi/cache" "gitea.com/go-chi/session" chi "github.com/go-chi/chi/v5" - "github.com/unknwon/com" "github.com/unrolled/render" "golang.org/x/crypto/pbkdf2" ) @@ -452,7 +452,7 @@ func (ctx *Context) CookieDecrypt(secret, val string) (string, bool) { } key := pbkdf2.Key([]byte(secret), []byte(secret), 1000, 16, sha256.New) - text, err = com.AESGCMDecrypt(key, text) + text, err = util.AESGCMDecrypt(key, text) return string(text), err == nil } @@ -466,7 +466,7 @@ func (ctx *Context) SetSuperSecureCookie(secret, name, value string, expiry int) // CookieEncrypt encrypts a given value using the provided secret func (ctx *Context) CookieEncrypt(secret, value string) string { key := pbkdf2.Key([]byte(secret), []byte(secret), 1000, 16, sha256.New) - text, err := com.AESGCMEncrypt(key, []byte(value)) + text, err := util.AESGCMEncrypt(key, []byte(value)) if err != nil { panic("error encrypting cookie: " + err.Error()) } diff --git a/modules/context/csrf.go b/modules/context/csrf.go index f7c9205b72c5e..1fb992e2ae409 100644 --- a/modules/context/csrf.go +++ b/modules/context/csrf.go @@ -19,14 +19,14 @@ package context import ( + "encoding/base32" + "fmt" "net/http" "time" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web/middleware" - - "github.com/unknwon/com" ) // CSRF represents a CSRF service and is used to get the current token and validate a suspect token. @@ -163,7 +163,12 @@ func prepareOptions(options []CsrfOptions) CsrfOptions { // Defaults. if len(opt.Secret) == 0 { - opt.Secret = string(com.RandomCreateBytes(10)) + randBytes, err := util.CryptoRandomBytes(8) + if err != nil { + // this panic can be handled by the recover() in http handlers + panic(fmt.Errorf("failed to generate random bytes: %w", err)) + } + opt.Secret = base32.StdEncoding.EncodeToString(randBytes) } if len(opt.Header) == 0 { opt.Header = "X-CSRFToken" diff --git a/modules/json/json.go b/modules/json/json.go index 3afa86023c6f8..4361262a2f241 100644 --- a/modules/json/json.go +++ b/modules/json/json.go @@ -8,7 +8,7 @@ package json import ( "bytes" "encoding/binary" - "encoding/json" + "encoding/json" //nolint:depguard "io" jsoniter "github.com/json-iterator/go" diff --git a/modules/markup/html.go b/modules/markup/html.go index 1769f76b2a26f..ddabb4fc0125e 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -840,8 +840,8 @@ func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) { res, err := vars.Expand(ctx.Metas["format"], ctx.Metas) if err != nil { - log.Error(err.Error()) - return + // here we could just log the error and continue the rendering + log.Error("unable to expand template vars for ref %s, err:%s", ref.Issue, err.Error()) } link = createLink(res, reftext, "ref-issue ref-external-issue") diff --git a/modules/repository/init.go b/modules/repository/init.go index abf4b9317283b..18843bd2cac61 100644 --- a/modules/repository/init.go +++ b/modules/repository/init.go @@ -62,7 +62,8 @@ func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir, } res, err := vars.Expand(string(data), match) if err != nil { - return fmt.Errorf("expand README.md: %v", err) + // here we could just log the error and continue the rendering + log.Error("unable to expand template vars for repo README: %s, err: %s", opts.Readme, err.Error()) } if err = os.WriteFile(filepath.Join(tmpDir, "README.md"), []byte(res), 0o644); err != nil { diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 2a189ec9b3f2b..3e49b2a8f17a7 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -27,7 +27,6 @@ import ( "code.gitea.io/gitea/modules/user" "code.gitea.io/gitea/modules/util" - "github.com/unknwon/com" gossh "golang.org/x/crypto/ssh" ini "gopkg.in/ini.v1" ) @@ -601,7 +600,7 @@ func loadFromConf(allowEmpty bool, extraConfig string) { Cfg.NameMapper = ini.SnackCase - homeDir, err := com.HomeDir() + homeDir, err := util.HomeDir() if err != nil { log.Fatal("Failed to get home directory: %v", err) } diff --git a/modules/templates/vars/vars.go b/modules/templates/vars/vars.go index 152819c32e553..a3b554e86956b 100644 --- a/modules/templates/vars/vars.go +++ b/modules/templates/vars/vars.go @@ -40,67 +40,65 @@ func IsErrNoMatchedVar(err error) bool { return ok } -// Expand replaces all variables like {var} to match -func Expand(template string, match map[string]string, subs ...string) (string, error) { - var ( - buf strings.Builder - key strings.Builder - enter bool - ) - for _, c := range template { - switch { - case c == '{': - if enter { - return "", ErrWrongSyntax{ - Template: template, - } - } - enter = true - case c == '}': - if !enter { - return "", ErrWrongSyntax{ - Template: template, - } - } - if key.Len() == 0 { - return "", ErrWrongSyntax{ - Template: template, - } - } +// Expand replaces all variables like {var} to match, if error occurs, the error part doesn't change and is returned as it is. +// `#' is a reversed char, templates can use `{#{}` to do escape and output char '{'. +func Expand(template string, match map[string]string) (string, error) { + var buf strings.Builder + var err error - if len(match) == 0 { - return "", ErrNoMatchedVar{ - Template: template, - Var: key.String(), - } - } + posBegin := 0 + strLen := len(template) + for posBegin < strLen { + // find the next `{` + pos := strings.IndexByte(template[posBegin:], '{') + if pos == -1 { + buf.WriteString(template[posBegin:]) + break + } - v, ok := match[key.String()] - if !ok { - if len(subs) == 0 { - return "", ErrNoMatchedVar{ - Template: template, - Var: key.String(), - } - } - v = subs[0] - } + // copy texts between vars + buf.WriteString(template[posBegin : posBegin+pos]) - if _, err := buf.WriteString(v); err != nil { - return "", err + // find the var between `{` and `}`/end + posBegin += pos + posEnd := posBegin + 1 + for posEnd < strLen { + if template[posEnd] == '#' { + // escape char, skip next + posEnd += 2 + continue + } else if template[posEnd] == '}' { + posEnd++ + break } - key.Reset() + posEnd++ + } - enter = false - case enter: - if _, err := key.WriteRune(c); err != nil { - return "", err - } - default: - if _, err := buf.WriteRune(c); err != nil { - return "", err + // the var part, it can be "{", "{}", "{..." or or "{...}" + part := template[posBegin:posEnd] + posBegin = posEnd + if part == "{}" || part[len(part)-1] != '}' { + // treat "{}" or "{..." as error + err = ErrWrongSyntax{Template: template} + buf.WriteString(part) + } else { + // now we get a valid key "{...}" + key := part[1 : len(part)-1] + if key[0] == '#' { + // escaped char + buf.WriteString(key[1:]) + } else { + // look up in the map + if val, ok := match[key]; ok { + buf.WriteString(val) + } else { + // write the non-existing var as it is + buf.WriteString(part) + err = ErrNoMatchedVar{Template: template, Var: key} + } } } } - return buf.String(), nil + + return buf.String(), err } diff --git a/modules/templates/vars/vars_test.go b/modules/templates/vars/vars_test.go index 1e80ac2b2ff7b..7b16e51ac43ec 100644 --- a/modules/templates/vars/vars_test.go +++ b/modules/templates/vars/vars_test.go @@ -25,13 +25,13 @@ func TestExpandVars(t *testing.T) { expected: "1", }, { - template: "expand {a}, {b} and {c}", + template: "expand {a}, {b} and {c}, with escaped {#{}", maps: map[string]string{ "a": "1", "b": "2", "c": "3", }, - expected: "expand 1, 2 and 3", + expected: "expand 1, 2 and 3, with escaped {", }, { template: "中文内容 {一}, {二} 和 {三} 中文结尾", @@ -44,14 +44,16 @@ func TestExpandVars(t *testing.T) { }, { template: "expand {{a}, {b} and {c}", + maps: map[string]string{ + "a": "foo", + "b": "bar", + }, + expected: "expand {{a}, bar and {c}", fail: true, }, { - template: "expand {}, {b} and {c}", - fail: true, - }, - { - template: "expand }, {b} and {c}", + template: "expand } {} and {", + expected: "expand } {} and {", fail: true, }, } diff --git a/modules/util/copy.go b/modules/util/copy.go index 46765849dc753..c359719276911 100644 --- a/modules/util/copy.go +++ b/modules/util/copy.go @@ -5,7 +5,7 @@ package util import ( - "github.com/unknwon/com" + "github.com/unknwon/com" //nolint:depguard ) // CopyFile copies file from source to target path. diff --git a/modules/util/path.go b/modules/util/path.go index f4acf92ba933e..90fcf82d3749e 100644 --- a/modules/util/path.go +++ b/modules/util/path.go @@ -154,6 +154,10 @@ func StatDir(rootPath string, includeDir ...bool) ([]string, error) { return statDir(rootPath, "", isIncludeDir, false, false) } +func isOSWindows() bool { + return runtime.GOOS == "windows" +} + // FileURLToPath extracts the path information from a file://... url. func FileURLToPath(u *url.URL) (string, error) { if u.Scheme != "file" { @@ -162,7 +166,7 @@ func FileURLToPath(u *url.URL) (string, error) { path := u.Path - if runtime.GOOS != "windows" { + if !isOSWindows() { return path, nil } @@ -173,3 +177,22 @@ func FileURLToPath(u *url.URL) (string, error) { } return path, nil } + +// HomeDir returns path of '~'(in Linux) on Windows, +// it returns error when the variable does not exist. +func HomeDir() (home string, err error) { + if isOSWindows() { + home = os.Getenv("USERPROFILE") + if home == "" { + home = os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH") + } + } else { + home = os.Getenv("HOME") + } + + if home == "" { + return "", errors.New("cannot get home directory") + } + + return home, nil +} diff --git a/modules/util/util.go b/modules/util/util.go index eceb72d57b459..c9dbada12fcb7 100644 --- a/modules/util/util.go +++ b/modules/util/util.go @@ -12,7 +12,7 @@ import ( "strconv" "strings" - "github.com/unknwon/com" + "github.com/unknwon/com" //nolint:depguard ) // OptionalBool a boolean that can be "null" @@ -184,7 +184,22 @@ func ToUpperASCII(s string) string { return string(b) } -// ToStr should be replaced +// ToStr (from unknwon/com): should be replaced. func ToStr(value interface{}, args ...int) string { return com.ToStr(value, args...) } + +// ToSnakeCase (from unknwon/com): should be replaced. +func ToSnakeCase(str string) string { + return com.ToSnakeCase(str) +} + +// AESGCMEncrypt (from unknwon/com): encrypts plaintext with the given key using AES in GCM mode. should be replaced. +func AESGCMEncrypt(key, plaintext []byte) ([]byte, error) { + return com.AESGCMEncrypt(key, plaintext) +} + +// AESGCMDecrypt (from unknwon/com): decrypts ciphertext with the given key using AES in GCM mode. should be replaced. +func AESGCMDecrypt(key, ciphertext []byte) ([]byte, error) { + return com.AESGCMDecrypt(key, ciphertext) +} diff --git a/modules/web/middleware/binding.go b/modules/web/middleware/binding.go index 9b0b1d7784c32..c9dc4a8f59a01 100644 --- a/modules/web/middleware/binding.go +++ b/modules/web/middleware/binding.go @@ -10,10 +10,10 @@ import ( "strings" "code.gitea.io/gitea/modules/translation" + "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/validation" "gitea.com/go-chi/binding" - "github.com/unknwon/com" ) // Form form binding interface @@ -22,7 +22,7 @@ type Form interface { } func init() { - binding.SetNameMapper(com.ToSnakeCase) + binding.SetNameMapper(util.ToSnakeCase) } // AssignForm assign form values back to the template data. @@ -43,7 +43,7 @@ func AssignForm(form interface{}, data map[string]interface{}) { if fieldName == "-" { continue } else if len(fieldName) == 0 { - fieldName = com.ToSnakeCase(field.Name) + fieldName = util.ToSnakeCase(field.Name) } data[fieldName] = val.Field(i).Interface() diff --git a/routers/web/repo/issue.go b/routers/web/repo/issue.go index cf5aeb14fd348..573a22fccddf4 100644 --- a/routers/web/repo/issue.go +++ b/routers/web/repo/issue.go @@ -1104,6 +1104,7 @@ func ViewIssue(ctx *context.Context) { metas["index"] = ctx.Params(":index") res, err := vars.Expand(extIssueUnit.ExternalTrackerConfig().ExternalTrackerFormat, metas) if err != nil { + log.Error("unable to expand template vars for issue url. issue: %s, err: %s", metas["index"], err.Error()) ctx.ServerError("Expand", err) return } From b9c1142555120bf6f0487912ab236ea8c5f567fa Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Fri, 1 Apr 2022 02:19:29 +0800 Subject: [PATCH 03/13] fine tune Expand behavior --- modules/templates/vars/vars.go | 15 ++++++--- modules/templates/vars/vars_test.go | 48 ++++++++++++++--------------- 2 files changed, 34 insertions(+), 29 deletions(-) diff --git a/modules/templates/vars/vars.go b/modules/templates/vars/vars.go index a3b554e86956b..d2b12effbf74f 100644 --- a/modules/templates/vars/vars.go +++ b/modules/templates/vars/vars.go @@ -7,6 +7,8 @@ package vars import ( "fmt" "strings" + "unicode" + "unicode/utf8" ) // ErrWrongSyntax represents a wrong syntax with a tempate @@ -40,9 +42,11 @@ func IsErrNoMatchedVar(err error) bool { return ok } -// Expand replaces all variables like {var} to match, if error occurs, the error part doesn't change and is returned as it is. -// `#' is a reversed char, templates can use `{#{}` to do escape and output char '{'. +// Expand replaces all variables like {var} to match, if error occurs, +// the error part doesn't change and is returned as it is. func Expand(template string, match map[string]string) (string, error) { + // in the future, if necessary, we can introduce some escape-char, + // for example: it will use `#' as a reversed char, templates will use `{#{}` to do escape and output char '{'. var buf strings.Builder var err error @@ -84,9 +88,10 @@ func Expand(template string, match map[string]string) (string, error) { } else { // now we get a valid key "{...}" key := part[1 : len(part)-1] - if key[0] == '#' { - // escaped char - buf.WriteString(key[1:]) + keyFirst, _ := utf8.DecodeRuneInString(key) + if unicode.IsSpace(keyFirst) || unicode.IsPunct(keyFirst) || unicode.IsControl(keyFirst) { + // the if key doesn't start with a letter, then we do not treat it as a var now + buf.WriteString(part) } else { // look up in the map if val, ok := match[key]; ok { diff --git a/modules/templates/vars/vars_test.go b/modules/templates/vars/vars_test.go index 7b16e51ac43ec..78930382a8abe 100644 --- a/modules/templates/vars/vars_test.go +++ b/modules/templates/vars/vars_test.go @@ -12,61 +12,61 @@ import ( func TestExpandVars(t *testing.T) { kases := []struct { - template string - maps map[string]string - expected string - fail bool + tmpl string + data map[string]string + out string + error bool }{ { - template: "{a}", - maps: map[string]string{ + tmpl: "{a}", + data: map[string]string{ "a": "1", }, - expected: "1", + out: "1", }, { - template: "expand {a}, {b} and {c}, with escaped {#{}", - maps: map[string]string{ + tmpl: "expand {a}, {b} and {c}, with non-var { } {#}", + data: map[string]string{ "a": "1", "b": "2", "c": "3", }, - expected: "expand 1, 2 and 3, with escaped {", + out: "expand 1, 2 and 3, with non-var { } {#}", }, { - template: "中文内容 {一}, {二} 和 {三} 中文结尾", - maps: map[string]string{ + tmpl: "中文内容 {一}, {二} 和 {三} 中文结尾", + data: map[string]string{ "一": "11", "二": "22", "三": "33", }, - expected: "中文内容 11, 22 和 33 中文结尾", + out: "中文内容 11, 22 和 33 中文结尾", }, { - template: "expand {{a}, {b} and {c}", - maps: map[string]string{ + tmpl: "expand {{a}, {b} and {c}", + data: map[string]string{ "a": "foo", "b": "bar", }, - expected: "expand {{a}, bar and {c}", - fail: true, + out: "expand {{a}, bar and {c}", + error: true, }, { - template: "expand } {} and {", - expected: "expand } {} and {", - fail: true, + tmpl: "expand } {} and {", + out: "expand } {} and {", + error: true, }, } for _, kase := range kases { - t.Run(kase.template, func(t *testing.T) { - res, err := Expand(kase.template, kase.maps) - if kase.fail { + t.Run(kase.tmpl, func(t *testing.T) { + res, err := Expand(kase.tmpl, kase.data) + if kase.error { assert.Error(t, err) } else { assert.NoError(t, err) } - assert.EqualValues(t, kase.expected, res) + assert.EqualValues(t, kase.out, res) }) } } From c202109265856556556aba9c5669fc2c9075211f Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Fri, 1 Apr 2022 02:35:17 +0800 Subject: [PATCH 04/13] fix goget response --- routers/web/goget.go | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/routers/web/goget.go b/routers/web/goget.go index da7f74e5484c5..d2d5186e70d2f 100644 --- a/routers/web/goget.go +++ b/routers/web/goget.go @@ -65,8 +65,7 @@ func goGet(ctx *context.Context) { if appURL.Scheme == string(setting.HTTP) { insecure = "--insecure " } - ctx.RespHeader().Set("Content-Type", "text/html") - ctx.Status(http.StatusOK) + res, err := vars.Expand(` @@ -85,17 +84,12 @@ func goGet(ctx *context.Context) { "Insecure": insecure, }) if err != nil { - log.Error(err.Error()) - _, _ = ctx.Write([]byte(` - - - invalid import path - - -`)) - ctx.Status(400) + log.Error("error occurs when rendering goget response: %v", err.Error()) + ctx.Error(http.StatusInternalServerError, "failed to render goget content") return } + ctx.RespHeader().Set("Content-Type", "text/html") + ctx.Status(http.StatusOK) _, _ = ctx.Write([]byte(res)) } From 5ea73ab3b6bb28d1a3c6953576371f66efdba69b Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 31 Mar 2022 23:06:20 +0200 Subject: [PATCH 05/13] Apply suggestions from code review Co-authored-by: Gusted --- modules/markup/html.go | 2 +- modules/repository/init.go | 2 +- modules/util/util.go | 43 ++++++++++++++++++++++++++++++++++++-- routers/web/goget.go | 2 +- routers/web/repo/issue.go | 2 +- 5 files changed, 45 insertions(+), 6 deletions(-) diff --git a/modules/markup/html.go b/modules/markup/html.go index f8613dda7f66c..c5d36e701fb01 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -842,7 +842,7 @@ func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) { res, err := vars.Expand(ctx.Metas["format"], ctx.Metas) if err != nil { // here we could just log the error and continue the rendering - log.Error("unable to expand template vars for ref %s, err:%s", ref.Issue, err.Error()) + log.Error("unable to expand template vars for ref %s, err: %v", ref.Issue, err) } link = createLink(res, reftext, "ref-issue ref-external-issue") diff --git a/modules/repository/init.go b/modules/repository/init.go index 3286f88b8bb68..07ba8229010a5 100644 --- a/modules/repository/init.go +++ b/modules/repository/init.go @@ -252,7 +252,7 @@ func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir, res, err := vars.Expand(string(data), match) if err != nil { // here we could just log the error and continue the rendering - log.Error("unable to expand template vars for repo README: %s, err: %s", opts.Readme, err.Error()) + log.Error("unable to expand template vars for repo README: %s, err: %v", opts.Readme, err) } if err = os.WriteFile(filepath.Join(tmpDir, "README.md"), []byte(res), 0o644); err != nil { diff --git a/modules/util/util.go b/modules/util/util.go index c9dbada12fcb7..cb2dd7e4e4728 100644 --- a/modules/util/util.go +++ b/modules/util/util.go @@ -196,10 +196,49 @@ func ToSnakeCase(str string) string { // AESGCMEncrypt (from unknwon/com): encrypts plaintext with the given key using AES in GCM mode. should be replaced. func AESGCMEncrypt(key, plaintext []byte) ([]byte, error) { - return com.AESGCMEncrypt(key, plaintext) + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + + nonce := make([]byte, gcm.NonceSize()) + if _, err := rand.Read(nonce); err != nil { + return nil, err + } + + ciphertext := gcm.Seal(nil, nonce, plaintext, nil) + return append(nonce, ciphertext...), nil } // AESGCMDecrypt (from unknwon/com): decrypts ciphertext with the given key using AES in GCM mode. should be replaced. func AESGCMDecrypt(key, ciphertext []byte) ([]byte, error) { - return com.AESGCMDecrypt(key, ciphertext) + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + + size := gcm.NonceSize() + if len(ciphertext)-size <= 0 { + return nil, errors.New("ciphertext is empty") + } + + nonce := ciphertext[:size] + ciphertext = ciphertext[size:] + + plainText, err := gcm.Open(nil, nonce, ciphertext, nil) + if err != nil { + return nil, err + } + + return plainText, nil } diff --git a/routers/web/goget.go b/routers/web/goget.go index d2d5186e70d2f..f7994856612c3 100644 --- a/routers/web/goget.go +++ b/routers/web/goget.go @@ -84,7 +84,7 @@ func goGet(ctx *context.Context) { "Insecure": insecure, }) if err != nil { - log.Error("error occurs when rendering goget response: %v", err.Error()) + log.Error("error occurs when rendering goget response: %v", err) ctx.Error(http.StatusInternalServerError, "failed to render goget content") return } diff --git a/routers/web/repo/issue.go b/routers/web/repo/issue.go index 8b9c513872b8a..a1a7200ba41fa 100644 --- a/routers/web/repo/issue.go +++ b/routers/web/repo/issue.go @@ -1114,7 +1114,7 @@ func ViewIssue(ctx *context.Context) { metas["index"] = ctx.Params(":index") res, err := vars.Expand(extIssueUnit.ExternalTrackerConfig().ExternalTrackerFormat, metas) if err != nil { - log.Error("unable to expand template vars for issue url. issue: %s, err: %s", metas["index"], err.Error()) + log.Error("unable to expand template vars for issue url. issue: %s, err: %v", metas["index"], err) ctx.ServerError("Expand", err) return } From e9008eb77ed5ec7c6b78c5a9bd26ec6232a801f1 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Fri, 1 Apr 2022 10:49:13 +0800 Subject: [PATCH 06/13] fix build --- modules/util/path.go | 2 ++ modules/util/util.go | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/util/path.go b/modules/util/path.go index 90fcf82d3749e..ed7cc62699446 100644 --- a/modules/util/path.go +++ b/modules/util/path.go @@ -181,6 +181,8 @@ func FileURLToPath(u *url.URL) (string, error) { // HomeDir returns path of '~'(in Linux) on Windows, // it returns error when the variable does not exist. func HomeDir() (home string, err error) { + // TODO: some users run Gitea with mismatched uid and "HOME=xxx" (they set HOME=xxx by environment manually) + // so at the moment we can not use `user.Current().HomeDir` if isOSWindows() { home = os.Getenv("USERPROFILE") if home == "" { diff --git a/modules/util/util.go b/modules/util/util.go index cb2dd7e4e4728..5c7cc9a327861 100644 --- a/modules/util/util.go +++ b/modules/util/util.go @@ -6,6 +6,8 @@ package util import ( "bytes" + "crypto/aes" + "crypto/cipher" "crypto/rand" "errors" "math/big" @@ -217,7 +219,7 @@ func AESGCMEncrypt(key, plaintext []byte) ([]byte, error) { // AESGCMDecrypt (from unknwon/com): decrypts ciphertext with the given key using AES in GCM mode. should be replaced. func AESGCMDecrypt(key, ciphertext []byte) ([]byte, error) { - block, err := aes.NewCipher(key) + block, err := aes.NewCipher(key) if err != nil { return nil, err } From 7376a8a66b29fd8edbcb260aeff7ab6e9693efa4 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Fri, 1 Apr 2022 11:46:41 +0800 Subject: [PATCH 07/13] fine tune --- modules/util/copy.go | 20 --------- modules/util/io.go | 2 +- modules/util/legacy.go | 84 +++++++++++++++++++++++++++++++++++++ modules/util/legacy_test.go | 42 +++++++++++++++++++ modules/util/util.go | 63 ---------------------------- 5 files changed, 127 insertions(+), 84 deletions(-) delete mode 100644 modules/util/copy.go create mode 100644 modules/util/legacy.go create mode 100644 modules/util/legacy_test.go diff --git a/modules/util/copy.go b/modules/util/copy.go deleted file mode 100644 index c359719276911..0000000000000 --- a/modules/util/copy.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package util - -import ( - "github.com/unknwon/com" //nolint:depguard -) - -// CopyFile copies file from source to target path. -func CopyFile(src, dest string) error { - return com.Copy(src, dest) -} - -// CopyDir copy files recursively from source to target directory. -// It returns error when error occurs in underlying functions. -func CopyDir(srcPath, destPath string) error { - return com.CopyDir(srcPath, destPath) -} diff --git a/modules/util/io.go b/modules/util/io.go index b467c0ac8a04a..0c677c359f5ea 100644 --- a/modules/util/io.go +++ b/modules/util/io.go @@ -9,7 +9,7 @@ import ( ) // ReadAtMost reads at most len(buf) bytes from r into buf. -// It returns the number of bytes copied. n is only less then len(buf) if r provides fewer bytes. +// It returns the number of bytes copied. n is only less than len(buf) if r provides fewer bytes. // If EOF occurs while reading, err will be nil. func ReadAtMost(r io.Reader, buf []byte) (n int, err error) { n, err = io.ReadFull(r, buf) diff --git a/modules/util/legacy.go b/modules/util/legacy.go new file mode 100644 index 0000000000000..c7da5415349a0 --- /dev/null +++ b/modules/util/legacy.go @@ -0,0 +1,84 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package util + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "errors" + + "github.com/unknwon/com" //nolint:depguard +) + +// CopyFile copies file from source to target path. +func CopyFile(src, dest string) error { + return com.Copy(src, dest) +} + +// CopyDir copy files recursively from source to target directory. +// It returns error when error occurs in underlying functions. +func CopyDir(srcPath, destPath string) error { + return com.CopyDir(srcPath, destPath) +} + +// ToStr converts any interface to string. should be replaced. +func ToStr(value interface{}, args ...int) string { + return com.ToStr(value, args...) +} + +// ToSnakeCase converts a string to snake_case. should be replaced. +func ToSnakeCase(str string) string { + return com.ToSnakeCase(str) +} + +// AESGCMEncrypt (from legacy package): encrypts plaintext with the given key using AES in GCM mode. should be replaced. +func AESGCMEncrypt(key, plaintext []byte) ([]byte, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + + nonce := make([]byte, gcm.NonceSize()) + if _, err := rand.Read(nonce); err != nil { + return nil, err + } + + ciphertext := gcm.Seal(nil, nonce, plaintext, nil) + return append(nonce, ciphertext...), nil +} + +// AESGCMDecrypt (from legacy package): decrypts ciphertext with the given key using AES in GCM mode. should be replaced. +func AESGCMDecrypt(key, ciphertext []byte) ([]byte, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + + size := gcm.NonceSize() + if len(ciphertext)-size <= 0 { + return nil, errors.New("ciphertext is empty") + } + + nonce := ciphertext[:size] + ciphertext = ciphertext[size:] + + plainText, err := gcm.Open(nil, nonce, ciphertext, nil) + if err != nil { + return nil, err + } + + return plainText, nil +} diff --git a/modules/util/legacy_test.go b/modules/util/legacy_test.go new file mode 100644 index 0000000000000..3070452b910c0 --- /dev/null +++ b/modules/util/legacy_test.go @@ -0,0 +1,42 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package util + +import ( + "crypto/aes" + "crypto/rand" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestAESGCMEncrypt(t *testing.T) { + t.Parallel() + + key := make([]byte, aes.BlockSize) + _, err := rand.Read(key) + assert.NoError(t, err) + + plaintext := []byte("this will be encrypted") + _, err = AESGCMEncrypt(key, plaintext) + assert.NoError(t, err) +} + +func TestAESGCMDecrypt(t *testing.T) { + t.Parallel() + + key := make([]byte, aes.BlockSize) + _, err := rand.Read(key) + assert.NoError(t, err) + + plaintext := []byte("this will be encrypted") + + ciphertext, err := AESGCMEncrypt(key, plaintext) + assert.NoError(t, err) + + decrypted, err := AESGCMDecrypt(key, ciphertext) + assert.NoError(t, err) + + assert.Equal(t, plaintext, decrypted) +} diff --git a/modules/util/util.go b/modules/util/util.go index 5c7cc9a327861..af6581f7cdbe3 100644 --- a/modules/util/util.go +++ b/modules/util/util.go @@ -6,15 +6,11 @@ package util import ( "bytes" - "crypto/aes" - "crypto/cipher" "crypto/rand" "errors" "math/big" "strconv" "strings" - - "github.com/unknwon/com" //nolint:depguard ) // OptionalBool a boolean that can be "null" @@ -185,62 +181,3 @@ func ToUpperASCII(s string) string { } return string(b) } - -// ToStr (from unknwon/com): should be replaced. -func ToStr(value interface{}, args ...int) string { - return com.ToStr(value, args...) -} - -// ToSnakeCase (from unknwon/com): should be replaced. -func ToSnakeCase(str string) string { - return com.ToSnakeCase(str) -} - -// AESGCMEncrypt (from unknwon/com): encrypts plaintext with the given key using AES in GCM mode. should be replaced. -func AESGCMEncrypt(key, plaintext []byte) ([]byte, error) { - block, err := aes.NewCipher(key) - if err != nil { - return nil, err - } - - gcm, err := cipher.NewGCM(block) - if err != nil { - return nil, err - } - - nonce := make([]byte, gcm.NonceSize()) - if _, err := rand.Read(nonce); err != nil { - return nil, err - } - - ciphertext := gcm.Seal(nil, nonce, plaintext, nil) - return append(nonce, ciphertext...), nil -} - -// AESGCMDecrypt (from unknwon/com): decrypts ciphertext with the given key using AES in GCM mode. should be replaced. -func AESGCMDecrypt(key, ciphertext []byte) ([]byte, error) { - block, err := aes.NewCipher(key) - if err != nil { - return nil, err - } - - gcm, err := cipher.NewGCM(block) - if err != nil { - return nil, err - } - - size := gcm.NonceSize() - if len(ciphertext)-size <= 0 { - return nil, errors.New("ciphertext is empty") - } - - nonce := ciphertext[:size] - ciphertext = ciphertext[size:] - - plainText, err := gcm.Open(nil, nonce, ciphertext, nil) - if err != nil { - return nil, err - } - - return plainText, nil -} From 6ded737c05d9c51892a4d0df7bc89a64ec54c690 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Fri, 1 Apr 2022 11:53:01 +0800 Subject: [PATCH 08/13] fix lint --- modules/util/legacy_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/util/legacy_test.go b/modules/util/legacy_test.go index 3070452b910c0..36382ffb390b3 100644 --- a/modules/util/legacy_test.go +++ b/modules/util/legacy_test.go @@ -7,8 +7,9 @@ package util import ( "crypto/aes" "crypto/rand" - "github.com/stretchr/testify/assert" "testing" + + "github.com/stretchr/testify/assert" ) func TestAESGCMEncrypt(t *testing.T) { From 554886d8ebe6ccf5d81da95cfcfa23feeeb07402 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Fri, 1 Apr 2022 11:59:14 +0800 Subject: [PATCH 09/13] fine tune --- modules/templates/vars/vars.go | 12 ++++-------- modules/templates/vars/vars_test.go | 2 +- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/modules/templates/vars/vars.go b/modules/templates/vars/vars.go index d2b12effbf74f..78bfaf103d10d 100644 --- a/modules/templates/vars/vars.go +++ b/modules/templates/vars/vars.go @@ -42,8 +42,8 @@ func IsErrNoMatchedVar(err error) bool { return ok } -// Expand replaces all variables like {var} to match, if error occurs, -// the error part doesn't change and is returned as it is. +// Expand replaces all variables like {var} to `match`, it always returns the expanded string regardless of errors +// if error occurs, the error part doesn't change and is returned as it is. func Expand(template string, match map[string]string) (string, error) { // in the future, if necessary, we can introduce some escape-char, // for example: it will use `#' as a reversed char, templates will use `{#{}` to do escape and output char '{'. @@ -67,14 +67,10 @@ func Expand(template string, match map[string]string) (string, error) { posBegin += pos posEnd := posBegin + 1 for posEnd < strLen { - if template[posEnd] == '#' { - // escape char, skip next - posEnd += 2 - continue - } else if template[posEnd] == '}' { + if template[posEnd] == '}' { posEnd++ break - } + } // in the future, if we need to support escape chars, we can do: if (isEscapeChar) { posEnd+=2 } posEnd++ } diff --git a/modules/templates/vars/vars_test.go b/modules/templates/vars/vars_test.go index 78930382a8abe..1cd7669c0095e 100644 --- a/modules/templates/vars/vars_test.go +++ b/modules/templates/vars/vars_test.go @@ -61,12 +61,12 @@ func TestExpandVars(t *testing.T) { for _, kase := range kases { t.Run(kase.tmpl, func(t *testing.T) { res, err := Expand(kase.tmpl, kase.data) + assert.EqualValues(t, kase.out, res) if kase.error { assert.Error(t, err) } else { assert.NoError(t, err) } - assert.EqualValues(t, kase.out, res) }) } } From d57054e0856db571467c6e6190852aea111ad1fa Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Fri, 1 Apr 2022 12:03:56 +0800 Subject: [PATCH 10/13] fine tune --- modules/templates/vars/vars.go | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/modules/templates/vars/vars.go b/modules/templates/vars/vars.go index 78bfaf103d10d..a22ea4d777fca 100644 --- a/modules/templates/vars/vars.go +++ b/modules/templates/vars/vars.go @@ -11,40 +11,28 @@ import ( "unicode/utf8" ) -// ErrWrongSyntax represents a wrong syntax with a tempate +// ErrWrongSyntax represents a wrong syntax with a template type ErrWrongSyntax struct { Template string } func (err ErrWrongSyntax) Error() string { - return fmt.Sprintf("Wrong syntax found in %s", err.Template) + return fmt.Sprintf("wrong syntax found in %s", err.Template) } -// IsErrWrongSyntax returns true if the error is ErrWrongSyntax -func IsErrWrongSyntax(err error) bool { - _, ok := err.(ErrWrongSyntax) - return ok -} - -// ErrNoMatchedVar represents an error that no matched vars -type ErrNoMatchedVar struct { +// ErrVarMissing represents an error that no matched variable +type ErrVarMissing struct { Template string Var string } -func (err ErrNoMatchedVar) Error() string { - return fmt.Sprintf("No matched variable %s found for %s", err.Var, err.Template) -} - -// IsErrNoMatchedVar returns true if the error is ErrNoMatchedVar -func IsErrNoMatchedVar(err error) bool { - _, ok := err.(ErrNoMatchedVar) - return ok +func (err ErrVarMissing) Error() string { + return fmt.Sprintf("the variable %s is missing for %s", err.Var, err.Template) } -// Expand replaces all variables like {var} to `match`, it always returns the expanded string regardless of errors +// Expand replaces all variables like {var} by `vars` map, it always returns the expanded string regardless of errors // if error occurs, the error part doesn't change and is returned as it is. -func Expand(template string, match map[string]string) (string, error) { +func Expand(template string, vars map[string]string) (string, error) { // in the future, if necessary, we can introduce some escape-char, // for example: it will use `#' as a reversed char, templates will use `{#{}` to do escape and output char '{'. var buf strings.Builder @@ -90,12 +78,12 @@ func Expand(template string, match map[string]string) (string, error) { buf.WriteString(part) } else { // look up in the map - if val, ok := match[key]; ok { + if val, ok := vars[key]; ok { buf.WriteString(val) } else { // write the non-existing var as it is buf.WriteString(part) - err = ErrNoMatchedVar{Template: template, Var: key} + err = ErrVarMissing{Template: template, Var: key} } } } From 73602a22217e5e961ed09b5af84ad8c6f10fb81c Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Fri, 1 Apr 2022 12:10:55 +0800 Subject: [PATCH 11/13] fine tune test --- modules/util/legacy_test.go | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/modules/util/legacy_test.go b/modules/util/legacy_test.go index 36382ffb390b3..cfda93d3ad34b 100644 --- a/modules/util/legacy_test.go +++ b/modules/util/legacy_test.go @@ -10,21 +10,10 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/unknwon/com" //nolint:depguard ) -func TestAESGCMEncrypt(t *testing.T) { - t.Parallel() - - key := make([]byte, aes.BlockSize) - _, err := rand.Read(key) - assert.NoError(t, err) - - plaintext := []byte("this will be encrypted") - _, err = AESGCMEncrypt(key, plaintext) - assert.NoError(t, err) -} - -func TestAESGCMDecrypt(t *testing.T) { +func TestAESGCM(t *testing.T) { t.Parallel() key := make([]byte, aes.BlockSize) @@ -40,4 +29,9 @@ func TestAESGCMDecrypt(t *testing.T) { assert.NoError(t, err) assert.Equal(t, plaintext, decrypted) + + // at the moment, we make sure the result is the same as the legacy package, this assertion can be removed in next round refactoring + legacy, err := com.AESGCMDecrypt(key, ciphertext) + assert.NoError(t, err) + assert.Equal(t, legacy, plaintext) } From 60f9948b4dd67b4c8f538b5f7049994e2428587c Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Fri, 1 Apr 2022 14:15:08 +0800 Subject: [PATCH 12/13] make go-import and go-source escaped correctly --- modules/context/repo.go | 16 ++++------------ routers/web/goget.go | 32 ++++++++++++-------------------- 2 files changed, 16 insertions(+), 32 deletions(-) diff --git a/modules/context/repo.go b/modules/context/repo.go index 002fea5de5621..a7c9a982c42b2 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -8,6 +8,7 @@ package context import ( "context" "fmt" + "html" "io" "net/http" "net/url" @@ -25,7 +26,6 @@ import ( "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" - "code.gitea.io/gitea/modules/templates/vars" "code.gitea.io/gitea/modules/util" asymkey_service "code.gitea.io/gitea/services/asymkey" @@ -308,17 +308,9 @@ func EarlyResponseForGoGetMeta(ctx *Context) { ctx.PlainText(http.StatusBadRequest, "invalid repository path") return } - res, err := vars.Expand(``, - map[string]string{ - "GoGetImport": ComposeGoGetImport(username, reponame), - "CloneLink": repo_model.ComposeHTTPSCloneURL(username, reponame), - }) - if err != nil { - log.Error(err.Error()) - ctx.PlainText(http.StatusInternalServerError, "expand vars failed") - return - } - ctx.PlainText(http.StatusOK, res) + goImportContent := fmt.Sprintf("%s git %s", ComposeGoGetImport(username, reponame), repo_model.ComposeHTTPSCloneURL(username, reponame)) + htmlMeta := fmt.Sprintf(``, html.EscapeString(goImportContent)) + ctx.PlainText(http.StatusOK, htmlMeta) } // RedirectToRepo redirect to a differently-named repository diff --git a/routers/web/goget.go b/routers/web/goget.go index f7994856612c3..a58739fe42a3c 100644 --- a/routers/web/goget.go +++ b/routers/web/goget.go @@ -5,6 +5,8 @@ package web import ( + "fmt" + "html" "net/http" "net/url" "path" @@ -12,9 +14,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/context" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/templates/vars" "code.gitea.io/gitea/modules/util" ) @@ -66,30 +66,22 @@ func goGet(ctx *context.Context) { insecure = "--insecure " } - res, err := vars.Expand(` + goGetImport := context.ComposeGoGetImport(ownerName, trimmedRepoName) + goImportContent := fmt.Sprintf("%s git %s", goGetImport, repo_model.ComposeHTTPSCloneURL(ownerName, repoName) /*CloneLink*/) + goSourceContent := fmt.Sprintf("%s _ %s %s", goGetImport, prefix+"{/dir}" /*GoDocDirectory*/, prefix+"{/dir}/{file}#L{line}" /*GoDocFile*/) + goGetCli := fmt.Sprintf("go get %s%s", insecure, goGetImport) + + res := fmt.Sprintf(` - - + + - go get {Insecure}{GoGetImport} + %s - -`, map[string]string{ - "GoGetImport": context.ComposeGoGetImport(ownerName, trimmedRepoName), - "CloneLink": repo_model.ComposeHTTPSCloneURL(ownerName, repoName), - "GoDocDirectory": prefix + "{/dir}", - "GoDocFile": prefix + "{/dir}/{file}#L{line}", - "Insecure": insecure, - }) - if err != nil { - log.Error("error occurs when rendering goget response: %v", err) - ctx.Error(http.StatusInternalServerError, "failed to render goget content") - return - } +`, html.EscapeString(goImportContent), html.EscapeString(goSourceContent), html.EscapeString(goGetCli)) ctx.RespHeader().Set("Content-Type", "text/html") - ctx.Status(http.StatusOK) _, _ = ctx.Write([]byte(res)) } From 5f115abc313c08d52ff52a8dd1bd29a6d738c343 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Fri, 1 Apr 2022 14:50:21 +0800 Subject: [PATCH 13/13] fix unit test --- integrations/goget_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/integrations/goget_test.go b/integrations/goget_test.go index 5dc9c5e0a8d17..504d869990fbe 100644 --- a/integrations/goget_test.go +++ b/integrations/goget_test.go @@ -29,8 +29,7 @@ func TestGoGet(t *testing.T) { go get --insecure %[1]s:%[2]s/blah/glah - -`, setting.Domain, setting.HTTPPort, setting.AppURL) +`, setting.Domain, setting.HTTPPort, setting.AppURL) assert.Equal(t, expected, resp.Body.String()) }