From 9a318609e6ede78b6ad721cd85d6157dab01cad5 Mon Sep 17 00:00:00 2001 From: jolheiser Date: Mon, 25 Nov 2019 20:41:05 -0600 Subject: [PATCH 01/17] Start expansion Signed-off-by: jolheiser --- models/repo.go | 48 ----------------- models/repo_generate.go | 114 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 110 insertions(+), 52 deletions(-) diff --git a/models/repo.go b/models/repo.go index 6c293da6f84bb..80281be3a3fbc 100644 --- a/models/repo.go +++ b/models/repo.go @@ -1360,54 +1360,6 @@ func prepareRepoCommit(e Engine, repo *Repository, tmpDir, repoPath string, opts return nil } -func generateRepoCommit(e Engine, repo, templateRepo *Repository, tmpDir string) error { - commitTimeStr := time.Now().Format(time.RFC3339) - authorSig := repo.Owner.NewGitSig() - - // Because this may call hooks we should pass in the environment - env := append(os.Environ(), - "GIT_AUTHOR_NAME="+authorSig.Name, - "GIT_AUTHOR_EMAIL="+authorSig.Email, - "GIT_AUTHOR_DATE="+commitTimeStr, - "GIT_COMMITTER_NAME="+authorSig.Name, - "GIT_COMMITTER_EMAIL="+authorSig.Email, - "GIT_COMMITTER_DATE="+commitTimeStr, - ) - - // Clone to temporary path and do the init commit. - templateRepoPath := templateRepo.repoPath(e) - _, stderr, err := process.GetManager().ExecDirEnv( - -1, "", - fmt.Sprintf("generateRepoCommit(git clone): %s", templateRepoPath), - env, - git.GitExecutable, "clone", "--depth", "1", templateRepoPath, tmpDir, - ) - if err != nil { - return fmt.Errorf("git clone: %v - %s", err, stderr) - } - - if err := os.RemoveAll(path.Join(tmpDir, ".git")); err != nil { - return fmt.Errorf("remove git dir: %v", err) - } - - if err := git.InitRepository(tmpDir, false); err != nil { - return err - } - - repoPath := repo.repoPath(e) - _, stderr, err = process.GetManager().ExecDirEnv( - -1, tmpDir, - fmt.Sprintf("generateRepoCommit(git remote add): %s", repoPath), - env, - git.GitExecutable, "remote", "add", "origin", repoPath, - ) - if err != nil { - return fmt.Errorf("git remote add: %v - %s", err, stderr) - } - - return initRepoCommit(tmpDir, repo.Owner) -} - func checkInitRepository(repoPath string) (err error) { // Somehow the directory could exist. if com.IsExist(repoPath) { diff --git a/models/repo_generate.go b/models/repo_generate.go index 234bdc27f2eda..0105f2fc234da 100644 --- a/models/repo_generate.go +++ b/models/repo_generate.go @@ -5,8 +5,11 @@ package models import ( + "code.gitea.io/gitea/modules/process" "fmt" + "io/ioutil" "os" + "path" "path/filepath" "strconv" "strings" @@ -36,8 +39,86 @@ func (gro GenerateRepoOptions) IsValid() bool { return gro.GitContent || gro.Topics || gro.GitHooks || gro.Webhooks || gro.Avatar || gro.IssueLabels // or other items as they are added } +func generateRepoCommit(e Engine, repo, templateRepo, generateRepo *Repository, tmpDir string) error { + commitTimeStr := time.Now().Format(time.RFC3339) + authorSig := repo.Owner.NewGitSig() + + // Because this may call hooks we should pass in the environment + env := append(os.Environ(), + "GIT_AUTHOR_NAME="+authorSig.Name, + "GIT_AUTHOR_EMAIL="+authorSig.Email, + "GIT_AUTHOR_DATE="+commitTimeStr, + "GIT_COMMITTER_NAME="+authorSig.Name, + "GIT_COMMITTER_EMAIL="+authorSig.Email, + "GIT_COMMITTER_DATE="+commitTimeStr, + ) + + // Clone to temporary path and do the init commit. + templateRepoPath := templateRepo.repoPath(e) + _, stderr, err := process.GetManager().ExecDirEnv( + -1, "", + fmt.Sprintf("generateRepoCommit(git clone): %s", templateRepoPath), + env, + git.GitExecutable, "clone", "--depth", "1", templateRepoPath, tmpDir, + ) + if err != nil { + return fmt.Errorf("git clone: %v - %s", err, stderr) + } + + if err := os.RemoveAll(path.Join(tmpDir, ".git")); err != nil { + return fmt.Errorf("remove git dir: %v", err) + } + + // Variable expansion in .template files + if err := filepath.Walk(tmpDir, func(path string, info os.FileInfo, walkErr error) error { + if walkErr != nil { + return walkErr + } + + if !strings.HasSuffix(path, ".template") { + return nil + } + + content, err := ioutil.ReadFile(path) + if err != nil { + return err + } + + fi, err := os.Create(strings.TrimSuffix(path, ".template")) + if err != nil { + return err + } + defer fi.Close() + + if _, err := fi.WriteString(generateExpansion(string(content), templateRepo, generateRepo)); err != nil { + return err + } + + return os.Remove(path) + }); err != nil { + return err + } + + if err := git.InitRepository(tmpDir, false); err != nil { + return err + } + + repoPath := repo.repoPath(e) + _, stderr, err = process.GetManager().ExecDirEnv( + -1, tmpDir, + fmt.Sprintf("generateRepoCommit(git remote add): %s", repoPath), + env, + git.GitExecutable, "remote", "add", "origin", repoPath, + ) + if err != nil { + return fmt.Errorf("git remote add: %v - %s", err, stderr) + } + + return initRepoCommit(tmpDir, repo.Owner) +} + // generateRepository initializes repository from template -func generateRepository(e Engine, repo, templateRepo *Repository) (err error) { +func generateRepository(e Engine, repo, templateRepo, generateRepo *Repository) (err error) { tmpDir := filepath.Join(os.TempDir(), "gitea-"+repo.Name+"-"+com.ToStr(time.Now().Nanosecond())) if err := os.MkdirAll(tmpDir, os.ModePerm); err != nil { @@ -50,7 +131,7 @@ func generateRepository(e Engine, repo, templateRepo *Repository) (err error) { } }() - if err = generateRepoCommit(e, repo, templateRepo, tmpDir); err != nil { + if err = generateRepoCommit(e, repo, templateRepo, generateRepo, tmpDir); err != nil { return fmt.Errorf("generateRepoCommit: %v", err) } @@ -95,7 +176,7 @@ func GenerateRepository(ctx DBContext, doer, owner *User, templateRepo *Reposito // GenerateGitContent generates git content from a template repository func GenerateGitContent(ctx DBContext, templateRepo, generateRepo *Repository) error { - if err := generateRepository(ctx.e, generateRepo, templateRepo); err != nil { + if err := generateRepository(ctx.e, generateRepo, templateRepo, generateRepo); err != nil { return err } @@ -144,7 +225,7 @@ func GenerateGitHooks(ctx DBContext, templateRepo, generateRepo *Repository) err return err } - generateHook.Content = templateHook.Content + generateHook.Content = generateExpansion(templateHook.Content, templateRepo, generateRepo) if err := generateHook.Update(); err != nil { return err } @@ -210,3 +291,28 @@ func GenerateIssueLabels(ctx DBContext, templateRepo, generateRepo *Repository) } return nil } + +func generateExpansion(src string, templateRepo, generateRepo *Repository) string { + return os.Expand(src, func(key string) string { + switch key { + case "REPO_NAME": + return generateRepo.Name + case "TEMPLATE_NAME": + return templateRepo.Name + case "REPO_OWNER": + return generateRepo.MustOwnerName() + case "TEMPLATE_OWNER": + return templateRepo.MustOwnerName() + case "REPO_HTTPS_URL": + return generateRepo.CloneLink().HTTPS + case "TEMPLATE_HTTPS_URL": + return templateRepo.CloneLink().HTTPS + case "REPO_SSH_URL": + return generateRepo.CloneLink().SSH + case "TEMPLATE_SSH_URL": + return templateRepo.CloneLink().SSH + default: + return "" + } + }) +} From f308abc69580a389af1a0937dcd2b11fbe25b9a5 Mon Sep 17 00:00:00 2001 From: jolheiser Date: Mon, 25 Nov 2019 22:05:13 -0600 Subject: [PATCH 02/17] _template rather than .template Signed-off-by: jolheiser --- models/repo_generate.go | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/models/repo_generate.go b/models/repo_generate.go index 0105f2fc234da..dc4cf0a3b74f1 100644 --- a/models/repo_generate.go +++ b/models/repo_generate.go @@ -56,7 +56,7 @@ func generateRepoCommit(e Engine, repo, templateRepo, generateRepo *Repository, // Clone to temporary path and do the init commit. templateRepoPath := templateRepo.repoPath(e) _, stderr, err := process.GetManager().ExecDirEnv( - -1, "", + 10*time.Minute, "", fmt.Sprintf("generateRepoCommit(git clone): %s", templateRepoPath), env, git.GitExecutable, "clone", "--depth", "1", templateRepoPath, tmpDir, @@ -69,22 +69,31 @@ func generateRepoCommit(e Engine, repo, templateRepo, generateRepo *Repository, return fmt.Errorf("remove git dir: %v", err) } - // Variable expansion in .template files + // Variable expansion in _template files if err := filepath.Walk(tmpDir, func(path string, info os.FileInfo, walkErr error) error { if walkErr != nil { return walkErr } - if !strings.HasSuffix(path, ".template") { + if info.IsDir() { return nil } + ext := filepath.Ext(path) + pathNoExt := strings.TrimSuffix(path, ext) + + if !strings.HasSuffix(pathNoExt, "_template") { + return nil + } + + newPath := fmt.Sprintf("%s%s", strings.TrimSuffix(pathNoExt, "_template"), ext) + content, err := ioutil.ReadFile(path) if err != nil { return err } - fi, err := os.Create(strings.TrimSuffix(path, ".template")) + fi, err := os.Create(newPath) if err != nil { return err } @@ -303,6 +312,10 @@ func generateExpansion(src string, templateRepo, generateRepo *Repository) strin return generateRepo.MustOwnerName() case "TEMPLATE_OWNER": return templateRepo.MustOwnerName() + case "REPO_LINK": + return generateRepo.Link() + case "TEMPLATE_LINK": + return templateRepo.Link() case "REPO_HTTPS_URL": return generateRepo.CloneLink().HTTPS case "TEMPLATE_HTTPS_URL": @@ -312,7 +325,7 @@ func generateExpansion(src string, templateRepo, generateRepo *Repository) strin case "TEMPLATE_SSH_URL": return templateRepo.CloneLink().SSH default: - return "" + return key } }) } From 047b4c4b14371736c6e7864dce15ca900d496daf Mon Sep 17 00:00:00 2001 From: jolheiser Date: Mon, 25 Nov 2019 22:25:01 -0600 Subject: [PATCH 03/17] Use ioutil Signed-off-by: jolheiser --- models/repo_generate.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/models/repo_generate.go b/models/repo_generate.go index dc4cf0a3b74f1..026745c338d71 100644 --- a/models/repo_generate.go +++ b/models/repo_generate.go @@ -93,13 +93,9 @@ func generateRepoCommit(e Engine, repo, templateRepo, generateRepo *Repository, return err } - fi, err := os.Create(newPath) - if err != nil { - return err - } - defer fi.Close() - - if _, err := fi.WriteString(generateExpansion(string(content), templateRepo, generateRepo)); err != nil { + if err := ioutil.WriteFile(newPath, + []byte(generateExpansion(string(content), templateRepo, generateRepo)), + 0644); err != nil { return err } From cc0eb97fb24e85fdd54a9c0f8ef6989b405ea6f0 Mon Sep 17 00:00:00 2001 From: John Olheiser <42128690+jolheiser@users.noreply.github.com> Date: Tue, 26 Nov 2019 08:55:11 -0600 Subject: [PATCH 04/17] Add descriptions to mapping --- models/repo_generate.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/models/repo_generate.go b/models/repo_generate.go index 026745c338d71..21d7a0195f170 100644 --- a/models/repo_generate.go +++ b/models/repo_generate.go @@ -304,6 +304,10 @@ func generateExpansion(src string, templateRepo, generateRepo *Repository) strin return generateRepo.Name case "TEMPLATE_NAME": return templateRepo.Name + case "REPO_DESCRIPTION": + return generateRepo.Description + case "TEMPLATE_DESCRIPTION": + return templateRepo.Description case "REPO_OWNER": return generateRepo.MustOwnerName() case "TEMPLATE_OWNER": From 9ba2f8e17bdbdaf63394bb6469da32abf9f5fa5e Mon Sep 17 00:00:00 2001 From: jolheiser Date: Tue, 26 Nov 2019 13:31:50 -0600 Subject: [PATCH 05/17] Start globbing Signed-off-by: jolheiser --- models/repo_generate.go | 72 +++++++++++++++++++++++++++++------------ 1 file changed, 51 insertions(+), 21 deletions(-) diff --git a/models/repo_generate.go b/models/repo_generate.go index 21d7a0195f170..5678fbc048162 100644 --- a/models/repo_generate.go +++ b/models/repo_generate.go @@ -7,6 +7,7 @@ package models import ( "code.gitea.io/gitea/modules/process" "fmt" + "github.com/gobwas/glob" "io/ioutil" "os" "path" @@ -39,6 +40,34 @@ func (gro GenerateRepoOptions) IsValid() bool { return gro.GitContent || gro.Topics || gro.GitHooks || gro.Webhooks || gro.Avatar || gro.IssueLabels // or other items as they are added } +func parseGiteaTemplate(tmpDir string) ([]glob.Glob, error) { + globs := make([]glob.Glob, 0) + gtPath := filepath.Join(tmpDir, ".giteatemplate") + if _, err := os.Stat(gtPath); err != nil { + return globs, err + } + content, err := ioutil.ReadFile(gtPath) + if err != nil { + return globs, err + } + + lines := strings.Split(string(content), "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + g, err := glob.Compile(line, '.', '/') + if err != nil { + log.Info("Invalid glob expression '%s' (skipped): %v", line, err) + continue + } + globs = append(globs, g) + } + + return globs, os.Remove(gtPath) +} + func generateRepoCommit(e Engine, repo, templateRepo, generateRepo *Repository, tmpDir string) error { commitTimeStr := time.Now().Format(time.RFC3339) authorSig := repo.Owner.NewGitSig() @@ -69,7 +98,12 @@ func generateRepoCommit(e Engine, repo, templateRepo, generateRepo *Repository, return fmt.Errorf("remove git dir: %v", err) } - // Variable expansion in _template files + // Variable expansion + globs, err := parseGiteaTemplate(tmpDir) + if err != nil { + return fmt.Errorf("parseGiteaTemplate: %v", err) + } + if err := filepath.Walk(tmpDir, func(path string, info os.FileInfo, walkErr error) error { if walkErr != nil { return walkErr @@ -79,27 +113,23 @@ func generateRepoCommit(e Engine, repo, templateRepo, generateRepo *Repository, return nil } - ext := filepath.Ext(path) - pathNoExt := strings.TrimSuffix(path, ext) - - if !strings.HasSuffix(pathNoExt, "_template") { - return nil - } - - newPath := fmt.Sprintf("%s%s", strings.TrimSuffix(pathNoExt, "_template"), ext) - - content, err := ioutil.ReadFile(path) - if err != nil { - return err - } - - if err := ioutil.WriteFile(newPath, - []byte(generateExpansion(string(content), templateRepo, generateRepo)), - 0644); err != nil { - return err + base := strings.TrimPrefix(path, tmpDir) + for _, g := range globs { + if g.Match(base) { + content, err := ioutil.ReadFile(path) + if err != nil { + return err + } + + if err := ioutil.WriteFile(path, + []byte(generateExpansion(string(content), templateRepo, generateRepo)), + 0644); err != nil { + return err + } + break + } } - - return os.Remove(path) + return nil }); err != nil { return err } From 234ec949ac2b357f5ccbd542c80a77e575aa5abb Mon Sep 17 00:00:00 2001 From: jolheiser Date: Wed, 27 Nov 2019 12:18:11 -0600 Subject: [PATCH 06/17] Tune globbing Signed-off-by: jolheiser --- models/repo_generate.go | 76 ++++++++++++++++++++++------------------- modules/git/repo.go | 4 +++ 2 files changed, 44 insertions(+), 36 deletions(-) diff --git a/models/repo_generate.go b/models/repo_generate.go index 5678fbc048162..90704967294e4 100644 --- a/models/repo_generate.go +++ b/models/repo_generate.go @@ -5,7 +5,6 @@ package models import ( - "code.gitea.io/gitea/modules/process" "fmt" "github.com/gobwas/glob" "io/ioutil" @@ -18,6 +17,8 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/process" + "code.gitea.io/gitea/modules/util" "github.com/unknwon/com" ) @@ -43,21 +44,24 @@ func (gro GenerateRepoOptions) IsValid() bool { func parseGiteaTemplate(tmpDir string) ([]glob.Glob, error) { globs := make([]glob.Glob, 0) gtPath := filepath.Join(tmpDir, ".giteatemplate") - if _, err := os.Stat(gtPath); err != nil { + if _, err := os.Stat(gtPath); os.IsNotExist(err) { + return globs, nil + } else if err != nil { return globs, err } + content, err := ioutil.ReadFile(gtPath) if err != nil { return globs, err } - lines := strings.Split(string(content), "\n") + lines := strings.Split(string(util.NormalizeEOL(content)), "\n") for _, line := range lines { line = strings.TrimSpace(line) if line == "" || strings.HasPrefix(line, "#") { continue } - g, err := glob.Compile(line, '.', '/') + g, err := glob.Compile(line, '/') if err != nil { log.Info("Invalid glob expression '%s' (skipped): %v", line, err) continue @@ -84,14 +88,10 @@ func generateRepoCommit(e Engine, repo, templateRepo, generateRepo *Repository, // Clone to temporary path and do the init commit. templateRepoPath := templateRepo.repoPath(e) - _, stderr, err := process.GetManager().ExecDirEnv( - 10*time.Minute, "", - fmt.Sprintf("generateRepoCommit(git clone): %s", templateRepoPath), - env, - git.GitExecutable, "clone", "--depth", "1", templateRepoPath, tmpDir, - ) - if err != nil { - return fmt.Errorf("git clone: %v - %s", err, stderr) + if err := git.Clone(templateRepoPath, tmpDir, git.CloneRepoOptions{ + Depth: 1, + }); err != nil { + return fmt.Errorf("git clone: %v", err) } if err := os.RemoveAll(path.Join(tmpDir, ".git")); err != nil { @@ -104,34 +104,38 @@ func generateRepoCommit(e Engine, repo, templateRepo, generateRepo *Repository, return fmt.Errorf("parseGiteaTemplate: %v", err) } - if err := filepath.Walk(tmpDir, func(path string, info os.FileInfo, walkErr error) error { - if walkErr != nil { - return walkErr - } - - if info.IsDir() { - return nil - } + // Avoid walking tree if there are no globs + if len(globs) > 0 { + tmpDirSlash := strings.TrimSuffix(filepath.ToSlash(tmpDir), "/") + "/" + if err := filepath.Walk(tmpDirSlash, func(path string, info os.FileInfo, walkErr error) error { + if walkErr != nil { + return walkErr + } - base := strings.TrimPrefix(path, tmpDir) - for _, g := range globs { - if g.Match(base) { - content, err := ioutil.ReadFile(path) - if err != nil { - return err - } + if info.IsDir() { + return nil + } - if err := ioutil.WriteFile(path, - []byte(generateExpansion(string(content), templateRepo, generateRepo)), - 0644); err != nil { - return err + base := strings.TrimPrefix(filepath.ToSlash(path), tmpDirSlash) + for _, g := range globs { + if g.Match(base) { + content, err := ioutil.ReadFile(path) + if err != nil { + return err + } + + if err := ioutil.WriteFile(path, + []byte(generateExpansion(string(content), templateRepo, generateRepo)), + 0644); err != nil { + return err + } + break } - break } + return nil + }); err != nil { + return err } - return nil - }); err != nil { - return err } if err := git.InitRepository(tmpDir, false); err != nil { @@ -139,7 +143,7 @@ func generateRepoCommit(e Engine, repo, templateRepo, generateRepo *Repository, } repoPath := repo.repoPath(e) - _, stderr, err = process.GetManager().ExecDirEnv( + _, stderr, err := process.GetManager().ExecDirEnv( -1, tmpDir, fmt.Sprintf("generateRepoCommit(git remote add): %s", repoPath), env, diff --git a/modules/git/repo.go b/modules/git/repo.go index ffca2524be1fe..7d25684265be8 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -161,6 +161,7 @@ type CloneRepoOptions struct { Branch string Shared bool NoCheckout bool + Depth int } // Clone clones original repository to target path. @@ -186,6 +187,9 @@ func Clone(from, to string, opts CloneRepoOptions) (err error) { if opts.NoCheckout { cmd.AddArguments("--no-checkout") } + if opts.Depth > 0 { + cmd.AddArguments("--depth", strconv.Itoa(opts.Depth)) + } if len(opts.Branch) > 0 { cmd.AddArguments("-b", opts.Branch) From 8f1f498a40491a0b0d7b8794fc37f473f88477b0 Mon Sep 17 00:00:00 2001 From: jolheiser Date: Wed, 27 Nov 2019 12:25:07 -0600 Subject: [PATCH 07/17] Re-arrange imports Signed-off-by: jolheiser --- models/repo_generate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/repo_generate.go b/models/repo_generate.go index 90704967294e4..fb868b00180b4 100644 --- a/models/repo_generate.go +++ b/models/repo_generate.go @@ -6,7 +6,6 @@ package models import ( "fmt" - "github.com/gobwas/glob" "io/ioutil" "os" "path" @@ -20,6 +19,7 @@ import ( "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/util" + "github.com/gobwas/glob" "github.com/unknwon/com" ) From 0ab0dafe61388fea2e57f505a83738fc90f653dd Mon Sep 17 00:00:00 2001 From: jolheiser Date: Wed, 27 Nov 2019 21:00:34 -0600 Subject: [PATCH 08/17] Don't expand git hooks Signed-off-by: jolheiser --- models/repo_generate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/repo_generate.go b/models/repo_generate.go index fb868b00180b4..4520b6191deca 100644 --- a/models/repo_generate.go +++ b/models/repo_generate.go @@ -264,7 +264,7 @@ func GenerateGitHooks(ctx DBContext, templateRepo, generateRepo *Repository) err return err } - generateHook.Content = generateExpansion(templateHook.Content, templateRepo, generateRepo) + generateHook.Content = templateHook.Content if err := generateHook.Update(); err != nil { return err } From fa275efe3baca9c2f80be7120b063163f87d6b38 Mon Sep 17 00:00:00 2001 From: jolheiser Date: Wed, 27 Nov 2019 21:31:20 -0600 Subject: [PATCH 09/17] Add glob tests for .giteatemplate Signed-off-by: jolheiser --- models/repo_generate.go | 37 +++++++++++++++++++------ models/repo_generate_test.go | 52 ++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 8 deletions(-) create mode 100644 models/repo_generate_test.go diff --git a/models/repo_generate.go b/models/repo_generate.go index 4520b6191deca..c7b90c7a8e015 100644 --- a/models/repo_generate.go +++ b/models/repo_generate.go @@ -41,20 +41,35 @@ func (gro GenerateRepoOptions) IsValid() bool { return gro.GitContent || gro.Topics || gro.GitHooks || gro.Webhooks || gro.Avatar || gro.IssueLabels // or other items as they are added } -func parseGiteaTemplate(tmpDir string) ([]glob.Glob, error) { - globs := make([]glob.Glob, 0) +type GiteaTemplate struct { + Path string + Content []byte +} + +func checkGiteaTemplate(tmpDir string) (*GiteaTemplate, error) { gtPath := filepath.Join(tmpDir, ".giteatemplate") if _, err := os.Stat(gtPath); os.IsNotExist(err) { - return globs, nil + return nil, nil } else if err != nil { - return globs, err + return nil, err } content, err := ioutil.ReadFile(gtPath) if err != nil { - return globs, err + return nil, err + } + + gt := &GiteaTemplate{ + Path: gtPath, + Content: content, } + return gt, nil +} + +func parseGiteaTemplate(content []byte) []glob.Glob { + globs := make([]glob.Glob, 0) + lines := strings.Split(string(util.NormalizeEOL(content)), "\n") for _, line := range lines { line = strings.TrimSpace(line) @@ -69,7 +84,7 @@ func parseGiteaTemplate(tmpDir string) ([]glob.Glob, error) { globs = append(globs, g) } - return globs, os.Remove(gtPath) + return globs } func generateRepoCommit(e Engine, repo, templateRepo, generateRepo *Repository, tmpDir string) error { @@ -99,11 +114,17 @@ func generateRepoCommit(e Engine, repo, templateRepo, generateRepo *Repository, } // Variable expansion - globs, err := parseGiteaTemplate(tmpDir) + gt, err := checkGiteaTemplate(tmpDir) if err != nil { - return fmt.Errorf("parseGiteaTemplate: %v", err) + return fmt.Errorf("checkGiteaTemplate: %v", err) } + if err := os.Remove(gt.Path); err != nil { + return fmt.Errorf("remove .giteatemplate: %v", err) + } + + globs := parseGiteaTemplate(gt.Content) + // Avoid walking tree if there are no globs if len(globs) > 0 { tmpDirSlash := strings.TrimSuffix(filepath.ToSlash(tmpDir), "/") + "/" diff --git a/models/repo_generate_test.go b/models/repo_generate_test.go new file mode 100644 index 0000000000000..c54115ddc67d9 --- /dev/null +++ b/models/repo_generate_test.go @@ -0,0 +1,52 @@ +package models + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +var giteaTemplate = []byte(` +# Header + +# All .go files +**.go + +# All text files in /text/ +text/*.txt + +# All files in modules folders +**/modules/* +`) + +func TestParseGiteaTemplate(t *testing.T) { + globs := parseGiteaTemplate(giteaTemplate) + assert.Equal(t, len(globs), 3) + + tt := []struct { + Path string + Match bool + }{ + {Path: "main.go", Match: true}, + {Path: "a/b/c/d/e.go", Match: true}, + {Path: "main.txt", Match: false}, + {Path: "a/b.txt", Match: false}, + {Path: "text/a.txt", Match: true}, + {Path: "text/b.txt", Match: true}, + {Path: "text/c.json", Match: false}, + {Path: "a/b/c/modules/README.md", Match: true}, + {Path: "a/b/c/modules/d/README.md", Match: false}, + } + + for _, tc := range tt { + t.Run(tc.Path, func(t *testing.T) { + match := false + for _, g := range globs { + if g.Match(tc.Path) { + match = true + break + } + } + assert.Equal(t, tc.Match, match) + }) + } +} From b2e4d61e0bf7b913b997fae6ff9a3e0a52f03eb7 Mon Sep 17 00:00:00 2001 From: jolheiser Date: Wed, 27 Nov 2019 21:49:23 -0600 Subject: [PATCH 10/17] Parse globs separately so they can be tested more easily Signed-off-by: jolheiser --- models/repo_generate.go | 52 +++++++++++++++++++----------------- models/repo_generate_test.go | 8 +++--- 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/models/repo_generate.go b/models/repo_generate.go index c7b90c7a8e015..d33ddce6fe1b5 100644 --- a/models/repo_generate.go +++ b/models/repo_generate.go @@ -41,9 +41,35 @@ func (gro GenerateRepoOptions) IsValid() bool { return gro.GitContent || gro.Topics || gro.GitHooks || gro.Webhooks || gro.Avatar || gro.IssueLabels // or other items as they are added } +// GiteaTemplate holds information about a .giteatemplate file type GiteaTemplate struct { Path string Content []byte + + globs []glob.Glob +} + +// Globs parses the .giteatemplate globs or returns them if they were already parsed +func (gt GiteaTemplate) Globs() []glob.Glob { + if gt.globs != nil { + return gt.globs + } + + gt.globs = make([]glob.Glob, 0) + lines := strings.Split(string(util.NormalizeEOL(gt.Content)), "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + g, err := glob.Compile(line, '/') + if err != nil { + log.Info("Invalid glob expression '%s' (skipped): %v", line, err) + continue + } + gt.globs = append(gt.globs, g) + } + return gt.globs } func checkGiteaTemplate(tmpDir string) (*GiteaTemplate, error) { @@ -67,26 +93,6 @@ func checkGiteaTemplate(tmpDir string) (*GiteaTemplate, error) { return gt, nil } -func parseGiteaTemplate(content []byte) []glob.Glob { - globs := make([]glob.Glob, 0) - - lines := strings.Split(string(util.NormalizeEOL(content)), "\n") - for _, line := range lines { - line = strings.TrimSpace(line) - if line == "" || strings.HasPrefix(line, "#") { - continue - } - g, err := glob.Compile(line, '/') - if err != nil { - log.Info("Invalid glob expression '%s' (skipped): %v", line, err) - continue - } - globs = append(globs, g) - } - - return globs -} - func generateRepoCommit(e Engine, repo, templateRepo, generateRepo *Repository, tmpDir string) error { commitTimeStr := time.Now().Format(time.RFC3339) authorSig := repo.Owner.NewGitSig() @@ -123,10 +129,8 @@ func generateRepoCommit(e Engine, repo, templateRepo, generateRepo *Repository, return fmt.Errorf("remove .giteatemplate: %v", err) } - globs := parseGiteaTemplate(gt.Content) - // Avoid walking tree if there are no globs - if len(globs) > 0 { + if len(gt.Globs()) > 0 { tmpDirSlash := strings.TrimSuffix(filepath.ToSlash(tmpDir), "/") + "/" if err := filepath.Walk(tmpDirSlash, func(path string, info os.FileInfo, walkErr error) error { if walkErr != nil { @@ -138,7 +142,7 @@ func generateRepoCommit(e Engine, repo, templateRepo, generateRepo *Repository, } base := strings.TrimPrefix(filepath.ToSlash(path), tmpDirSlash) - for _, g := range globs { + for _, g := range gt.Globs() { if g.Match(base) { content, err := ioutil.ReadFile(path) if err != nil { diff --git a/models/repo_generate_test.go b/models/repo_generate_test.go index c54115ddc67d9..d10ebcc1d323d 100644 --- a/models/repo_generate_test.go +++ b/models/repo_generate_test.go @@ -18,9 +18,9 @@ text/*.txt **/modules/* `) -func TestParseGiteaTemplate(t *testing.T) { - globs := parseGiteaTemplate(giteaTemplate) - assert.Equal(t, len(globs), 3) +func TestGiteaTemplate(t *testing.T) { + gt := GiteaTemplate{Content: giteaTemplate} + assert.Equal(t, len(gt.Globs()), 3) tt := []struct { Path string @@ -40,7 +40,7 @@ func TestParseGiteaTemplate(t *testing.T) { for _, tc := range tt { t.Run(tc.Path, func(t *testing.T) { match := false - for _, g := range globs { + for _, g := range gt.Globs() { if g.Match(tc.Path) { match = true break From 8e3e0953026f8538d60c3e430fbdd6b6daf50ec4 Mon Sep 17 00:00:00 2001 From: jolheiser Date: Thu, 28 Nov 2019 20:26:55 -0600 Subject: [PATCH 11/17] Change template location and add docs Signed-off-by: jolheiser --- docs/content/doc/features/gitea-directory.md | 53 ++++++++++++++++++++ models/repo_generate.go | 2 +- 2 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 docs/content/doc/features/gitea-directory.md diff --git a/docs/content/doc/features/gitea-directory.md b/docs/content/doc/features/gitea-directory.md new file mode 100644 index 0000000000000..08775e3a8997e --- /dev/null +++ b/docs/content/doc/features/gitea-directory.md @@ -0,0 +1,53 @@ +--- +date: "2019-11-28:00:00+02:00" +title: "The .gitea Directory" +slug: "gitea-directory" +weight: 40 +toc: true +draft: false +menu: + sidebar: + parent: "features" + name: "The .gitea Directory" + weight: 50 + identifier: "gitea-directory" +--- + +# The .gitea directory +Gitea repositories can include a `.gitea` directory at their base which will store settings/configurations for certain features. + +## Templates +Gitea v1.11.0 includes template repositories, and one feature implemented with them is auto-expansion of specific variables within your template files. +To tell Gitea which files to expand, you must include a `template` file inside the `.gitea` directory of the template repository. +Gitea uses [gobwas/glob](https://github.com/gobwas/glob) for its glob syntax. It closely resembles a traditional `.gitignore`, however there may be slight differences. + +### Example `.gitea/template` file +All paths should be from the base of the repository +```gitignore +# All .go files, anywhere in the repository +**.go + +# All text files in the text directory +text/*.txt + +# A specific file +a/b/c/d.json +``` +**NOTE:** The `template` file will be removed from the `.gitea` directory when a repository is generated from the template. + +### Variable Expansion +In any file matched by the above globs, certain variables will be expanded. +All variables must be of the form `$VAR` or `${VAR}`. To escape an expansion, use a double `$$`, such as `$$VAR` or `$${VAR}` + +| Variable | Expands To | +|----------------------|-----------------------------------------------------| +| REPO_NAME | The name of the generated repository | +| TEMPLATE_NAME | The name of the template repository | +| REPO_DESCRIPTION | The description of the generated repository | +| TEMPLATE_DESCRIPTION | The description of the template repository | +| REPO_LINK | The URL to the generated repository | +| TEMPLATE_LINK | The URL to the template repository | +| REPO_HTTPS_URL | The HTTP(S) clone link for the generated repository | +| TEMPLATE_HTTPS_URL | The HTTP(S) clone link for the template repository | +| REPO_SSH_URL | The SSH clone link for the generated repository | +| TEMPLATE_SSH_URL | The SSH clone link for the template repository | diff --git a/models/repo_generate.go b/models/repo_generate.go index d33ddce6fe1b5..0636674e019b4 100644 --- a/models/repo_generate.go +++ b/models/repo_generate.go @@ -73,7 +73,7 @@ func (gt GiteaTemplate) Globs() []glob.Glob { } func checkGiteaTemplate(tmpDir string) (*GiteaTemplate, error) { - gtPath := filepath.Join(tmpDir, ".giteatemplate") + gtPath := filepath.Join(tmpDir, ".gitea", "template") if _, err := os.Stat(gtPath); os.IsNotExist(err) { return nil, nil } else if err != nil { From 57cf49a56e6eacd89c5a55ce89a5cac743bbe8cc Mon Sep 17 00:00:00 2001 From: jolheiser Date: Thu, 28 Nov 2019 20:30:21 -0600 Subject: [PATCH 12/17] nit Signed-off-by: jolheiser --- models/repo_generate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/repo_generate.go b/models/repo_generate.go index 0636674e019b4..e5a3070d8d471 100644 --- a/models/repo_generate.go +++ b/models/repo_generate.go @@ -49,7 +49,7 @@ type GiteaTemplate struct { globs []glob.Glob } -// Globs parses the .giteatemplate globs or returns them if they were already parsed +// Globs parses the .gitea/template globs or returns them if they were already parsed func (gt GiteaTemplate) Globs() []glob.Glob { if gt.globs != nil { return gt.globs From f7d55ff87b4939d94178950d9af948d297837cf9 Mon Sep 17 00:00:00 2001 From: John Olheiser <42128690+jolheiser@users.noreply.github.com> Date: Thu, 28 Nov 2019 21:11:45 -0600 Subject: [PATCH 13/17] Update docs/content/doc/features/gitea-directory.md Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> --- docs/content/doc/features/gitea-directory.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/doc/features/gitea-directory.md b/docs/content/doc/features/gitea-directory.md index 08775e3a8997e..e2ef3d2b94e8c 100644 --- a/docs/content/doc/features/gitea-directory.md +++ b/docs/content/doc/features/gitea-directory.md @@ -22,7 +22,7 @@ To tell Gitea which files to expand, you must include a `template` file inside t Gitea uses [gobwas/glob](https://github.com/gobwas/glob) for its glob syntax. It closely resembles a traditional `.gitignore`, however there may be slight differences. ### Example `.gitea/template` file -All paths should be from the base of the repository +All paths are relative to the base of the repository ```gitignore # All .go files, anywhere in the repository **.go From 98cba56f032c91df0bbfd75893e214a98f698c47 Mon Sep 17 00:00:00 2001 From: John Olheiser <42128690+jolheiser@users.noreply.github.com> Date: Thu, 28 Nov 2019 21:11:56 -0600 Subject: [PATCH 14/17] Update docs/content/doc/features/gitea-directory.md Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> --- docs/content/doc/features/gitea-directory.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/doc/features/gitea-directory.md b/docs/content/doc/features/gitea-directory.md index e2ef3d2b94e8c..15204dc3ebaaf 100644 --- a/docs/content/doc/features/gitea-directory.md +++ b/docs/content/doc/features/gitea-directory.md @@ -17,7 +17,7 @@ menu: Gitea repositories can include a `.gitea` directory at their base which will store settings/configurations for certain features. ## Templates -Gitea v1.11.0 includes template repositories, and one feature implemented with them is auto-expansion of specific variables within your template files. +Gitea includes template repositories, and one feature implemented with them is auto-expansion of specific variables within your template files. To tell Gitea which files to expand, you must include a `template` file inside the `.gitea` directory of the template repository. Gitea uses [gobwas/glob](https://github.com/gobwas/glob) for its glob syntax. It closely resembles a traditional `.gitignore`, however there may be slight differences. From b98f1ea27aa3fc89249faec395f5692b1ba09032 Mon Sep 17 00:00:00 2001 From: jolheiser Date: Thu, 28 Nov 2019 21:14:32 -0600 Subject: [PATCH 15/17] Add upper-lower case match Signed-off-by: jolheiser --- docs/content/doc/features/gitea-directory.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/content/doc/features/gitea-directory.md b/docs/content/doc/features/gitea-directory.md index 15204dc3ebaaf..e598969bcd45d 100644 --- a/docs/content/doc/features/gitea-directory.md +++ b/docs/content/doc/features/gitea-directory.md @@ -32,6 +32,9 @@ text/*.txt # A specific file a/b/c/d.json + +# Batch files in both upper or lower case can be matched +**.[bB][aA][tT] ``` **NOTE:** The `template` file will be removed from the `.gitea` directory when a repository is generated from the template. From 981e18294e5a599dee5ac9cf76926d7ed6a03ec4 Mon Sep 17 00:00:00 2001 From: jolheiser Date: Thu, 28 Nov 2019 21:26:19 -0600 Subject: [PATCH 16/17] Nits Signed-off-by: jolheiser --- models/repo_generate_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/models/repo_generate_test.go b/models/repo_generate_test.go index d10ebcc1d323d..53ab4fcd3d559 100644 --- a/models/repo_generate_test.go +++ b/models/repo_generate_test.go @@ -1,8 +1,13 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + package models import ( - "github.com/stretchr/testify/assert" "testing" + + "github.com/stretchr/testify/assert" ) var giteaTemplate = []byte(` From 98f84002ce1a8d68bde8dab8db7769773b580f94 Mon Sep 17 00:00:00 2001 From: John Olheiser <42128690+jolheiser@users.noreply.github.com> Date: Fri, 29 Nov 2019 07:58:37 -0600 Subject: [PATCH 17/17] Update models/repo_generate.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> --- models/repo_generate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/repo_generate.go b/models/repo_generate.go index e5a3070d8d471..56a3940ac18c2 100644 --- a/models/repo_generate.go +++ b/models/repo_generate.go @@ -41,7 +41,7 @@ func (gro GenerateRepoOptions) IsValid() bool { return gro.GitContent || gro.Topics || gro.GitHooks || gro.Webhooks || gro.Avatar || gro.IssueLabels // or other items as they are added } -// GiteaTemplate holds information about a .giteatemplate file +// GiteaTemplate holds information about a .gitea/template file type GiteaTemplate struct { Path string Content []byte