From 3edfaaf8b7857d83231cdca104a6cc48fc44ee71 Mon Sep 17 00:00:00 2001 From: Matti R Date: Mon, 1 Jun 2020 23:44:07 -0400 Subject: [PATCH 1/4] refactor repo_stats to use os.pipe --- modules/git/repo_stats.go | 150 +++++++++++++++++++++----------------- 1 file changed, 84 insertions(+), 66 deletions(-) diff --git a/modules/git/repo_stats.go b/modules/git/repo_stats.go index bfa368b6dfb7d..b9300652a7429 100644 --- a/modules/git/repo_stats.go +++ b/modules/git/repo_stats.go @@ -6,8 +6,9 @@ package git import ( "bufio" - "bytes" + "context" "fmt" + "os" "sort" "strconv" "strings" @@ -49,6 +50,15 @@ func (repo *Repository) GetCodeActivityStats(fromTime time.Time, branch string) } stats.CommitCountInAllBranches = c + stderrReader, stderrWriter, err := os.Pipe() + if err != nil { + return nil, fmt.Errorf("Unable to open stderr pipe: %v", err) + } + defer func() { + _ = stderrReader.Close() + _ = stderrWriter.Close() + }() + args := []string{"log", "--numstat", "--no-merges", "--pretty=format:---%n%h%n%an%n%ae%n", "--date=iso", fmt.Sprintf("--since='%s'", since)} if len(branch) == 0 { args = append(args, "--branches=*") @@ -56,79 +66,87 @@ func (repo *Repository) GetCodeActivityStats(fromTime time.Time, branch string) args = append(args, "--first-parent", branch) } - stdout, err = NewCommand(args...).RunInDirBytes(repo.Path) - if err != nil { - return nil, err - } + err = NewCommand(args...).RunInDirTimeoutEnvFullPipelineFunc( + nil, -1, repo.Path, + nil, stderrWriter, nil, + func(ctx context.Context, cancel context.CancelFunc) error { + _ = stderrWriter.Close() - scanner := bufio.NewScanner(bytes.NewReader(stdout)) - scanner.Split(bufio.ScanLines) - stats.CommitCount = 0 - stats.Additions = 0 - stats.Deletions = 0 - authors := make(map[string]*CodeActivityAuthor) - files := make(map[string]bool) - var author string - p := 0 - for scanner.Scan() { - l := strings.TrimSpace(scanner.Text()) - if l == "---" { - p = 1 - } else if p == 0 { - continue - } else { - p++ - } - if p > 4 && len(l) == 0 { - continue - } - switch p { - case 1: // Separator - case 2: // Commit sha-1 - stats.CommitCount++ - case 3: // Author - author = l - case 4: // E-mail - email := strings.ToLower(l) - if _, ok := authors[email]; !ok { - authors[email] = &CodeActivityAuthor{ - Name: author, - Email: email, - Commits: 0, + scanner := bufio.NewScanner(stderrReader) + scanner.Split(bufio.ScanLines) + stats.CommitCount = 0 + stats.Additions = 0 + stats.Deletions = 0 + authors := make(map[string]*CodeActivityAuthor) + files := make(map[string]bool) + var author string + p := 0 + for scanner.Scan() { + l := strings.TrimSpace(scanner.Text()) + if l == "---" { + p = 1 + } else if p == 0 { + continue + } else { + p++ } - } - authors[email].Commits++ - default: // Changed file - if parts := strings.Fields(l); len(parts) >= 3 { - if parts[0] != "-" { - if c, err := strconv.ParseInt(strings.TrimSpace(parts[0]), 10, 64); err == nil { - stats.Additions += c - } + if p > 4 && len(l) == 0 { + continue } - if parts[1] != "-" { - if c, err := strconv.ParseInt(strings.TrimSpace(parts[1]), 10, 64); err == nil { - stats.Deletions += c + switch p { + case 1: // Separator + case 2: // Commit sha-1 + stats.CommitCount++ + case 3: // Author + author = l + case 4: // E-mail + email := strings.ToLower(l) + if _, ok := authors[email]; !ok { + authors[email] = &CodeActivityAuthor{ + Name: author, + Email: email, + Commits: 0, + } + } + authors[email].Commits++ + default: // Changed file + if parts := strings.Fields(l); len(parts) >= 3 { + if parts[0] != "-" { + if c, err := strconv.ParseInt(strings.TrimSpace(parts[0]), 10, 64); err == nil { + stats.Additions += c + } + } + if parts[1] != "-" { + if c, err := strconv.ParseInt(strings.TrimSpace(parts[1]), 10, 64); err == nil { + stats.Deletions += c + } + } + if _, ok := files[parts[2]]; !ok { + files[parts[2]] = true + } } - } - if _, ok := files[parts[2]]; !ok { - files[parts[2]] = true } } - } - } - a := make([]*CodeActivityAuthor, 0, len(authors)) - for _, v := range authors { - a = append(a, v) + a := make([]*CodeActivityAuthor, 0, len(authors)) + for _, v := range authors { + a = append(a, v) + } + // Sort authors descending depending on commit count + sort.Slice(a, func(i, j int) bool { + return a[i].Commits > a[j].Commits + }) + + stats.AuthorCount = int64(len(authors)) + stats.ChangedFiles = int64(len(files)) + stats.Authors = a + + _ = stderrReader.Close() + return nil + }) + if err != nil { + return nil, err } - // Sort authors descending depending on commit count - sort.Slice(a, func(i, j int) bool { - return a[i].Commits > a[j].Commits - }) - - stats.AuthorCount = int64(len(authors)) - stats.ChangedFiles = int64(len(files)) - stats.Authors = a return stats, nil } From f767742973342917f06e0b2572ca66c21a6aa7b0 Mon Sep 17 00:00:00 2001 From: Matti R Date: Tue, 2 Jun 2020 00:10:15 -0400 Subject: [PATCH 2/4] woops. missing reader. --- modules/git/repo_stats.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/git/repo_stats.go b/modules/git/repo_stats.go index b9300652a7429..13e56345d1db7 100644 --- a/modules/git/repo_stats.go +++ b/modules/git/repo_stats.go @@ -68,7 +68,7 @@ func (repo *Repository) GetCodeActivityStats(fromTime time.Time, branch string) err = NewCommand(args...).RunInDirTimeoutEnvFullPipelineFunc( nil, -1, repo.Path, - nil, stderrWriter, nil, + nil, stderrWriter, stderrReader, func(ctx context.Context, cancel context.CancelFunc) error { _ = stderrWriter.Close() From aefccc85f214b0db53064a2c1ccd3177829e95fd Mon Sep 17 00:00:00 2001 From: Matti R Date: Tue, 2 Jun 2020 00:37:18 -0400 Subject: [PATCH 3/4] stdout not stderr, woops --- modules/git/command.go | 1 + modules/git/repo_stats.go | 16 ++++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/modules/git/command.go b/modules/git/command.go index 14fab4ef3c8eb..fd4c985e89962 100644 --- a/modules/git/command.go +++ b/modules/git/command.go @@ -1,4 +1,5 @@ // Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2017 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. diff --git a/modules/git/repo_stats.go b/modules/git/repo_stats.go index 13e56345d1db7..27e24b3c5ef12 100644 --- a/modules/git/repo_stats.go +++ b/modules/git/repo_stats.go @@ -50,13 +50,13 @@ func (repo *Repository) GetCodeActivityStats(fromTime time.Time, branch string) } stats.CommitCountInAllBranches = c - stderrReader, stderrWriter, err := os.Pipe() + stdoutReader, stdoutWriter, err := os.Pipe() if err != nil { - return nil, fmt.Errorf("Unable to open stderr pipe: %v", err) + return nil, err } defer func() { - _ = stderrReader.Close() - _ = stderrWriter.Close() + _ = stdoutReader.Close() + _ = stdoutWriter.Close() }() args := []string{"log", "--numstat", "--no-merges", "--pretty=format:---%n%h%n%an%n%ae%n", "--date=iso", fmt.Sprintf("--since='%s'", since)} @@ -68,11 +68,11 @@ func (repo *Repository) GetCodeActivityStats(fromTime time.Time, branch string) err = NewCommand(args...).RunInDirTimeoutEnvFullPipelineFunc( nil, -1, repo.Path, - nil, stderrWriter, stderrReader, + stdoutWriter, nil, nil, func(ctx context.Context, cancel context.CancelFunc) error { - _ = stderrWriter.Close() + _ = stdoutWriter.Close() - scanner := bufio.NewScanner(stderrReader) + scanner := bufio.NewScanner(stdoutReader) scanner.Split(bufio.ScanLines) stats.CommitCount = 0 stats.Additions = 0 @@ -141,7 +141,7 @@ func (repo *Repository) GetCodeActivityStats(fromTime time.Time, branch string) stats.ChangedFiles = int64(len(files)) stats.Authors = a - _ = stderrReader.Close() + _ = stdoutReader.Close() return nil }) if err != nil { From bcd50d031c0c4068d7f3b482ebce8aef8d0be66f Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 5 Sep 2020 20:41:19 +0100 Subject: [PATCH 4/4] Fix copyright date and ensure that the stderr is collected Signed-off-by: Andrew Thornton --- modules/git/command.go | 3 +-- modules/git/repo_stats.go | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/git/command.go b/modules/git/command.go index 2cbc2864dc76f..d40c0bfa2322b 100644 --- a/modules/git/command.go +++ b/modules/git/command.go @@ -1,6 +1,5 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. // Copyright 2015 The Gogs Authors. All rights reserved. -// Copyright 2017 The Gitea Authors. All rights reserved. +// Copyright 2016 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. diff --git a/modules/git/repo_stats.go b/modules/git/repo_stats.go index 27e24b3c5ef12..59b8177401cca 100644 --- a/modules/git/repo_stats.go +++ b/modules/git/repo_stats.go @@ -66,9 +66,10 @@ func (repo *Repository) GetCodeActivityStats(fromTime time.Time, branch string) args = append(args, "--first-parent", branch) } + stderr := new(strings.Builder) err = NewCommand(args...).RunInDirTimeoutEnvFullPipelineFunc( nil, -1, repo.Path, - stdoutWriter, nil, nil, + stdoutWriter, stderr, nil, func(ctx context.Context, cancel context.CancelFunc) error { _ = stdoutWriter.Close() @@ -145,7 +146,7 @@ func (repo *Repository) GetCodeActivityStats(fromTime time.Time, branch string) return nil }) if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to get GetCodeActivityStats for repository.\nError: %w\nStderr: %s", err, stderr) } return stats, nil