Merge remote-tracking branch 'giteaofficial/main'

* giteaofficial/main:
  [skip ci] Updated translations via Crowdin
  Add option to disable ambiguous unicode characters detection (go-gitea#28454)
  Adjust object format interface (go-gitea#28469)
  Remove duplicate option in admin screen and now-unused translation keys (go-gitea#28492)
  [skip ci] Updated translations via Crowdin
  Initalize stroage for orphaned repository doctor (go-gitea#28487)

Author: zjjhot

custom/conf/app.example.ini

@@ -1212,6 +1212,9 @@ LEVEL = Info
 ;; Max size of files to be displayed (default is 8MiB)
 ;MAX_DISPLAY_FILE_SIZE = 8388608
 ;;
+;; Detect ambiguous unicode characters in file contents and show warnings on the UI
+;AMBIGUOUS_UNICODE_DETECTION = true
+;;
 ;; Whether the email of the user should be shown in the Explore Users page
 ;SHOW_USER_EMAIL = true
 ;;

docs/content/administration/config-cheat-sheet.en-us.md

@@ -220,6 +220,7 @@ The following configuration set `Content-Type: application/vnd.android.package-a
 - `THEMES`:  **gitea-auto,gitea-light,gitea-dark**: All available themes. Allow users select personalized themes.
   regardless of the value of `DEFAULT_THEME`.
 - `MAX_DISPLAY_FILE_SIZE`: **8388608**: Max size of files to be displayed (default is 8MiB)
+- `AMBIGUOUS_UNICODE_DETECTION`: **true**: Detect ambiguous unicode characters in file contents and show warnings on the UI
 - `REACTIONS`: All available reactions users can choose on issues/prs and comments
     Values can be emoji alias (:smile:) or a unicode emoji.
     For custom reactions, add a tightly cropped square image to public/assets/img/emoji/reaction_name.png

models/git/branch_test.go

@@ -20,6 +20,7 @@ import (
 func TestAddDeletedBranch(t *testing.T) {
 	assert.NoError(t, unittest.PrepareTestDatabase())
 	repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1})
+	assert.EqualValues(t, git.Sha1ObjectFormat.Name(), repo.ObjectFormatName)
 	firstBranch := unittest.AssertExistsAndLoadBean(t, &git_model.Branch{ID: 1})
 
 	assert.True(t, firstBranch.IsDeleted)
@@ -29,8 +30,9 @@ func TestAddDeletedBranch(t *testing.T) {
 	secondBranch := unittest.AssertExistsAndLoadBean(t, &git_model.Branch{RepoID: repo.ID, Name: "branch2"})
 	assert.True(t, secondBranch.IsDeleted)
 
+	objectFormat := git.ObjectFormatFromName(repo.ObjectFormatName)
 	commit := &git.Commit{
-		ID:            repo.ObjectFormat.MustIDFromString(secondBranch.CommitID),
+		ID:            objectFormat.MustIDFromString(secondBranch.CommitID),
 		CommitMessage: secondBranch.CommitMessage,
 		Committer: &git.Signature{
 			When: secondBranch.CommitTime.AsLocalTime(),

models/repo/repo.go

@@ -180,7 +180,7 @@ type Repository struct {
 	IsFsckEnabled                   bool               `xorm:"NOT NULL DEFAULT true"`
 	CloseIssuesViaCommitInAnyBranch bool               `xorm:"NOT NULL DEFAULT false"`
 	Topics                          []string           `xorm:"TEXT JSON"`
-	ObjectFormat                    git.ObjectFormat   `xorm:"-"`
+	ObjectFormatName                string             `xorm:"-"`
 
 	TrustModel TrustModelType
 
@@ -277,7 +277,9 @@ func (repo *Repository) AfterLoad() {
 	repo.NumOpenProjects = repo.NumProjects - repo.NumClosedProjects
 	repo.NumOpenActionRuns = repo.NumActionRuns - repo.NumClosedActionRuns
 
-	repo.ObjectFormat = git.ObjectFormatFromID(git.Sha1)
+	// this is a temporary behaviour to support old repos, next step is to store the object format in the database
+	// and read from database so this line could be removed. To not depend on git module, we use a constant variable here
+	repo.ObjectFormatName = "sha1"
 }
 
 // LoadAttributes loads attributes of the repository.

modules/charset/escape.go

@@ -8,32 +8,31 @@
 package charset
 
 import (
-	"bufio"
+	"html/template"
 	"io"
 	"strings"
 
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/translation"
 )
 
 // RuneNBSP is the codepoint for NBSP
 const RuneNBSP = 0xa0
 
 // EscapeControlHTML escapes the unicode control sequences in a provided html document
-func EscapeControlHTML(text string, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, output string) {
+func EscapeControlHTML(html template.HTML, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, output template.HTML) {
 	sb := &strings.Builder{}
-	outputStream := &HTMLStreamerWriter{Writer: sb}
-	streamer := NewEscapeStreamer(locale, outputStream, allowed...).(*escapeStreamer)
-
-	if err := StreamHTML(strings.NewReader(text), streamer); err != nil {
-		streamer.escaped.HasError = true
-		log.Error("Error whilst escaping: %v", err)
-	}
-	return streamer.escaped, sb.String()
+	escaped, _ = EscapeControlReader(strings.NewReader(string(html)), sb, locale, allowed...) // err has been handled in EscapeControlReader
+	return escaped, template.HTML(sb.String())
 }
 
-// EscapeControlReaders escapes the unicode control sequences in a provided reader of HTML content and writer in a locale and returns the findings as an EscapeStatus and the escaped []byte
+// EscapeControlReader escapes the unicode control sequences in a provided reader of HTML content and writer in a locale and returns the findings as an EscapeStatus
 func EscapeControlReader(reader io.Reader, writer io.Writer, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, err error) {
+	if !setting.UI.AmbiguousUnicodeDetection {
+		_, err = io.Copy(writer, reader)
+		return &EscapeStatus{}, err
+	}
 	outputStream := &HTMLStreamerWriter{Writer: writer}
 	streamer := NewEscapeStreamer(locale, outputStream, allowed...).(*escapeStreamer)
 
@@ -43,41 +42,3 @@ func EscapeControlReader(reader io.Reader, writer io.Writer, locale translation.
 	}
 	return streamer.escaped, err
 }
-
-// EscapeControlStringReader escapes the unicode control sequences in a provided reader of string content and writer in a locale and returns the findings as an EscapeStatus and the escaped []byte. HTML line breaks are not inserted after every newline by this method.
-func EscapeControlStringReader(reader io.Reader, writer io.Writer, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, err error) {
-	bufRd := bufio.NewReader(reader)
-	outputStream := &HTMLStreamerWriter{Writer: writer}
-	streamer := NewEscapeStreamer(locale, outputStream, allowed...).(*escapeStreamer)
-
-	for {
-		line, rdErr := bufRd.ReadString('\n')
-		if len(line) > 0 {
-			if err := streamer.Text(line); err != nil {
-				streamer.escaped.HasError = true
-				log.Error("Error whilst escaping: %v", err)
-				return streamer.escaped, err
-			}
-		}
-		if rdErr != nil {
-			if rdErr != io.EOF {
-				err = rdErr
-			}
-			break
-		}
-	}
-	return streamer.escaped, err
-}
-
-// EscapeControlString escapes the unicode control sequences in a provided string and returns the findings as an EscapeStatus and the escaped string
-func EscapeControlString(text string, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, output string) {
-	sb := &strings.Builder{}
-	outputStream := &HTMLStreamerWriter{Writer: sb}
-	streamer := NewEscapeStreamer(locale, outputStream, allowed...).(*escapeStreamer)
-
-	if err := streamer.Text(text); err != nil {
-		streamer.escaped.HasError = true
-		log.Error("Error whilst escaping: %v", err)
-	}
-	return streamer.escaped, sb.String()
-}

modules/charset/escape_stream.go

@@ -64,7 +64,7 @@ func (e *escapeStreamer) Text(data string) error {
 			until, next = nextIdxs[0]+pos, nextIdxs[1]+pos
 		}
 
-		// from pos until until we know that the runes are not \r\t\n or even ' '
+		// from pos until we know that the runes are not \r\t\n or even ' '
 		runes := make([]rune, 0, next-until)
 		positions := make([]int, 0, next-until+1)
 

modules/charset/escape_test.go

@@ -4,11 +4,14 @@
 package charset
 
 import (
-	"reflect"
 	"strings"
 	"testing"
 
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/test"
 	"code.gitea.io/gitea/modules/translation"
+
+	"github.com/stretchr/testify/assert"
 )
 
 type escapeControlTest struct {
@@ -132,22 +135,8 @@ then resh (ר), and finally heh (ה) (which should appear leftmost).`,
 	},
 }
 
-func TestEscapeControlString(t *testing.T) {
-	for _, tt := range escapeControlTests {
-		t.Run(tt.name, func(t *testing.T) {
-			status, result := EscapeControlString(tt.text, &translation.MockLocale{})
-			if !reflect.DeepEqual(*status, tt.status) {
-				t.Errorf("EscapeControlString() status = %v, wanted= %v", status, tt.status)
-			}
-			if result != tt.result {
-				t.Errorf("EscapeControlString()\nresult= %v,\nwanted= %v", result, tt.result)
-			}
-		})
-	}
-}
-
 func TestEscapeControlReader(t *testing.T) {
-	// lets add some control characters to the tests
+	// add some control characters to the tests
 	tests := make([]escapeControlTest, 0, len(escapeControlTests)*3)
 	copy(tests, escapeControlTests)
 
@@ -169,29 +158,20 @@ func TestEscapeControlReader(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			input := strings.NewReader(tt.text)
 			output := &strings.Builder{}
-			status, err := EscapeControlReader(input, output, &translation.MockLocale{})
-			result := output.String()
-			if err != nil {
-				t.Errorf("EscapeControlReader(): err = %v", err)
-			}
-
-			if !reflect.DeepEqual(*status, tt.status) {
-				t.Errorf("EscapeControlReader() status = %v, wanted= %v", status, tt.status)
-			}
-			if result != tt.result {
-				t.Errorf("EscapeControlReader()\nresult= %v,\nwanted= %v", result, tt.result)
-			}
+			status, err := EscapeControlReader(strings.NewReader(tt.text), output, &translation.MockLocale{})
+			assert.NoError(t, err)
+			assert.Equal(t, tt.status, *status)
+			assert.Equal(t, tt.result, output.String())
 		})
 	}
 }
 
-func TestEscapeControlReader_panic(t *testing.T) {
-	bs := make([]byte, 0, 20479)
-	bs = append(bs, 'A')
-	for i := 0; i < 6826; i++ {
-		bs = append(bs, []byte("—")...)
-	}
-	_, _ = EscapeControlString(string(bs), &translation.MockLocale{})
+func TestSettingAmbiguousUnicodeDetection(t *testing.T) {
+	defer test.MockVariableValue(&setting.UI.AmbiguousUnicodeDetection, true)()
+	_, out := EscapeControlHTML("a test", &translation.MockLocale{})
+	assert.EqualValues(t, `a<span class="escaped-code-point" data-escaped="[U+00A0]"><span class="char"> </span></span>test`, out)
+	setting.UI.AmbiguousUnicodeDetection = false
+	_, out = EscapeControlHTML("a test", &translation.MockLocale{})
+	assert.EqualValues(t, `a test`, out)
 }

modules/doctor/repository.go

@@ -9,6 +9,7 @@ import (
 	"code.gitea.io/gitea/models/db"
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/storage"
 	repo_service "code.gitea.io/gitea/services/repository"
 
 	"xorm.io/builder"
@@ -31,6 +32,10 @@ func countOrphanedRepos(ctx context.Context) (int64, error) {
 
 // deleteOrphanedRepos delete repository where user of owner_id do not exist
 func deleteOrphanedRepos(ctx context.Context) (int64, error) {
+	if err := storage.Init(); err != nil {
+		return 0, err
+	}
+
 	batchSize := db.MaxBatchInsertSize("repository")
 	e := db.GetEngine(ctx)
 	var deleted int64

modules/git/blame_test.go

@@ -39,7 +39,7 @@ func TestReadingBlameOutput(t *testing.T) {
 		}
 
 		for _, bypass := range []bool{false, true} {
-			blameReader, err := CreateBlameReader(ctx, &Sha1ObjectFormat{}, "./tests/repos/repo5_pulls", commit, "README.md", bypass)
+			blameReader, err := CreateBlameReader(ctx, Sha1ObjectFormat, "./tests/repos/repo5_pulls", commit, "README.md", bypass)
 			assert.NoError(t, err)
 			assert.NotNil(t, blameReader)
 			defer blameReader.Close()

modules/git/command.go

@@ -14,7 +14,6 @@ import (
 	"os/exec"
 	"strings"
 	"time"
-	"unsafe"
 
 	"code.gitea.io/gitea/modules/git/internal" //nolint:depguard // only this file can use the internal type CmdArg, other files and packages should use AddXxx functions
 	"code.gitea.io/gitea/modules/log"
@@ -389,15 +388,11 @@ func (r *runStdError) IsExitCode(code int) bool {
 	return false
 }
 
-func bytesToString(b []byte) string {
-	return *(*string)(unsafe.Pointer(&b)) // that's what Golang's strings.Builder.String() does (go/src/strings/builder.go)
-}
-
 // RunStdString runs the command with options and returns stdout/stderr as string. and store stderr to returned error (err combined with stderr).
 func (c *Command) RunStdString(opts *RunOpts) (stdout, stderr string, runErr RunStdError) {
 	stdoutBytes, stderrBytes, err := c.RunStdBytes(opts)
-	stdout = bytesToString(stdoutBytes)
-	stderr = bytesToString(stderrBytes)
+	stdout = util.UnsafeBytesToString(stdoutBytes)
+	stderr = util.UnsafeBytesToString(stderrBytes)
 	if err != nil {
 		return stdout, stderr, &runStdError{err: err, stderr: stderr}
 	}
@@ -432,7 +427,7 @@ func (c *Command) RunStdBytes(opts *RunOpts) (stdout, stderr []byte, runErr RunS
 	err := c.Run(newOpts)
 	stderr = stderrBuf.Bytes()
 	if err != nil {
-		return nil, stderr, &runStdError{err: err, stderr: bytesToString(stderr)}
+		return nil, stderr, &runStdError{err: err, stderr: util.UnsafeBytesToString(stderr)}
 	}
 	// even if there is no err, there could still be some stderr output
 	return stdoutBuf.Bytes(), stderr, nil

modules/git/commit.go

@@ -236,7 +236,7 @@ func (c *Commit) IsForcePush(oldCommitID string) (bool, error) {
 	if err != nil {
 		return false, err
 	}
-	if oldCommitID == objectFormat.Empty().String() {
+	if oldCommitID == objectFormat.EmptyObjectID().String() {
 		return false, nil
 	}