Merge branch 'main' into fix-index-names-when-recreating-tables-mysql

Author: 6543

custom/conf/app.example.ini

@@ -378,6 +378,10 @@ INTERNAL_TOKEN=
 ;;
 ;; Validate against https://haveibeenpwned.com/Passwords to see if a password has been exposed
 ;PASSWORD_CHECK_PWN = false
+;;
+;; Cache successful token hashes. API tokens are stored in the DB as pbkdf2 hashes however, this means that there is a potentially significant hashing load when there are multiple API operations.
+;; This cache will store the successfully hashed tokens in a LRU cache as a balance between performance and security.
+;SUCCESSFUL_TOKENS_CACHE_SIZE = 20
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2123,3 +2127,11 @@ PATH =
 ;;
 ;; Minio enabled ssl only available when STORAGE_TYPE is `minio`
 ;MINIO_USE_SSL = false
+
+;[proxy]
+;; Enable the proxy, all requests to external via HTTP will be affected
+;PROXY_ENABLED = false
+;; Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy/no_proxy
+;PROXY_URL =
+;; Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts.
+;PROXY_HOSTS =

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -441,6 +441,7 @@ relation to port exhaustion.
     - spec - use one or more special characters as ``!"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~``
     - off - do not check password complexity
 - `PASSWORD_CHECK_PWN`: **false**: Check [HaveIBeenPwned](https://haveibeenpwned.com/Passwords) to see if a password has been exposed.
+- `SUCCESSFUL_TOKENS_CACHE_SIZE`: **20**: Cache successful token hashes. API tokens are stored in the DB as pbkdf2 hashes however, this means that there is a potentially significant hashing load when there are multiple API operations. This cache will store the successfully hashed tokens in a LRU cache as a balance between performance and security. 
 
 ## OpenID (`openid`)
 
@@ -548,8 +549,8 @@ Define allowed algorithms and their minimum key length (use -1 to disable a type
 - `DELIVER_TIMEOUT`: **5**: Delivery timeout (sec) for shooting webhooks.
 - `SKIP_TLS_VERIFY`: **false**: Allow insecure certification.
 - `PAGING_NUM`: **10**: Number of webhook history events that are shown in one page.
-- `PROXY_URL`: ****: Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy
-- `PROXY_HOSTS`: ****: Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts.
+- `PROXY_URL`: **\<empty\>**: Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy. If not given, will use global proxy setting.
+- `PROXY_HOSTS`: **\<empty\>`**: Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts. If not given, will use global proxy setting.
 
 ## Mailer (`mailer`)
 
@@ -949,6 +950,7 @@ Task queue configuration has been moved to `queue.task`. However, the below conf
 - `ALLOWED_DOMAINS`: **\<empty\>**: Domains allowlist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas.
 - `BLOCKED_DOMAINS`: **\<empty\>**: Domains blocklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `ALLOWED_DOMAINS` is not blank, this option will be ignored.
 - `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291
+- `SKIP_TLS_VERIFY`: **false**: Allow skip tls verify
 
 ## Mirror (`mirror`)
 
@@ -1022,6 +1024,19 @@ is `data/repo-archive` and the default of `MINIO_BASE_PATH` is `repo-archive/`.
 - `MINIO_BASE_PATH`: **repo-archive/**: Minio base path on the bucket only available when `STORAGE_TYPE` is `minio`
 - `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio`
 
+## Proxy (`proxy`)
+
+- `PROXY_ENABLED`: **false**: Enable the proxy if true, all requests to external via HTTP will be affected, if false, no proxy will be used even environment http_proxy/https_proxy
+- `PROXY_URL`: **\<empty\>**: Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy
+- `PROXY_HOSTS`: **\<empty\>**: Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts.
+
+i.e.
+```ini
+PROXY_ENABLED = true
+PROXY_URL = socks://127.0.0.1:1080
+PROXY_HOSTS = *.github.com
+```
+
 ## Other (`other`)
 
 - `SHOW_FOOTER_BRANDING`: **false**: Show Gitea branding in the footer.

docs/content/doc/advanced/config-cheat-sheet.zh-cn.md

@@ -332,6 +332,7 @@ IS_INPUT_FILE = false
 - `ALLOWED_DOMAINS`: **\<empty\>**: 迁移仓库的域名白名单，默认为空，表示允许从任意域名迁移仓库，多个域名用逗号分隔。
 - `BLOCKED_DOMAINS`: **\<empty\>**: 迁移仓库的域名黑名单，默认为空，多个域名用逗号分隔。如果 `ALLOWED_DOMAINS` 不为空，此选项将会被忽略。
 - `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918
+- `SKIP_TLS_VERIFY`: **false**: 允许忽略 TLS 认证
 
 ## LFS (`lfs`)
 
@@ -397,6 +398,19 @@ Repository archive 的存储配置。 如果 `STORAGE_TYPE` 为空，则此配
 - `MINIO_BASE_PATH`: **repo-archive/**: Minio base path ，仅当 `STORAGE_TYPE` 为 `minio` 时有效。
 - `MINIO_USE_SSL`: **false**: Minio 是否启用 ssl ，仅当 `STORAGE_TYPE` 为 `minio` 时有效。
 
+## Proxy (`proxy`)
+
+- `PROXY_ENABLED`: **false**: 是否启用全局代理。如果为否，则不使用代理，环境变量中的代理也不使用
+- `PROXY_URL`: **\<empty\>**: 代理服务器地址，支持 http://, https//, socks://，为空则不启用代理而使用环境变量中的 http_proxy/https_proxy
+- `PROXY_HOSTS`: **\<empty\>**: 逗号分隔的多个需要代理的网址，支持 * 号匹配符号， ** 表示匹配所有网站
+
+i.e.
+```ini
+PROXY_ENABLED = true
+PROXY_URL = socks://127.0.0.1:1080
+PROXY_HOSTS = *.github.com
+```
+
 ## Other (`other`)
 
 - `SHOW_FOOTER_BRANDING`: 为真则在页面底部显示Gitea的字样。

models/models.go

@@ -17,6 +17,7 @@ import (
 
 	// Needed for the MySQL driver
 	_ "github.com/go-sql-driver/mysql"
+	lru "github.com/hashicorp/golang-lru"
 	"xorm.io/xorm"
 	"xorm.io/xorm/names"
 	"xorm.io/xorm/schemas"
@@ -234,6 +235,15 @@ func NewEngine(ctx context.Context, migrateFunc func(*xorm.Engine) error) (err e
 		return fmt.Errorf("sync database struct error: %v", err)
 	}
 
+	if setting.SuccessfulTokensCacheSize > 0 {
+		successfulAccessTokenCache, err = lru.New(setting.SuccessfulTokensCacheSize)
+		if err != nil {
+			return fmt.Errorf("unable to allocate AccessToken cache: %v", err)
+		}
+	} else {
+		successfulAccessTokenCache = nil
+	}
+
 	return nil
 }
 

models/token.go

@@ -14,8 +14,11 @@ import (
 	"code.gitea.io/gitea/modules/util"
 
 	gouuid "github.com/google/uuid"
+	lru "github.com/hashicorp/golang-lru"
 )
 
+var successfulAccessTokenCache *lru.Cache
+
 // AccessToken represents a personal access token.
 type AccessToken struct {
 	ID             int64 `xorm:"pk autoincr"`
@@ -52,6 +55,21 @@ func NewAccessToken(t *AccessToken) error {
 	return err
 }
 
+func getAccessTokenIDFromCache(token string) int64 {
+	if successfulAccessTokenCache == nil {
+		return 0
+	}
+	tInterface, ok := successfulAccessTokenCache.Get(token)
+	if !ok {
+		return 0
+	}
+	t, ok := tInterface.(int64)
+	if !ok {
+		return 0
+	}
+	return t
+}
+
 // GetAccessTokenBySHA returns access token by given token value
 func GetAccessTokenBySHA(token string) (*AccessToken, error) {
 	if token == "" {
@@ -66,17 +84,38 @@ func GetAccessTokenBySHA(token string) (*AccessToken, error) {
 			return nil, ErrAccessTokenNotExist{token}
 		}
 	}
-	var tokens []AccessToken
+
 	lastEight := token[len(token)-8:]
+
+	if id := getAccessTokenIDFromCache(token); id > 0 {
+		token := &AccessToken{
+			TokenLastEight: lastEight,
+		}
+		// Re-get the token from the db in case it has been deleted in the intervening period
+		has, err := x.ID(id).Get(token)
+		if err != nil {
+			return nil, err
+		}
+		if has {
+			return token, nil
+		}
+		successfulAccessTokenCache.Remove(token)
+	}
+
+	var tokens []AccessToken
 	err := x.Table(&AccessToken{}).Where("token_last_eight = ?", lastEight).Find(&tokens)
 	if err != nil {
 		return nil, err
 	} else if len(tokens) == 0 {
 		return nil, ErrAccessTokenNotExist{token}
 	}
+
 	for _, t := range tokens {
 		tempHash := hashToken(token, t.TokenSalt)
 		if subtle.ConstantTimeCompare([]byte(t.TokenHash), []byte(tempHash)) == 1 {
+			if successfulAccessTokenCache != nil {
+				successfulAccessTokenCache.Add(token, t.ID)
+			}
 			return &t, nil
 		}
 	}

modules/git/command.go

@@ -110,24 +110,47 @@ func (c *Command) RunInDirTimeoutEnvFullPipeline(env []string, timeout time.Dura
 // RunInDirTimeoutEnvFullPipelineFunc executes the command in given directory with given timeout,
 // it pipes stdout and stderr to given io.Writer and passes in an io.Reader as stdin. Between cmd.Start and cmd.Wait the passed in function is run.
 func (c *Command) RunInDirTimeoutEnvFullPipelineFunc(env []string, timeout time.Duration, dir string, stdout, stderr io.Writer, stdin io.Reader, fn func(context.Context, context.CancelFunc) error) error {
-	if timeout == -1 {
-		timeout = defaultCommandExecutionTimeout
+	return c.RunWithContext(&RunContext{
+		Env:          env,
+		Timeout:      timeout,
+		Dir:          dir,
+		Stdout:       stdout,
+		Stderr:       stderr,
+		Stdin:        stdin,
+		PipelineFunc: fn,
+	})
+}
+
+// RunContext represents parameters to run the command
+type RunContext struct {
+	Env            []string
+	Timeout        time.Duration
+	Dir            string
+	Stdout, Stderr io.Writer
+	Stdin          io.Reader
+	PipelineFunc   func(context.Context, context.CancelFunc) error
+}
+
+// RunWithContext run the command with context
+func (c *Command) RunWithContext(rc *RunContext) error {
+	if rc.Timeout == -1 {
+		rc.Timeout = defaultCommandExecutionTimeout
 	}
 
-	if len(dir) == 0 {
+	if len(rc.Dir) == 0 {
 		log.Debug("%s", c)
 	} else {
-		log.Debug("%s: %v", dir, c)
+		log.Debug("%s: %v", rc.Dir, c)
 	}
 
-	ctx, cancel := context.WithTimeout(c.parentContext, timeout)
+	ctx, cancel := context.WithTimeout(c.parentContext, rc.Timeout)
 	defer cancel()
 
 	cmd := exec.CommandContext(ctx, c.name, c.args...)
-	if env == nil {
+	if rc.Env == nil {
 		cmd.Env = os.Environ()
 	} else {
-		cmd.Env = env
+		cmd.Env = rc.Env
 	}
 
 	cmd.Env = append(
@@ -141,23 +164,23 @@ func (c *Command) RunInDirTimeoutEnvFullPipelineFunc(env []string, timeout time.
 	if goVersionLessThan115 {
 		cmd.Env = append(cmd.Env, "GODEBUG=asyncpreemptoff=1")
 	}
-	cmd.Dir = dir
-	cmd.Stdout = stdout
-	cmd.Stderr = stderr
-	cmd.Stdin = stdin
+	cmd.Dir = rc.Dir
+	cmd.Stdout = rc.Stdout
+	cmd.Stderr = rc.Stderr
+	cmd.Stdin = rc.Stdin
 	if err := cmd.Start(); err != nil {
 		return err
 	}
 
 	desc := c.desc
 	if desc == "" {
-		desc = fmt.Sprintf("%s %s %s [repo_path: %s]", GitExecutable, c.name, strings.Join(c.args, " "), dir)
+		desc = fmt.Sprintf("%s %s %s [repo_path: %s]", GitExecutable, c.name, strings.Join(c.args, " "), rc.Dir)
 	}
 	pid := process.GetManager().Add(desc, cancel)
 	defer process.GetManager().Remove(pid)
 
-	if fn != nil {
-		err := fn(ctx, cancel)
+	if rc.PipelineFunc != nil {
+		err := rc.PipelineFunc(ctx, cancel)
 		if err != nil {
 			cancel()
 			_ = cmd.Wait()

modules/git/repo.go

@@ -9,11 +9,15 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
+	"net/url"
 	"os"
 	"path"
 	"strconv"
 	"strings"
 	"time"
+
+	"code.gitea.io/gitea/modules/proxy"
 )
 
 // GPGSettings represents the default GPG settings for this repository
@@ -99,12 +103,12 @@ type CloneRepoOptions struct {
 }
 
 // Clone clones original repository to target path.
-func Clone(from, to string, opts CloneRepoOptions) (err error) {
+func Clone(from, to string, opts CloneRepoOptions) error {
 	return CloneWithContext(DefaultContext, from, to, opts)
 }
 
 // CloneWithContext clones original repository to target path.
-func CloneWithContext(ctx context.Context, from, to string, opts CloneRepoOptions) (err error) {
+func CloneWithContext(ctx context.Context, from, to string, opts CloneRepoOptions) error {
 	cargs := make([]string, len(GlobalCommandArgs))
 	copy(cargs, GlobalCommandArgs)
 	return CloneWithArgs(ctx, from, to, cargs, opts)
@@ -146,8 +150,24 @@ func CloneWithArgs(ctx context.Context, from, to string, args []string, opts Clo
 		opts.Timeout = -1
 	}
 
-	_, err = cmd.RunTimeout(opts.Timeout)
-	return err
+	var envs = os.Environ()
+	u, err := url.Parse(from)
+	if err == nil && (strings.EqualFold(u.Scheme, "http") || strings.EqualFold(u.Scheme, "https")) {
+		if proxy.Match(u.Host) {
+			envs = append(envs, fmt.Sprintf("https_proxy=%s", proxy.GetProxyURL()))
+		}
+	}
+
+	var stderr = new(bytes.Buffer)
+	if err = cmd.RunWithContext(&RunContext{
+		Timeout: opts.Timeout,
+		Env:     envs,
+		Stdout:  io.Discard,
+		Stderr:  stderr,
+	}); err != nil {
+		return ConcatenateError(err, stderr.String())
+	}
+	return nil
 }
 
 // PullRemoteOptions options when pull from remote

modules/lfs/client.go

@@ -24,9 +24,9 @@ type Client interface {
 }
 
 // NewClient creates a LFS client
-func NewClient(endpoint *url.URL) Client {
+func NewClient(endpoint *url.URL, skipTLSVerify bool) Client {
 	if endpoint.Scheme == "file" {
 		return newFilesystemClient(endpoint)
 	}
-	return newHTTPClient(endpoint)
+	return newHTTPClient(endpoint, skipTLSVerify)
 }

modules/lfs/client_test.go

@@ -13,10 +13,10 @@ import (
 
 func TestNewClient(t *testing.T) {
 	u, _ := url.Parse("file:///test")
-	c := NewClient(u)
+	c := NewClient(u, true)
 	assert.IsType(t, &FilesystemClient{}, c)
 
 	u, _ = url.Parse("https://test.com/lfs")
-	c = NewClient(u)
+	c = NewClient(u, true)
 	assert.IsType(t, &HTTPClient{}, c)
 }

modules/lfs/http_client.go

@@ -7,6 +7,7 @@ package lfs
 import (
 	"bytes"
 	"context"
+	"crypto/tls"
 	"errors"
 	"fmt"
 	"net/http"
@@ -15,6 +16,7 @@ import (
 
 	"code.gitea.io/gitea/modules/json"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/proxy"
 )
 
 const batchSize = 20
@@ -32,8 +34,13 @@ func (c *HTTPClient) BatchSize() int {
 	return batchSize
 }
 
-func newHTTPClient(endpoint *url.URL) *HTTPClient {
-	hc := &http.Client{}
+func newHTTPClient(endpoint *url.URL, skipTLSVerify bool) *HTTPClient {
+	hc := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: skipTLSVerify},
+			Proxy:           proxy.Proxy(),
+		},
+	}
 
 	client := &HTTPClient{
 		client:    hc,