Skip to content

Commit 62e6c9b

Browse files
lunnyguillep2ktystuyfzand
authored
Add a storage layer for attachments (#11387)
* Add a storage layer for attachments * Fix some bug * fix test * Fix copyright head and lint * Fix bug * Add setting for minio and flags for migrate-storage * Add documents * fix lint * Add test for minio store type on attachments * fix test * fix test * Apply suggestions from code review Co-authored-by: guillep2k <[email protected]> * Add warning when storage migrated successfully * Fix drone * fix test * rebase * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * refactor the codes * add trace * Fix test * remove log on xorm * Fi download bug * Add a storage layer for attachments * Add setting for minio and flags for migrate-storage * fix lint * Add test for minio store type on attachments * Apply suggestions from code review Co-authored-by: guillep2k <[email protected]> * Fix drone * fix test * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * refactor the codes * add trace * Fix test * Add URL function to serve attachments directly from S3/Minio * Add ability to enable/disable redirection in attachment configuration * Fix typo * Add a storage layer for attachments * Add setting for minio and flags for migrate-storage * fix lint * Add test for minio store type on attachments * Apply suggestions from code review Co-authored-by: guillep2k <[email protected]> * Fix drone * fix test * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * don't change unrelated files * Fix lint * Fix build * update go.mod and go.sum * Use github.com/minio/minio-go/v6 * Remove unused function * Upgrade minio to v7 and some other improvements * fix lint * Fix go mod Co-authored-by: guillep2k <[email protected]> Co-authored-by: Tyler <[email protected]>
1 parent 02fbe1e commit 62e6c9b

File tree

330 files changed

+62096
-328
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

330 files changed

+62096
-328
lines changed

‎.drone.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,14 @@ services:
151151
discovery.type: single-node
152152
image: elasticsearch:7.5.0
153153

154+
- name: minio
155+
image: minio/minio:RELEASE.2020-05-16T01-33-21Z
156+
commands:
157+
- minio server /data
158+
environment:
159+
MINIO_ACCESS_KEY: 123456
160+
MINIO_SECRET_KEY: 12345678
161+
154162
steps:
155163
- name: fetch-tags
156164
pull: default

‎cmd/migrate_storage.go

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
// Copyright 2020 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package cmd
6+
7+
import (
8+
"context"
9+
"fmt"
10+
11+
"code.gitea.io/gitea/models"
12+
"code.gitea.io/gitea/models/migrations"
13+
"code.gitea.io/gitea/modules/log"
14+
"code.gitea.io/gitea/modules/setting"
15+
"code.gitea.io/gitea/modules/storage"
16+
17+
"github.com/urfave/cli"
18+
)
19+
20+
// CmdMigrateStorage represents the available migrate storage sub-command.
21+
var CmdMigrateStorage = cli.Command{
22+
Name: "migrate-storage",
23+
Usage: "Migrate the storage",
24+
Description: "This is a command for migrating storage.",
25+
Action: runMigrateStorage,
26+
Flags: []cli.Flag{
27+
cli.StringFlag{
28+
Name: "type, t",
29+
Value: "",
30+
Usage: "Kinds of files to migrate, currently only 'attachments' is supported",
31+
},
32+
cli.StringFlag{
33+
Name: "store, s",
34+
Value: "local",
35+
Usage: "New storage type, local or minio",
36+
},
37+
cli.StringFlag{
38+
Name: "path, p",
39+
Value: "",
40+
Usage: "New storage placement if store is local (leave blank for default)",
41+
},
42+
cli.StringFlag{
43+
Name: "minio-endpoint",
44+
Value: "",
45+
Usage: "Minio storage endpoint",
46+
},
47+
cli.StringFlag{
48+
Name: "minio-access-key-id",
49+
Value: "",
50+
Usage: "Minio storage accessKeyID",
51+
},
52+
cli.StringFlag{
53+
Name: "minio-secret-access-key",
54+
Value: "",
55+
Usage: "Minio storage secretAccessKey",
56+
},
57+
cli.StringFlag{
58+
Name: "minio-bucket",
59+
Value: "",
60+
Usage: "Minio storage bucket",
61+
},
62+
cli.StringFlag{
63+
Name: "minio-location",
64+
Value: "",
65+
Usage: "Minio storage location to create bucket",
66+
},
67+
cli.StringFlag{
68+
Name: "minio-base-path",
69+
Value: "",
70+
Usage: "Minio storage basepath on the bucket",
71+
},
72+
cli.BoolFlag{
73+
Name: "minio-use-ssl",
74+
Usage: "Enable SSL for minio",
75+
},
76+
},
77+
}
78+
79+
func migrateAttachments(dstStorage storage.ObjectStorage) error {
80+
return models.IterateAttachment(func(attach *models.Attachment) error {
81+
_, err := storage.Copy(dstStorage, attach.RelativePath(), storage.Attachments, attach.RelativePath())
82+
return err
83+
})
84+
}
85+
86+
func runMigrateStorage(ctx *cli.Context) error {
87+
if err := initDB(); err != nil {
88+
return err
89+
}
90+
91+
log.Trace("AppPath: %s", setting.AppPath)
92+
log.Trace("AppWorkPath: %s", setting.AppWorkPath)
93+
log.Trace("Custom path: %s", setting.CustomPath)
94+
log.Trace("Log path: %s", setting.LogRootPath)
95+
setting.InitDBConfig()
96+
97+
if err := models.NewEngine(context.Background(), migrations.Migrate); err != nil {
98+
log.Fatal("Failed to initialize ORM engine: %v", err)
99+
return err
100+
}
101+
102+
if err := storage.Init(); err != nil {
103+
return err
104+
}
105+
106+
tp := ctx.String("type")
107+
switch tp {
108+
case "attachments":
109+
var dstStorage storage.ObjectStorage
110+
var err error
111+
switch ctx.String("store") {
112+
case "local":
113+
p := ctx.String("path")
114+
if p == "" {
115+
log.Fatal("Path must be given when store is loal")
116+
return nil
117+
}
118+
dstStorage, err = storage.NewLocalStorage(p)
119+
case "minio":
120+
dstStorage, err = storage.NewMinioStorage(
121+
context.Background(),
122+
ctx.String("minio-endpoint"),
123+
ctx.String("minio-access-key-id"),
124+
ctx.String("minio-secret-access-key"),
125+
ctx.String("minio-bucket"),
126+
ctx.String("minio-location"),
127+
ctx.String("minio-base-path"),
128+
ctx.Bool("minio-use-ssl"),
129+
)
130+
default:
131+
return fmt.Errorf("Unsupported attachments store type: %s", ctx.String("store"))
132+
}
133+
134+
if err != nil {
135+
return err
136+
}
137+
if err := migrateAttachments(dstStorage); err != nil {
138+
return err
139+
}
140+
141+
log.Warn("All files have been copied to the new placement but old files are still on the orignial placement.")
142+
143+
return nil
144+
}
145+
146+
return nil
147+
}

‎custom/conf/app.example.ini

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -749,14 +749,35 @@ ENABLE_FEDERATED_AVATAR = false
749749
[attachment]
750750
; Whether attachments are enabled. Defaults to `true`
751751
ENABLED = true
752-
; Path for attachments. Defaults to `data/attachments`
753-
PATH = data/attachments
752+
754753
; One or more allowed types, e.g. "image/jpeg|image/png". Use "*/*" for all types.
755754
ALLOWED_TYPES = image/jpeg|image/png|application/zip|application/gzip
756755
; Max size of each file. Defaults to 4MB
757756
MAX_SIZE = 4
758757
; Max number of files per upload. Defaults to 5
759758
MAX_FILES = 5
759+
; Storage type for attachments, `local` for local disk or `minio` for s3 compatible
760+
; object storage service, default is `local`.
761+
STORE_TYPE = local
762+
; Allows the storage driver to redirect to authenticated URLs to serve files directly
763+
; Currently, only `minio` is supported.
764+
SERVE_DIRECT = false
765+
; Path for attachments. Defaults to `data/attachments` only available when STORE_TYPE is `local`
766+
PATH = data/attachments
767+
; Minio endpoint to connect only available when STORE_TYPE is `minio`
768+
MINIO_ENDPOINT = localhost:9000
769+
; Minio accessKeyID to connect only available when STORE_TYPE is `minio`
770+
MINIO_ACCESS_KEY_ID =
771+
; Minio secretAccessKey to connect only available when STORE_TYPE is `minio`
772+
MINIO_SECRET_ACCESS_KEY =
773+
; Minio bucket to store the attachments only available when STORE_TYPE is `minio`
774+
MINIO_BUCKET = gitea
775+
; Minio location to create bucket only available when STORE_TYPE is `minio`
776+
MINIO_LOCATION = us-east-1
777+
; Minio base path on the bucket only available when STORE_TYPE is `minio`
778+
MINIO_BASE_PATH = attachments/
779+
; Minio enabled ssl only available when STORE_TYPE is `minio`
780+
MINIO_USE_SSL = false
760781

761782
[time]
762783
; Specifies the format for fully outputted dates. Defaults to RFC1123

‎docs/content/doc/advanced/config-cheat-sheet.en-us.md

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -470,11 +470,20 @@ set name for unique queues. Individual queues will default to
470470
## Attachment (`attachment`)
471471

472472
- `ENABLED`: **true**: Enable this to allow uploading attachments.
473-
- `PATH`: **data/attachments**: Path to store attachments.
474473
- `ALLOWED_TYPES`: **see app.example.ini**: Allowed MIME types, e.g. `image/jpeg|image/png`.
475474
Use `*/*` for all types.
476475
- `MAX_SIZE`: **4**: Maximum size (MB).
477476
- `MAX_FILES`: **5**: Maximum number of attachments that can be uploaded at once.
477+
- `STORE_TYPE`: **local**: Storage type for attachments, `local` for local disk or `minio` for s3 compatible object storage service, default is `local`.
478+
- `SERVE_DIRECT`: **false**: Allows the storage driver to redirect to authenticated URLs to serve files directly. Currently, only Minio/S3 is supported via signed URLs, local does nothing.
479+
- `PATH`: **data/attachments**: Path to store attachments only available when STORE_TYPE is `local`
480+
- `MINIO_ENDPOINT`: **localhost:9000**: Minio endpoint to connect only available when STORE_TYPE is `minio`
481+
- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when STORE_TYPE is `minio`
482+
- `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when STORE_TYPE is `minio`
483+
- `MINIO_BUCKET`: **gitea**: Minio bucket to store the attachments only available when STORE_TYPE is `minio`
484+
- `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when STORE_TYPE is `minio`
485+
- `MINIO_BASE_PATH`: **attachments/**: Minio base path on the bucket only available when STORE_TYPE is `minio`
486+
- `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when STORE_TYPE is `minio`
478487

479488
## Log (`log`)
480489

‎docs/content/doc/advanced/config-cheat-sheet.zh-cn.md

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -180,10 +180,18 @@ menu:
180180
## Attachment (`attachment`)
181181

182182
- `ENABLED`: 是否允许用户上传附件。
183-
- `PATH`: 附件存储路径
184183
- `ALLOWED_TYPES`: 允许上传的附件类型。比如：`image/jpeg|image/png`，用 `*/*` 表示允许任何类型。
185184
- `MAX_SIZE`: 附件最大限制，单位 MB，比如： `4`。
186185
- `MAX_FILES`: 一次最多上传的附件数量，比如： `5`。
186+
- `STORE_TYPE`: **local**: 附件存储类型，`local` 将存储到本地文件夹， `minio` 将存储到 s3 兼容的对象存储服务中。
187+
- `PATH`: **data/attachments**: 附件存储路径，仅当 `STORE_TYPE` 为 `local` 时有效。
188+
- `MINIO_ENDPOINT`: **localhost:9000**: Minio 终端，仅当 `STORE_TYPE` 是 `minio` 时有效。
189+
- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID ，仅当 `STORE_TYPE` 是 `minio` 时有效。
190+
- `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey，仅当 `STORE_TYPE` 是 `minio` 时有效。
191+
- `MINIO_BUCKET`: **gitea**: Minio bucket to store the attachments，仅当 `STORE_TYPE` 是 `minio` 时有效。
192+
- `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket，仅当 `STORE_TYPE` 是 `minio` 时有效。
193+
- `MINIO_BASE_PATH`: **attachments/**: Minio base path on the bucket，仅当 `STORE_TYPE` 是 `minio` 时有效。
194+
- `MINIO_USE_SSL`: **false**: Minio enabled ssl，仅当 `STORE_TYPE` 是 `minio` 时有效。
187195

188196
关于 `ALLOWED_TYPES`， 在 (*)unix 系统中可以使用`file -I <filename>` 来快速获得对应的 `MIME type`。
189197

‎go.mod

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ require (
7474
github.com/mgechev/revive v1.0.2
7575
github.com/mholt/archiver/v3 v3.3.0
7676
github.com/microcosm-cc/bluemonday v1.0.3-0.20191119130333-0a75d7616912
77+
github.com/minio/minio-go/v7 v7.0.4
7778
github.com/mitchellh/go-homedir v1.1.0
7879
github.com/msteinert/pam v0.0.0-20151204160544-02ccfbfaf0cc
7980
github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5
@@ -104,17 +105,17 @@ require (
104105
github.com/yuin/goldmark-highlighting v0.0.0-20200307114337-60d527fdb691
105106
github.com/yuin/goldmark-meta v0.0.0-20191126180153-f0638e958b60
106107
golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de
107-
golang.org/x/net v0.0.0-20200625001655-4c5254603344
108+
golang.org/x/net v0.0.0-20200707034311-ab3426394381
108109
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
109-
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1
110-
golang.org/x/text v0.3.2
110+
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae
111+
golang.org/x/text v0.3.3
111112
golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 // indirect
112113
golang.org/x/tools v0.0.0-20200814230902-9882f1d1823d
113114
google.golang.org/appengine v1.6.5 // indirect
114115
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
115116
gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175 // indirect
116117
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
117-
gopkg.in/ini.v1 v1.52.0
118+
gopkg.in/ini.v1 v1.57.0
118119
gopkg.in/ldap.v3 v3.0.2
119120
gopkg.in/yaml.v2 v2.3.0
120121
mvdan.cc/xurls/v2 v2.1.0

0 commit comments

Comments
 (0)