From 10ef5577b49c324e90c4b631136cf155d71a0bd9 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sun, 17 Oct 2021 14:42:28 +0800 Subject: [PATCH 1/9] sync git hook path --- contrib/pr/checkout.go | 2 +- modules/appstate/appstate.go | 17 ++++++ modules/appstate/file.go | 59 ++++++++++++++++++++ modules/appstate/item_runtime.go | 15 +++++ modules/repository/hooks.go | 58 +++++++++++++------ modules/setting/appstate.go | 24 ++++++++ modules/setting/setting.go | 1 + routers/init.go | 95 +++++++++++++++++++------------- 8 files changed, 216 insertions(+), 55 deletions(-) create mode 100644 modules/appstate/appstate.go create mode 100644 modules/appstate/file.go create mode 100644 modules/appstate/item_runtime.go create mode 100644 modules/setting/appstate.go diff --git a/contrib/pr/checkout.go b/contrib/pr/checkout.go index d831ebdabdb2f..05e9c88fde375 100644 --- a/contrib/pr/checkout.go +++ b/contrib/pr/checkout.go @@ -91,7 +91,7 @@ func runPR() { dbCfg.NewKey("DB_TYPE", "sqlite3") dbCfg.NewKey("PATH", ":memory:") - routers.NewServices() + routers.InitGitServices() setting.Database.LogSQL = true //x, err = xorm.NewEngine("sqlite3", "file::memory:?cache=shared") diff --git a/modules/appstate/appstate.go b/modules/appstate/appstate.go new file mode 100644 index 0000000000000..17f7df90b72c4 --- /dev/null +++ b/modules/appstate/appstate.go @@ -0,0 +1,17 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package appstate + +// StateStore is the interface to get/set app state items +type StateStore interface { + Get(item StateItem) error + Set(item StateItem) error + Delete(item StateItem) error +} + +// StateItem provides the name for a state item. the name will be used to generate filenames, etc +type StateItem interface { + Name() string +} diff --git a/modules/appstate/file.go b/modules/appstate/file.go new file mode 100644 index 0000000000000..659c45c3648c3 --- /dev/null +++ b/modules/appstate/file.go @@ -0,0 +1,59 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package appstate + +import ( + "io/fs" + "io/ioutil" + "os" + "path" + + "code.gitea.io/gitea/modules/json" +) + +// FileStore can be used to store app state items in local filesystem +type FileStore struct { + path string +} + +func (f *FileStore) genFilePath(item StateItem) string { + return path.Join(f.path, item.Name()) +} + +// Get reads the state item +func (f *FileStore) Get(item StateItem) error { + b, e := ioutil.ReadFile(f.genFilePath(item)) + if os.IsNotExist(e) { + return nil + } + if e != nil { + return e + } + e = json.Unmarshal(b, item) + return e +} + +//Set saves the state item +func (f *FileStore) Set(item StateItem) error { + b, e := json.Marshal(item) + if e != nil { + return e + } + return ioutil.WriteFile(f.genFilePath(item), b, fs.FileMode(0644)) +} + +//Delete removes the state item +func (f *FileStore) Delete(item StateItem) error { + return os.Remove(f.genFilePath(item)) +} + +//NewFileStore returns a new file store +func NewFileStore(path string) (*FileStore, error) { + _ = os.Mkdir(path, fs.FileMode(0755)) + if _, err := os.Stat(path); err != nil { + return nil, err + } + return &FileStore{path: path}, nil +} diff --git a/modules/appstate/item_runtime.go b/modules/appstate/item_runtime.go new file mode 100644 index 0000000000000..d5b4bc15b7c03 --- /dev/null +++ b/modules/appstate/item_runtime.go @@ -0,0 +1,15 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package appstate + +// RuntimeState contains app state for runtime, and we can save remote version for update checker here in future +type RuntimeState struct { + LastAppPath string `json:"last_app_path"` +} + +//Name returns the item name +func (a RuntimeState) Name() string { + return "runtime-state" +} diff --git a/modules/repository/hooks.go b/modules/repository/hooks.go index 6072dda0163fc..63f00b8f80933 100644 --- a/modules/repository/hooks.go +++ b/modules/repository/hooks.go @@ -23,64 +23,90 @@ import ( func getHookTemplates() (hookNames, hookTpls, giteaHookTpls []string) { hookNames = []string{"pre-receive", "update", "post-receive"} hookTpls = []string{ + // for pre-receive fmt.Sprintf(`#!/usr/bin/env %s +# AUTO GENERATED BY GITEA, DO NOT MODIFY data=$(cat) exitcodes="" hookname=$(basename $0) GIT_DIR=${GIT_DIR:-$(dirname $0)/..} for hook in ${GIT_DIR}/hooks/${hookname}.d/*; do -test -x "${hook}" && test -f "${hook}" || continue -echo "${data}" | "${hook}" -exitcodes="${exitcodes} $?" + test -x "${hook}" && test -f "${hook}" || continue + echo "${data}" | "${hook}" + exitcodes="${exitcodes} $?" done for i in ${exitcodes}; do -[ ${i} -eq 0 ] || exit ${i} + [ ${i} -eq 0 ] || exit ${i} done `, setting.ScriptType), + + // for update fmt.Sprintf(`#!/usr/bin/env %s +# AUTO GENERATED BY GITEA, DO NOT MODIFY exitcodes="" hookname=$(basename $0) GIT_DIR=${GIT_DIR:-$(dirname $0/..)} for hook in ${GIT_DIR}/hooks/${hookname}.d/*; do -test -x "${hook}" && test -f "${hook}" || continue -"${hook}" $1 $2 $3 -exitcodes="${exitcodes} $?" + test -x "${hook}" && test -f "${hook}" || continue + "${hook}" $1 $2 $3 + exitcodes="${exitcodes} $?" done for i in ${exitcodes}; do -[ ${i} -eq 0 ] || exit ${i} + [ ${i} -eq 0 ] || exit ${i} done `, setting.ScriptType), + + // for post-receive fmt.Sprintf(`#!/usr/bin/env %s +# AUTO GENERATED BY GITEA, DO NOT MODIFY data=$(cat) exitcodes="" hookname=$(basename $0) GIT_DIR=${GIT_DIR:-$(dirname $0)/..} for hook in ${GIT_DIR}/hooks/${hookname}.d/*; do -test -x "${hook}" && test -f "${hook}" || continue -echo "${data}" | "${hook}" -exitcodes="${exitcodes} $?" + test -x "${hook}" && test -f "${hook}" || continue + echo "${data}" | "${hook}" + exitcodes="${exitcodes} $?" done for i in ${exitcodes}; do -[ ${i} -eq 0 ] || exit ${i} + [ ${i} -eq 0 ] || exit ${i} done `, setting.ScriptType), } + giteaHookTpls = []string{ - fmt.Sprintf("#!/usr/bin/env %s\n%s hook --config=%s pre-receive\n", setting.ScriptType, util.ShellEscape(setting.AppPath), util.ShellEscape(setting.CustomConf)), - fmt.Sprintf("#!/usr/bin/env %s\n%s hook --config=%s update $1 $2 $3\n", setting.ScriptType, util.ShellEscape(setting.AppPath), util.ShellEscape(setting.CustomConf)), - fmt.Sprintf("#!/usr/bin/env %s\n%s hook --config=%s post-receive\n", setting.ScriptType, util.ShellEscape(setting.AppPath), util.ShellEscape(setting.CustomConf)), + // for pre-receive + fmt.Sprintf(`#!/usr/bin/env %s +# AUTO GENERATED BY GITEA, DO NOT MODIFY +%s hook --config=%s pre-receive +`, setting.ScriptType, util.ShellEscape(setting.AppPath), util.ShellEscape(setting.CustomConf)), + + // for update + fmt.Sprintf(`#!/usr/bin/env %s +# AUTO GENERATED BY GITEA, DO NOT MODIFY +%s hook --config=%s update $1 $2 $3 +`, setting.ScriptType, util.ShellEscape(setting.AppPath), util.ShellEscape(setting.CustomConf)), + + // for post-receive + fmt.Sprintf(`#!/usr/bin/env %s +# AUTO GENERATED BY GITEA, DO NOT MODIFY +%s hook --config=%s post-receive +`, setting.ScriptType, util.ShellEscape(setting.AppPath), util.ShellEscape(setting.CustomConf)), } if git.SupportProcReceive { hookNames = append(hookNames, "proc-receive") hookTpls = append(hookTpls, - fmt.Sprintf("#!/usr/bin/env %s\n%s hook --config=%s proc-receive\n", setting.ScriptType, util.ShellEscape(setting.AppPath), util.ShellEscape(setting.CustomConf))) + fmt.Sprintf(`#!/usr/bin/env %s +# AUTO GENERATED BY GITEA, DO NOT MODIFY +%s hook --config=%s proc-receive +`, setting.ScriptType, util.ShellEscape(setting.AppPath), util.ShellEscape(setting.CustomConf))) giteaHookTpls = append(giteaHookTpls, "") } diff --git a/modules/setting/appstate.go b/modules/setting/appstate.go new file mode 100644 index 0000000000000..8b93579d21a89 --- /dev/null +++ b/modules/setting/appstate.go @@ -0,0 +1,24 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package setting + +import ( + "path" + + "code.gitea.io/gitea/modules/appstate" + "code.gitea.io/gitea/modules/log" +) + +// AppState contains the state items for the app +var AppState appstate.StateStore + +func newAppState() { + var err error + appStatePath := path.Join(AppDataPath, "appstate") + AppState, err = appstate.NewFileStore(appStatePath) + if err != nil { + log.Fatal("failed to init AppState, err = %v", err) + } +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 2133184cfc40d..b0ff65a86824c 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -1188,6 +1188,7 @@ func CreateOrAppendToCustomConf(callback func(cfg *ini.File)) { func NewServices() { InitDBConfig() newService() + newAppState() newOAuth2Client() NewLogServices(false) newCacheService() diff --git a/routers/init.go b/routers/init.go index 52eacfd02b17e..bca7c23e7f610 100644 --- a/routers/init.go +++ b/routers/init.go @@ -6,9 +6,12 @@ package routers import ( "context" + "reflect" + "runtime" "strings" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/appstate" "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/cron" "code.gitea.io/gitea/modules/eventsource" @@ -22,6 +25,7 @@ import ( "code.gitea.io/gitea/modules/markup/external" repo_migrations "code.gitea.io/gitea/modules/migrations" "code.gitea.io/gitea/modules/notification" + repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/ssh" "code.gitea.io/gitea/modules/storage" @@ -45,23 +49,46 @@ import ( "gitea.com/go-chi/session" ) -// NewServices init new services -func NewServices() { - setting.NewServices() - if err := storage.Init(); err != nil { - log.Fatal("storage init failed: %v", err) +func guaranteeInit(fn func() error) { + err := fn() + if err != nil { + ptr := reflect.ValueOf(fn).Pointer() + fi := runtime.FuncForPC(ptr) + log.Fatal("%s init failed: %v", fi.Name(), err) } - if err := repository.NewContext(); err != nil { - log.Fatal("repository init failed: %v", err) +} + +func guaranteeInitCtx(ctx context.Context, fn func(ctx context.Context) error) { + err := fn(ctx) + if err != nil { + ptr := reflect.ValueOf(fn).Pointer() + fi := runtime.FuncForPC(ptr) + log.Fatal("%s(ctx) init failed: %v", fi.Name(), err) } - mailer.NewContext() - if err := cache.NewContext(); err != nil { - log.Fatal("Unable to start cache service: %v", err) +} + +// InitGitServices init new services for git, this is also called in `contrib/pr/checkout.go` +func InitGitServices() { + setting.NewServices() + guaranteeInit(storage.Init) + guaranteeInit(repository.NewContext) +} + +func syncAppPathForGitHooks(ctx context.Context) (err error) { + runtimeState := new(appstate.RuntimeState) + if err = setting.AppState.Get(runtimeState); err != nil { + return err } - notification.NewContext() - if err := archiver.Init(); err != nil { - log.Fatal("archiver init failed: %v", err) + if runtimeState.LastAppPath != setting.AppPath { + log.Info("AppPath changed from '%s' to '%s', sync repository hooks ...", runtimeState.LastAppPath, setting.AppPath) + err = repo_module.SyncRepositoryHooks(ctx) + if err != nil { + return err + } + runtimeState.LastAppPath = setting.AppPath + return setting.AppState.Set(runtimeState) } + return nil } // GlobalInit is for global configuration reload-able. @@ -71,9 +98,7 @@ func GlobalInit(ctx context.Context) { log.Fatal("Gitea is not installed") } - if err := git.Init(ctx); err != nil { - log.Fatal("Git module init failed: %v", err) - } + guaranteeInitCtx(ctx, git.Init) log.Info(git.VersionInfo()) git.CheckLFSVersion() @@ -87,7 +112,11 @@ func GlobalInit(ctx context.Context) { // Setup i18n translation.InitLocales() - NewServices() + InitGitServices() + mailer.NewContext() + guaranteeInit(cache.NewContext) + notification.NewContext() + guaranteeInit(archiver.Init) highlight.NewContext() external.RegisterRenderers() @@ -98,15 +127,11 @@ func GlobalInit(ctx context.Context) { } else if setting.Database.UseSQLite3 { log.Fatal("SQLite3 is set in settings but NOT Supported") } - if err := common.InitDBEngine(ctx); err == nil { - log.Info("ORM engine initialization successful!") - } else { - log.Fatal("ORM engine initialization failed: %v", err) - } - if err := oauth2.Init(); err != nil { - log.Fatal("Failed to initialize OAuth2 support: %v", err) - } + guaranteeInitCtx(ctx, common.InitDBEngine) + log.Info("ORM engine initialization successful!") + + guaranteeInit(oauth2.Init) models.NewRepoContext() @@ -114,22 +139,17 @@ func GlobalInit(ctx context.Context) { cron.NewContext() issue_indexer.InitIssueIndexer(false) code_indexer.Init() - if err := stats_indexer.Init(); err != nil { - log.Fatal("Failed to initialize repository stats indexer queue: %v", err) - } + guaranteeInit(stats_indexer.Init) + mirror_service.InitSyncMirrors() webhook.InitDeliverHooks() - if err := pull_service.Init(); err != nil { - log.Fatal("Failed to initialize test pull requests queue: %v", err) - } - if err := task.Init(); err != nil { - log.Fatal("Failed to initialize task scheduler: %v", err) - } - if err := repo_migrations.Init(); err != nil { - log.Fatal("Failed to initialize repository migrations: %v", err) - } + guaranteeInit(pull_service.Init) + guaranteeInit(task.Init) + guaranteeInit(repo_migrations.Init) eventsource.GetManager().Init() + guaranteeInitCtx(ctx, syncAppPathForGitHooks) + if setting.SSH.StartBuiltinServer { ssh.Listen(setting.SSH.ListenHost, setting.SSH.ListenPort, setting.SSH.ServerCiphers, setting.SSH.ServerKeyExchanges, setting.SSH.ServerMACs) log.Info("SSH server started on %s:%d. Cipher list (%v), key exchange algorithms (%v), MACs (%v)", setting.SSH.ListenHost, setting.SSH.ListenPort, setting.SSH.ServerCiphers, setting.SSH.ServerKeyExchanges, setting.SSH.ServerMACs) @@ -137,7 +157,6 @@ func GlobalInit(ctx context.Context) { ssh.Unused() } auth.Init() - svg.Init() } From fd4c82ee19ced249a7f30f506405beb8fcdda599 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sun, 17 Oct 2021 15:07:18 +0800 Subject: [PATCH 2/9] also do models.RewriteAllPublicKeys --- modules/appstate/file.go | 6 +++--- modules/appstate/item_runtime.go | 2 +- routers/init.go | 13 +++++++------ 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/modules/appstate/file.go b/modules/appstate/file.go index 659c45c3648c3..bdafd8a4cfac7 100644 --- a/modules/appstate/file.go +++ b/modules/appstate/file.go @@ -35,7 +35,7 @@ func (f *FileStore) Get(item StateItem) error { return e } -//Set saves the state item +// Set saves the state item func (f *FileStore) Set(item StateItem) error { b, e := json.Marshal(item) if e != nil { @@ -44,12 +44,12 @@ func (f *FileStore) Set(item StateItem) error { return ioutil.WriteFile(f.genFilePath(item), b, fs.FileMode(0644)) } -//Delete removes the state item +// Delete removes the state item func (f *FileStore) Delete(item StateItem) error { return os.Remove(f.genFilePath(item)) } -//NewFileStore returns a new file store +// NewFileStore returns a new file store func NewFileStore(path string) (*FileStore, error) { _ = os.Mkdir(path, fs.FileMode(0755)) if _, err := os.Stat(path); err != nil { diff --git a/modules/appstate/item_runtime.go b/modules/appstate/item_runtime.go index d5b4bc15b7c03..7fdc53f64247a 100644 --- a/modules/appstate/item_runtime.go +++ b/modules/appstate/item_runtime.go @@ -9,7 +9,7 @@ type RuntimeState struct { LastAppPath string `json:"last_app_path"` } -//Name returns the item name +// Name returns the item name func (a RuntimeState) Name() string { return "runtime-state" } diff --git a/routers/init.go b/routers/init.go index bca7c23e7f610..6faf7947834a0 100644 --- a/routers/init.go +++ b/routers/init.go @@ -74,17 +74,18 @@ func InitGitServices() { guaranteeInit(repository.NewContext) } -func syncAppPathForGitHooks(ctx context.Context) (err error) { +func syncAppPathForGit(ctx context.Context) (err error) { runtimeState := new(appstate.RuntimeState) if err = setting.AppState.Get(runtimeState); err != nil { return err } if runtimeState.LastAppPath != setting.AppPath { log.Info("AppPath changed from '%s' to '%s', sync repository hooks ...", runtimeState.LastAppPath, setting.AppPath) - err = repo_module.SyncRepositoryHooks(ctx) - if err != nil { - return err - } + guaranteeInitCtx(ctx, repo_module.SyncRepositoryHooks) + + log.Info("AppPath changed from '%s' to '%s', sync ssh keys ...", runtimeState.LastAppPath, setting.AppPath) + guaranteeInit(models.RewriteAllPublicKeys) + runtimeState.LastAppPath = setting.AppPath return setting.AppState.Set(runtimeState) } @@ -148,7 +149,7 @@ func GlobalInit(ctx context.Context) { guaranteeInit(repo_migrations.Init) eventsource.GetManager().Init() - guaranteeInitCtx(ctx, syncAppPathForGitHooks) + guaranteeInitCtx(ctx, syncAppPathForGit) if setting.SSH.StartBuiltinServer { ssh.Listen(setting.SSH.ListenHost, setting.SSH.ListenPort, setting.SSH.ServerCiphers, setting.SSH.ServerKeyExchanges, setting.SSH.ServerMACs) From 9e063090144ff2c1ceed5d421fd2a7eb6d302c8e Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sun, 17 Oct 2021 18:34:15 +0800 Subject: [PATCH 3/9] fix --- modules/setting/setting.go | 12 ++++++++++++ routers/init.go | 10 ++++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index b0ff65a86824c..c23becea7eefb 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -687,6 +687,18 @@ func NewContext() { StaticRootPath = sec.Key("STATIC_ROOT_PATH").MustString(StaticRootPath) StaticCacheTime = sec.Key("STATIC_CACHE_TIME").MustDuration(6 * time.Hour) AppDataPath = sec.Key("APP_DATA_PATH").MustString(path.Join(AppWorkPath, "data")) + if _, err = os.Stat(AppDataPath); err != nil { + // FIXME: Many calls to MkdirAll are using 0644 (not executable) or `os.ModePerm`(0o777, world-wide writable), which are incorrect. + // FIXME: There are too many calls to MkdirAll in old code. It is incorrect. + // For example, if someDir=/mnt/vol1/gitea-home/data, if the mount point /mnt/vol1 is not mounted when Gitea runs, + // then gitea will make new empty directories in /mnt/vol1, all are stored in the root filesystem. + // The correct behavior should be: creating parent directories is end users' duty. We only create sub-directories in existing parent directories. + // Now we can take the first step to do correctly (Mkdir) in other packages, and prepare the AppDataPath here, then make a refactor in future. + err = os.MkdirAll(AppDataPath, os.FileMode(0755)) + if err != nil { + log.Fatal("Failed to create the directory for app data path '%s'", AppDataPath) + } + } EnableGzip = sec.Key("ENABLE_GZIP").MustBool() EnablePprof = sec.Key("ENABLE_PPROF").MustBool(false) PprofDataPath = sec.Key("PPROF_DATA_PATH").MustString(path.Join(AppWorkPath, "data/tmp/pprof")) diff --git a/routers/init.go b/routers/init.go index 6faf7947834a0..a8def6891200d 100644 --- a/routers/init.go +++ b/routers/init.go @@ -54,7 +54,7 @@ func guaranteeInit(fn func() error) { if err != nil { ptr := reflect.ValueOf(fn).Pointer() fi := runtime.FuncForPC(ptr) - log.Fatal("%s init failed: %v", fi.Name(), err) + log.Fatal("%s failed: %v", fi.Name(), err) } } @@ -63,7 +63,7 @@ func guaranteeInitCtx(ctx context.Context, fn func(ctx context.Context) error) { if err != nil { ptr := reflect.ValueOf(fn).Pointer() fi := runtime.FuncForPC(ptr) - log.Fatal("%s(ctx) init failed: %v", fi.Name(), err) + log.Fatal("%s(ctx) failed: %v", fi.Name(), err) } } @@ -80,10 +80,12 @@ func syncAppPathForGit(ctx context.Context) (err error) { return err } if runtimeState.LastAppPath != setting.AppPath { - log.Info("AppPath changed from '%s' to '%s', sync repository hooks ...", runtimeState.LastAppPath, setting.AppPath) + log.Info("AppPath changed from '%s' to '%s'", runtimeState.LastAppPath, setting.AppPath) + + log.Info("re-sync repository hooks ...") guaranteeInitCtx(ctx, repo_module.SyncRepositoryHooks) - log.Info("AppPath changed from '%s' to '%s', sync ssh keys ...", runtimeState.LastAppPath, setting.AppPath) + log.Info("re-write ssh public keys ...") guaranteeInit(models.RewriteAllPublicKeys) runtimeState.LastAppPath = setting.AppPath From c2fab24cb9eb895468faa4e8b1a7e528abfb8955 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sun, 17 Oct 2021 22:27:24 +0800 Subject: [PATCH 4/9] fix --- modules/appstate/file.go | 4 ++-- modules/setting/appstate.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/appstate/file.go b/modules/appstate/file.go index bdafd8a4cfac7..53b130ccafedc 100644 --- a/modules/appstate/file.go +++ b/modules/appstate/file.go @@ -8,7 +8,7 @@ import ( "io/fs" "io/ioutil" "os" - "path" + "path/filepath" "code.gitea.io/gitea/modules/json" ) @@ -19,7 +19,7 @@ type FileStore struct { } func (f *FileStore) genFilePath(item StateItem) string { - return path.Join(f.path, item.Name()) + return filepath.Join(f.path, item.Name()) } // Get reads the state item diff --git a/modules/setting/appstate.go b/modules/setting/appstate.go index 8b93579d21a89..e85644112a3f6 100644 --- a/modules/setting/appstate.go +++ b/modules/setting/appstate.go @@ -5,7 +5,7 @@ package setting import ( - "path" + "path/filepath" "code.gitea.io/gitea/modules/appstate" "code.gitea.io/gitea/modules/log" @@ -16,7 +16,7 @@ var AppState appstate.StateStore func newAppState() { var err error - appStatePath := path.Join(AppDataPath, "appstate") + appStatePath := filepath.Join(AppDataPath, "appstate") AppState, err = appstate.NewFileStore(appStatePath) if err != nil { log.Fatal("failed to init AppState, err = %v", err) From c8faf20f0c33a1224bcdd8be6df8bc0ea426f9b6 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Mon, 18 Oct 2021 16:01:48 +0800 Subject: [PATCH 5/9] apply suggestion: guaranteeInit => mustInit --- routers/init.go | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/routers/init.go b/routers/init.go index a8def6891200d..b52ab2a384ff9 100644 --- a/routers/init.go +++ b/routers/init.go @@ -49,7 +49,7 @@ import ( "gitea.com/go-chi/session" ) -func guaranteeInit(fn func() error) { +func mustInit(fn func() error) { err := fn() if err != nil { ptr := reflect.ValueOf(fn).Pointer() @@ -58,7 +58,7 @@ func guaranteeInit(fn func() error) { } } -func guaranteeInitCtx(ctx context.Context, fn func(ctx context.Context) error) { +func mustInitCtx(ctx context.Context, fn func(ctx context.Context) error) { err := fn(ctx) if err != nil { ptr := reflect.ValueOf(fn).Pointer() @@ -70,8 +70,8 @@ func guaranteeInitCtx(ctx context.Context, fn func(ctx context.Context) error) { // InitGitServices init new services for git, this is also called in `contrib/pr/checkout.go` func InitGitServices() { setting.NewServices() - guaranteeInit(storage.Init) - guaranteeInit(repository.NewContext) + mustInit(storage.Init) + mustInit(repository.NewContext) } func syncAppPathForGit(ctx context.Context) (err error) { @@ -83,10 +83,10 @@ func syncAppPathForGit(ctx context.Context) (err error) { log.Info("AppPath changed from '%s' to '%s'", runtimeState.LastAppPath, setting.AppPath) log.Info("re-sync repository hooks ...") - guaranteeInitCtx(ctx, repo_module.SyncRepositoryHooks) + mustInitCtx(ctx, repo_module.SyncRepositoryHooks) log.Info("re-write ssh public keys ...") - guaranteeInit(models.RewriteAllPublicKeys) + mustInit(models.RewriteAllPublicKeys) runtimeState.LastAppPath = setting.AppPath return setting.AppState.Set(runtimeState) @@ -101,7 +101,7 @@ func GlobalInit(ctx context.Context) { log.Fatal("Gitea is not installed") } - guaranteeInitCtx(ctx, git.Init) + mustInitCtx(ctx, git.Init) log.Info(git.VersionInfo()) git.CheckLFSVersion() @@ -117,9 +117,9 @@ func GlobalInit(ctx context.Context) { InitGitServices() mailer.NewContext() - guaranteeInit(cache.NewContext) + mustInit(cache.NewContext) notification.NewContext() - guaranteeInit(archiver.Init) + mustInit(archiver.Init) highlight.NewContext() external.RegisterRenderers() @@ -131,10 +131,10 @@ func GlobalInit(ctx context.Context) { log.Fatal("SQLite3 is set in settings but NOT Supported") } - guaranteeInitCtx(ctx, common.InitDBEngine) + mustInitCtx(ctx, common.InitDBEngine) log.Info("ORM engine initialization successful!") - guaranteeInit(oauth2.Init) + mustInit(oauth2.Init) models.NewRepoContext() @@ -142,16 +142,16 @@ func GlobalInit(ctx context.Context) { cron.NewContext() issue_indexer.InitIssueIndexer(false) code_indexer.Init() - guaranteeInit(stats_indexer.Init) + mustInit(stats_indexer.Init) mirror_service.InitSyncMirrors() webhook.InitDeliverHooks() - guaranteeInit(pull_service.Init) - guaranteeInit(task.Init) - guaranteeInit(repo_migrations.Init) + mustInit(pull_service.Init) + mustInit(task.Init) + mustInit(repo_migrations.Init) eventsource.GetManager().Init() - guaranteeInitCtx(ctx, syncAppPathForGit) + mustInitCtx(ctx, syncAppPathForGit) if setting.SSH.StartBuiltinServer { ssh.Listen(setting.SSH.ListenHost, setting.SSH.ListenPort, setting.SSH.ServerCiphers, setting.SSH.ServerKeyExchanges, setting.SSH.ServerMACs) From fda7e515fa567d01e366ec3763010839924fea74 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Mon, 18 Oct 2021 22:08:18 +0800 Subject: [PATCH 6/9] fix --- modules/appstate/appstate.go | 1 - modules/appstate/file.go | 5 ----- routers/init.go | 4 ++-- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/modules/appstate/appstate.go b/modules/appstate/appstate.go index 17f7df90b72c4..1617d978da807 100644 --- a/modules/appstate/appstate.go +++ b/modules/appstate/appstate.go @@ -8,7 +8,6 @@ package appstate type StateStore interface { Get(item StateItem) error Set(item StateItem) error - Delete(item StateItem) error } // StateItem provides the name for a state item. the name will be used to generate filenames, etc diff --git a/modules/appstate/file.go b/modules/appstate/file.go index 53b130ccafedc..db484bf162962 100644 --- a/modules/appstate/file.go +++ b/modules/appstate/file.go @@ -44,11 +44,6 @@ func (f *FileStore) Set(item StateItem) error { return ioutil.WriteFile(f.genFilePath(item), b, fs.FileMode(0644)) } -// Delete removes the state item -func (f *FileStore) Delete(item StateItem) error { - return os.Remove(f.genFilePath(item)) -} - // NewFileStore returns a new file store func NewFileStore(path string) (*FileStore, error) { _ = os.Mkdir(path, fs.FileMode(0755)) diff --git a/routers/init.go b/routers/init.go index b52ab2a384ff9..447414e791d66 100644 --- a/routers/init.go +++ b/routers/init.go @@ -74,9 +74,9 @@ func InitGitServices() { mustInit(repository.NewContext) } -func syncAppPathForGit(ctx context.Context) (err error) { +func syncAppPathForGit(ctx context.Context) error { runtimeState := new(appstate.RuntimeState) - if err = setting.AppState.Get(runtimeState); err != nil { + if err := setting.AppState.Get(runtimeState); err != nil { return err } if runtimeState.LastAppPath != setting.AppPath { From d1b3d1ce1f2aafddfcb8c235e7745212e42c0e2d Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Tue, 19 Oct 2021 12:41:09 +0800 Subject: [PATCH 7/9] fix comment --- modules/setting/setting.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index c23becea7eefb..228fad6b48493 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -688,13 +688,13 @@ func NewContext() { StaticCacheTime = sec.Key("STATIC_CACHE_TIME").MustDuration(6 * time.Hour) AppDataPath = sec.Key("APP_DATA_PATH").MustString(path.Join(AppWorkPath, "data")) if _, err = os.Stat(AppDataPath); err != nil { - // FIXME: Many calls to MkdirAll are using 0644 (not executable) or `os.ModePerm`(0o777, world-wide writable), which are incorrect. // FIXME: There are too many calls to MkdirAll in old code. It is incorrect. // For example, if someDir=/mnt/vol1/gitea-home/data, if the mount point /mnt/vol1 is not mounted when Gitea runs, // then gitea will make new empty directories in /mnt/vol1, all are stored in the root filesystem. // The correct behavior should be: creating parent directories is end users' duty. We only create sub-directories in existing parent directories. - // Now we can take the first step to do correctly (Mkdir) in other packages, and prepare the AppDataPath here, then make a refactor in future. - err = os.MkdirAll(AppDataPath, os.FileMode(0755)) + // For quickstart, the parent directories should be created automatically for first startup (eg: a flag or a check of INSTALL_LOCK). + // Now we can take the first step to do correctly (using Mkdir) in other packages, and prepare the AppDataPath here, then make a refactor in future. + err = os.MkdirAll(AppDataPath, os.ModePerm) if err != nil { log.Fatal("Failed to create the directory for app data path '%s'", AppDataPath) } From 96dca591e54a8498cae94bb531ca6da81056d9ac Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Tue, 19 Oct 2021 17:45:33 +0800 Subject: [PATCH 8/9] use db instead of file --- models/appstate/appstate.go | 53 +++++++++++++++++++++++++ models/migrations/migrations.go | 2 + models/migrations/v200.go | 23 +++++++++++ modules/appstate/appstate.go | 9 +++++ modules/appstate/appstate_test.go | 66 +++++++++++++++++++++++++++++++ modules/appstate/db.go | 37 +++++++++++++++++ modules/appstate/file.go | 54 ------------------------- modules/setting/appstate.go | 24 ----------- modules/setting/setting.go | 1 - routers/init.go | 6 +-- 10 files changed, 193 insertions(+), 82 deletions(-) create mode 100644 models/appstate/appstate.go create mode 100644 models/migrations/v200.go create mode 100644 modules/appstate/appstate_test.go create mode 100644 modules/appstate/db.go delete mode 100644 modules/appstate/file.go delete mode 100644 modules/setting/appstate.go diff --git a/models/appstate/appstate.go b/models/appstate/appstate.go new file mode 100644 index 0000000000000..05665f51ef6fd --- /dev/null +++ b/models/appstate/appstate.go @@ -0,0 +1,53 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package appstate + +import ( + "context" + + "code.gitea.io/gitea/models/db" +) + +// AppState represents a state record in database +// if one day we would make Gitea run as a cluster, +// we can introduce a new field `Scope` here to store different states for different nodes +type AppState struct { + ID string `xorm:"pk varchar(200)"` + Revision int64 + Content string `xorm:"LONGTEXT"` +} + +// SaveAppStateContent saves the app state item to database +func SaveAppStateContent(key, content string) error { + return db.WithTx(func(ctx context.Context) error { + eng := db.GetEngine(ctx) + // try to update existing row + res, err := eng.Exec("UPDATE app_state SET revision=revision+1, content=? WHERE id=?", content, key) + if err != nil { + return err + } + rows, _ := res.RowsAffected() + if rows != 0 { + // the existing row is updated, so we can return + return nil + } + // if no existing row, insert a new row + _, err = eng.Insert(&AppState{ID: key, Content: content}) + return err + }) +} + +// GetAppStateContent gets an app state from database +func GetAppStateContent(key string) (content string, err error) { + e := db.GetEngine(db.DefaultContext) + appState := &AppState{ID: key} + has, err := e.Get(appState) + if err != nil { + return "", err + } else if !has { + return "", nil + } + return appState.Content, nil +} diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 3a41cf88918d3..b1c91beef6bdd 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -352,6 +352,8 @@ var migrations = []Migration{ NewMigration("Add issue content history table", addTableIssueContentHistory), // v199 -> v200 NewMigration("Add remote version table", addRemoteVersionTable), + // v200 -> v201 + NewMigration("Add table app_state", addTableAppState), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v200.go b/models/migrations/v200.go new file mode 100644 index 0000000000000..56ac06cb13662 --- /dev/null +++ b/models/migrations/v200.go @@ -0,0 +1,23 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "fmt" + + "xorm.io/xorm" +) + +func addTableAppState(x *xorm.Engine) error { + type AppState struct { + ID string `xorm:"pk varchar(200)"` + Revision int64 + Content string `xorm:"LONGTEXT"` + } + if err := x.Sync2(new(AppState)); err != nil { + return fmt.Errorf("Sync2: %v", err) + } + return nil +} diff --git a/modules/appstate/appstate.go b/modules/appstate/appstate.go index 1617d978da807..f65f5367e235d 100644 --- a/modules/appstate/appstate.go +++ b/modules/appstate/appstate.go @@ -14,3 +14,12 @@ type StateStore interface { type StateItem interface { Name() string } + +// AppState contains the state items for the app +var AppState StateStore + +// Init initialize AppState interface +func Init() error { + AppState = &DBStore{} + return nil +} diff --git a/modules/appstate/appstate_test.go b/modules/appstate/appstate_test.go new file mode 100644 index 0000000000000..6c5326c89c9cb --- /dev/null +++ b/modules/appstate/appstate_test.go @@ -0,0 +1,66 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package appstate + +import ( + "path/filepath" + "testing" + + "code.gitea.io/gitea/models/appstate" + "code.gitea.io/gitea/models/db" + + "github.com/stretchr/testify/assert" +) + +func TestMain(m *testing.M) { + db.MainTest(m, filepath.Join("..", ".."), "") +} + +type testItem1 struct { + Val1 string + Val2 int +} + +func (*testItem1) Name() string { + return "test-item1" +} + +type testItem2 struct { + K string +} + +func (*testItem2) Name() string { + return "test-item2" +} + +func TestAppStateDB(t *testing.T) { + assert.NoError(t, db.PrepareTestDatabase()) + _ = db.GetEngine(db.DefaultContext).Sync2(new(appstate.AppState)) + + as := &DBStore{} + + item1 := new(testItem1) + assert.NoError(t, as.Get(item1)) + assert.Equal(t, "", item1.Val1) + assert.EqualValues(t, 0, item1.Val2) + + item1 = new(testItem1) + item1.Val1 = "a" + item1.Val2 = 2 + assert.NoError(t, as.Set(item1)) + + item2 := new(testItem2) + item2.K = "V" + assert.NoError(t, as.Set(item2)) + + item1 = new(testItem1) + assert.NoError(t, as.Get(item1)) + assert.Equal(t, "a", item1.Val1) + assert.EqualValues(t, 2, item1.Val2) + + item2 = new(testItem2) + assert.NoError(t, as.Get(item2)) + assert.Equal(t, "V", item2.K) +} diff --git a/modules/appstate/db.go b/modules/appstate/db.go new file mode 100644 index 0000000000000..a594b01d850b6 --- /dev/null +++ b/modules/appstate/db.go @@ -0,0 +1,37 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package appstate + +import ( + "code.gitea.io/gitea/models/appstate" + "code.gitea.io/gitea/modules/json" + + "github.com/yuin/goldmark/util" +) + +// DBStore can be used to store app state items in local filesystem +type DBStore struct { +} + +// Get reads the state item +func (f *DBStore) Get(item StateItem) error { + content, err := appstate.GetAppStateContent(item.Name()) + if err != nil { + return err + } + if content == "" { + return nil + } + return json.Unmarshal(util.StringToReadOnlyBytes(content), item) +} + +// Set saves the state item +func (f *DBStore) Set(item StateItem) error { + b, err := json.Marshal(item) + if err != nil { + return err + } + return appstate.SaveAppStateContent(item.Name(), util.BytesToReadOnlyString(b)) +} diff --git a/modules/appstate/file.go b/modules/appstate/file.go deleted file mode 100644 index db484bf162962..0000000000000 --- a/modules/appstate/file.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2021 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package appstate - -import ( - "io/fs" - "io/ioutil" - "os" - "path/filepath" - - "code.gitea.io/gitea/modules/json" -) - -// FileStore can be used to store app state items in local filesystem -type FileStore struct { - path string -} - -func (f *FileStore) genFilePath(item StateItem) string { - return filepath.Join(f.path, item.Name()) -} - -// Get reads the state item -func (f *FileStore) Get(item StateItem) error { - b, e := ioutil.ReadFile(f.genFilePath(item)) - if os.IsNotExist(e) { - return nil - } - if e != nil { - return e - } - e = json.Unmarshal(b, item) - return e -} - -// Set saves the state item -func (f *FileStore) Set(item StateItem) error { - b, e := json.Marshal(item) - if e != nil { - return e - } - return ioutil.WriteFile(f.genFilePath(item), b, fs.FileMode(0644)) -} - -// NewFileStore returns a new file store -func NewFileStore(path string) (*FileStore, error) { - _ = os.Mkdir(path, fs.FileMode(0755)) - if _, err := os.Stat(path); err != nil { - return nil, err - } - return &FileStore{path: path}, nil -} diff --git a/modules/setting/appstate.go b/modules/setting/appstate.go deleted file mode 100644 index e85644112a3f6..0000000000000 --- a/modules/setting/appstate.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2021 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package setting - -import ( - "path/filepath" - - "code.gitea.io/gitea/modules/appstate" - "code.gitea.io/gitea/modules/log" -) - -// AppState contains the state items for the app -var AppState appstate.StateStore - -func newAppState() { - var err error - appStatePath := filepath.Join(AppDataPath, "appstate") - AppState, err = appstate.NewFileStore(appStatePath) - if err != nil { - log.Fatal("failed to init AppState, err = %v", err) - } -} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 228fad6b48493..1c6ce2abdedde 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -1200,7 +1200,6 @@ func CreateOrAppendToCustomConf(callback func(cfg *ini.File)) { func NewServices() { InitDBConfig() newService() - newAppState() newOAuth2Client() NewLogServices(false) newCacheService() diff --git a/routers/init.go b/routers/init.go index 447414e791d66..a4f5f606bac53 100644 --- a/routers/init.go +++ b/routers/init.go @@ -76,7 +76,7 @@ func InitGitServices() { func syncAppPathForGit(ctx context.Context) error { runtimeState := new(appstate.RuntimeState) - if err := setting.AppState.Get(runtimeState); err != nil { + if err := appstate.AppState.Get(runtimeState); err != nil { return err } if runtimeState.LastAppPath != setting.AppPath { @@ -89,7 +89,7 @@ func syncAppPathForGit(ctx context.Context) error { mustInit(models.RewriteAllPublicKeys) runtimeState.LastAppPath = setting.AppPath - return setting.AppState.Set(runtimeState) + return appstate.AppState.Set(runtimeState) } return nil } @@ -133,7 +133,7 @@ func GlobalInit(ctx context.Context) { mustInitCtx(ctx, common.InitDBEngine) log.Info("ORM engine initialization successful!") - + mustInit(appstate.Init) mustInit(oauth2.Init) models.NewRepoContext() From 716aa35faed6ce544d82674bfc52aeb926ba2f75 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Tue, 19 Oct 2021 19:12:22 +0800 Subject: [PATCH 9/9] fix model --- models/appstate/appstate.go | 4 ++++ modules/appstate/appstate_test.go | 2 -- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/models/appstate/appstate.go b/models/appstate/appstate.go index 05665f51ef6fd..aa5a59e1a3aed 100644 --- a/models/appstate/appstate.go +++ b/models/appstate/appstate.go @@ -19,6 +19,10 @@ type AppState struct { Content string `xorm:"LONGTEXT"` } +func init() { + db.RegisterModel(new(AppState)) +} + // SaveAppStateContent saves the app state item to database func SaveAppStateContent(key, content string) error { return db.WithTx(func(ctx context.Context) error { diff --git a/modules/appstate/appstate_test.go b/modules/appstate/appstate_test.go index 6c5326c89c9cb..d8ab0a45fd80f 100644 --- a/modules/appstate/appstate_test.go +++ b/modules/appstate/appstate_test.go @@ -8,7 +8,6 @@ import ( "path/filepath" "testing" - "code.gitea.io/gitea/models/appstate" "code.gitea.io/gitea/models/db" "github.com/stretchr/testify/assert" @@ -37,7 +36,6 @@ func (*testItem2) Name() string { func TestAppStateDB(t *testing.T) { assert.NoError(t, db.PrepareTestDatabase()) - _ = db.GetEngine(db.DefaultContext).Sync2(new(appstate.AppState)) as := &DBStore{}