Skip to content

Commit cbaa1de

Browse files
zeripathtechknowlogick
authored andcommitted
Add Graceful shutdown for Windows and hooks for shutdown of goroutines (#8964)
* Graceful Shutdown for windows and others Restructures modules/graceful, adding shutdown for windows, removing and replacing the old minwinsvc code. Creates a new waitGroup - terminate which allows for goroutines to finish up after the shutdown of the servers. Shutdown and terminate hooks are added for goroutines. * Remove unused functions - these can be added in a different PR * Add startup timeout functionality * Document STARTUP_TIMEOUT
1 parent d7ac972 commit cbaa1de

30 files changed

+666
-497
lines changed

cmd/web.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -227,7 +227,8 @@ func runWeb(ctx *cli.Context) error {
227227
log.Critical("Failed to start server: %v", err)
228228
}
229229
log.Info("HTTP Listener: %s Closed", listenAddr)
230-
graceful.WaitForServers()
230+
graceful.Manager.WaitForServers()
231+
graceful.Manager.WaitForTerminate()
231232
log.Close()
232233
return nil
233234
}

cmd/web_graceful.go

+2-4
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
// +build !windows
2-
31
// Copyright 2016 The Gitea Authors. All rights reserved.
42
// Use of this source code is governed by a MIT-style
53
// license that can be found in the LICENSE file.
@@ -27,11 +25,11 @@ func runHTTPSWithTLSConfig(listenAddr string, tlsConfig *tls.Config, m http.Hand
2725

2826
// NoHTTPRedirector tells our cleanup routine that we will not be using a fallback http redirector
2927
func NoHTTPRedirector() {
30-
graceful.InformCleanup()
28+
graceful.Manager.InformCleanup()
3129
}
3230

3331
// NoMainListener tells our cleanup routine that we will not be using a possibly provided listener
3432
// for our main HTTP/HTTPS service
3533
func NoMainListener() {
36-
graceful.InformCleanup()
34+
graceful.Manager.InformCleanup()
3735
}

cmd/web_windows.go

-37
This file was deleted.

custom/conf/app.ini.sample

+4-1
Original file line numberDiff line numberDiff line change
@@ -287,6 +287,9 @@ ALLOW_GRACEFUL_RESTARTS = true
287287
; shutting down. Force shutdown if this process takes longer than this delay.
288288
; set to a negative value to disable
289289
GRACEFUL_HAMMER_TIME = 60s
290+
; Allows the setting of a startup timeout and waithint for Windows as SVC service
291+
; 0 disables this.
292+
STARTUP_TIMEOUT = 0
290293
; Static resources, includes resources on custom/, public/ and all uploaded avatars web browser cache time, default is 6h
291294
STATIC_CACHE_TIME = 6h
292295

@@ -897,4 +900,4 @@ QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0"
897900
; Max attempts per http/https request on migrations.
898901
MAX_ATTEMPTS = 3
899902
; Backoff time per http/https request retry (seconds)
900-
RETRY_BACKOFF = 3
903+
RETRY_BACKOFF = 3

docs/content/doc/advanced/config-cheat-sheet.en-us.md

+1
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,7 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
189189
- `LETSENCRYPT_EMAIL`: **[email protected]**: Email used by Letsencrypt to notify about problems with issued certificates. (No default)
190190
- `ALLOW_GRACEFUL_RESTARTS`: **true**: Perform a graceful restart on SIGHUP
191191
- `GRACEFUL_HAMMER_TIME`: **60s**: After a restart the parent process will stop accepting new connections and will allow requests to finish before stopping. Shutdown will be forced if it takes longer than this time.
192+
- `STARTUP_TIMEOUT`: **0**: Shutsdown the server if startup takes longer than the provided time. On Windows setting this sends a waithint to the SVC host to tell the SVC host startup may take some time. Please note startup is determined by the opening of the listeners - HTTP/HTTPS/SSH. Indexers may take longer to startup and can have their own timeouts.
192193

193194
## Database (`database`)
194195

models/repo_indexer.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ func InitRepoIndexer() {
8484
if setting.Indexer.StartupTimeout > 0 {
8585
go func() {
8686
timeout := setting.Indexer.StartupTimeout
87-
if graceful.IsChild && setting.GracefulHammerTime > 0 {
87+
if graceful.Manager.IsChild() && setting.GracefulHammerTime > 0 {
8888
timeout += setting.GracefulHammerTime
8989
}
9090
select {

modules/graceful/cleanup.go

-40
This file was deleted.

modules/graceful/graceful_windows.go

-16
This file was deleted.

modules/graceful/manager.go

+187
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
// Copyright 2019 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package graceful
6+
7+
import (
8+
"time"
9+
10+
"code.gitea.io/gitea/modules/log"
11+
"code.gitea.io/gitea/modules/setting"
12+
)
13+
14+
type state uint8
15+
16+
const (
17+
stateInit state = iota
18+
stateRunning
19+
stateShuttingDown
20+
stateTerminate
21+
)
22+
23+
// There are three places that could inherit sockets:
24+
//
25+
// * HTTP or HTTPS main listener
26+
// * HTTP redirection fallback
27+
// * SSH
28+
//
29+
// If you add an additional place you must increment this number
30+
// and add a function to call manager.InformCleanup if it's not going to be used
31+
const numberOfServersToCreate = 3
32+
33+
// Manager represents the graceful server manager interface
34+
var Manager *gracefulManager
35+
36+
func init() {
37+
Manager = newGracefulManager()
38+
}
39+
40+
func (g *gracefulManager) doShutdown() {
41+
if !g.setStateTransition(stateRunning, stateShuttingDown) {
42+
return
43+
}
44+
g.lock.Lock()
45+
close(g.shutdown)
46+
g.lock.Unlock()
47+
48+
if setting.GracefulHammerTime >= 0 {
49+
go g.doHammerTime(setting.GracefulHammerTime)
50+
}
51+
go func() {
52+
g.WaitForServers()
53+
<-time.After(1 * time.Second)
54+
g.doTerminate()
55+
}()
56+
}
57+
58+
func (g *gracefulManager) doHammerTime(d time.Duration) {
59+
time.Sleep(d)
60+
select {
61+
case <-g.hammer:
62+
default:
63+
log.Warn("Setting Hammer condition")
64+
close(g.hammer)
65+
}
66+
67+
}
68+
69+
func (g *gracefulManager) doTerminate() {
70+
if !g.setStateTransition(stateShuttingDown, stateTerminate) {
71+
return
72+
}
73+
g.lock.Lock()
74+
close(g.terminate)
75+
g.lock.Unlock()
76+
}
77+
78+
// IsChild returns if the current process is a child of previous Gitea process
79+
func (g *gracefulManager) IsChild() bool {
80+
return g.isChild
81+
}
82+
83+
// IsShutdown returns a channel which will be closed at shutdown.
84+
// The order of closure is IsShutdown, IsHammer (potentially), IsTerminate
85+
func (g *gracefulManager) IsShutdown() <-chan struct{} {
86+
g.lock.RLock()
87+
if g.shutdown == nil {
88+
g.lock.RUnlock()
89+
g.lock.Lock()
90+
if g.shutdown == nil {
91+
g.shutdown = make(chan struct{})
92+
}
93+
defer g.lock.Unlock()
94+
return g.shutdown
95+
}
96+
defer g.lock.RUnlock()
97+
return g.shutdown
98+
}
99+
100+
// IsHammer returns a channel which will be closed at hammer
101+
// The order of closure is IsShutdown, IsHammer (potentially), IsTerminate
102+
// Servers running within the running server wait group should respond to IsHammer
103+
// if not shutdown already
104+
func (g *gracefulManager) IsHammer() <-chan struct{} {
105+
g.lock.RLock()
106+
if g.hammer == nil {
107+
g.lock.RUnlock()
108+
g.lock.Lock()
109+
if g.hammer == nil {
110+
g.hammer = make(chan struct{})
111+
}
112+
defer g.lock.Unlock()
113+
return g.hammer
114+
}
115+
defer g.lock.RUnlock()
116+
return g.hammer
117+
}
118+
119+
// IsTerminate returns a channel which will be closed at terminate
120+
// The order of closure is IsShutdown, IsHammer (potentially), IsTerminate
121+
// IsTerminate will only close once all running servers have stopped
122+
func (g *gracefulManager) IsTerminate() <-chan struct{} {
123+
g.lock.RLock()
124+
if g.terminate == nil {
125+
g.lock.RUnlock()
126+
g.lock.Lock()
127+
if g.terminate == nil {
128+
g.terminate = make(chan struct{})
129+
}
130+
defer g.lock.Unlock()
131+
return g.terminate
132+
}
133+
defer g.lock.RUnlock()
134+
return g.terminate
135+
}
136+
137+
// ServerDone declares a running server done and subtracts one from the
138+
// running server wait group. Users probably do not want to call this
139+
// and should use one of the RunWithShutdown* functions
140+
func (g *gracefulManager) ServerDone() {
141+
g.runningServerWaitGroup.Done()
142+
}
143+
144+
// WaitForServers waits for all running servers to finish. Users should probably
145+
// instead use AtTerminate or IsTerminate
146+
func (g *gracefulManager) WaitForServers() {
147+
g.runningServerWaitGroup.Wait()
148+
}
149+
150+
// WaitForTerminate waits for all terminating actions to finish.
151+
// Only the main go-routine should use this
152+
func (g *gracefulManager) WaitForTerminate() {
153+
g.terminateWaitGroup.Wait()
154+
}
155+
156+
func (g *gracefulManager) getState() state {
157+
g.lock.RLock()
158+
defer g.lock.RUnlock()
159+
return g.state
160+
}
161+
162+
func (g *gracefulManager) setStateTransition(old, new state) bool {
163+
if old != g.getState() {
164+
return false
165+
}
166+
g.lock.Lock()
167+
if g.state != old {
168+
g.lock.Unlock()
169+
return false
170+
}
171+
g.state = new
172+
g.lock.Unlock()
173+
return true
174+
}
175+
176+
func (g *gracefulManager) setState(st state) {
177+
g.lock.Lock()
178+
defer g.lock.Unlock()
179+
180+
g.state = st
181+
}
182+
183+
// InformCleanup tells the cleanup wait group that we have either taken a listener
184+
// or will not be taking a listener
185+
func (g *gracefulManager) InformCleanup() {
186+
g.createServerWaitGroup.Done()
187+
}

0 commit comments

Comments
 (0)