Skip to content

Commit f7d3f4e

Browse files
feature: support graceful shutdown
If connected to Tarantool 2.10 or newer and WatchersFeature is required, after this patch connection supports server graceful shutdown [1]. In this case, server will wait until all client requests will be finished and client disconnects before going down (server also may go down by timeout). Client reconnect will happen if connection options enable reconnect. 1. https://www.tarantool.io/en/doc/latest/dev_guide/internals/iproto/graceful_shutdown/ Closes #214
1 parent 03a1cc1 commit f7d3f4e

File tree

4 files changed

+563
-11
lines changed

4 files changed

+563
-11
lines changed

CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ Versioning](http://semver.org/spec/v2.0.0.html) except to the first release.
1515
- Error type support in MessagePack (#209)
1616
- Event subscription support (#119)
1717
- Session settings support (#215)
18+
- Support graceful shutdown (#214)
1819

1920
### Changed
2021

connection.go

Lines changed: 135 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -25,14 +25,17 @@ const ignoreStreamId = 0
2525
const (
2626
connDisconnected = 0
2727
connConnected = 1
28-
connClosed = 2
28+
connShutdown = 2
29+
connClosed = 3
2930
)
3031

3132
const (
3233
connTransportNone = ""
3334
connTransportSsl = "ssl"
3435
)
3536

37+
const shutdownEventKey = "box.shutdown"
38+
3639
type ConnEventKind int
3740
type ConnLogKind int
3841

@@ -45,6 +48,8 @@ const (
4548
ReconnectFailed
4649
// Either reconnect attempts exhausted, or explicit Close is called.
4750
Closed
51+
// Shutdown signals that shutdown callback is processing.
52+
Shutdown
4853

4954
// LogReconnectFailed is logged when reconnect attempt failed.
5055
LogReconnectFailed ConnLogKind = iota + 1
@@ -134,10 +139,19 @@ func (d defaultLogger) Report(event ConnLogKind, conn *Connection, v ...interfac
134139
// always returns array of array (array of tuples for space related methods).
135140
// For Eval* and Call* Tarantool always returns array, but does not forces
136141
// array of arrays.
142+
//
143+
// If connected to Tarantool 2.10 or newer and WatchersFeature is required,
144+
// connection supports server graceful shutdown. In this case, server will
145+
// wait until all client requests will be finished and client disconnects
146+
// before going down (server also may go down by timeout). Client reconnect will
147+
// happen if connection options enable reconnect.
148+
//
149+
// More on graceful shutdown: https://www.tarantool.io/en/doc/latest/dev_guide/internals/iproto/graceful_shutdown/
137150
type Connection struct {
138151
addr string
139152
c net.Conn
140153
mutex sync.Mutex
154+
cond *sync.Cond
141155
// Schema contains schema loaded on connection.
142156
Schema *Schema
143157
// requestId contains the last request ID for requests with nil context.
@@ -162,6 +176,11 @@ type Connection struct {
162176
serverProtocolInfo ProtocolInfo
163177
// watchMap is a map of key -> chan watchState.
164178
watchMap sync.Map
179+
180+
// shutdownWatcher is the "box.shutdown" event watcher.
181+
shutdownWatcher Watcher
182+
// requestCnt is a counter of active requests.
183+
requestCnt uint32
165184
}
166185

167186
var _ = Connector(&Connection{}) // Check compatibility with connector interface.
@@ -385,6 +404,8 @@ func Connect(addr string, opts Opts) (conn *Connection, err error) {
385404
conn.opts.Logger = defaultLogger{}
386405
}
387406

407+
conn.cond = sync.NewCond(&conn.mutex)
408+
388409
if err = conn.createConnection(false); err != nil {
389410
ter, ok := err.(Error)
390411
if conn.opts.Reconnect <= 0 {
@@ -421,6 +442,16 @@ func Connect(addr string, opts Opts) (conn *Connection, err error) {
421442
}
422443
}
423444

445+
// Subscribe shutdown event to process graceful shutdown.
446+
if conn.isWatchersRequired() {
447+
watcher, werr := conn.NewWatcher(shutdownEventKey, shutdownEventCallback)
448+
if werr != nil {
449+
conn.closeConnection(werr, true)
450+
return nil, werr
451+
}
452+
conn.shutdownWatcher = watcher
453+
}
454+
424455
return conn, err
425456
}
426457

@@ -589,6 +620,7 @@ func (conn *Connection) dial() (err error) {
589620
conn.lockShards()
590621
conn.c = connection
591622
atomic.StoreUint32(&conn.state, connConnected)
623+
conn.cond.Broadcast()
592624
conn.unlockShards()
593625
go conn.writer(w, connection)
594626
go conn.reader(r, connection)
@@ -762,10 +794,19 @@ func (conn *Connection) closeConnection(neterr error, forever bool) (err error)
762794
if conn.state != connClosed {
763795
close(conn.control)
764796
atomic.StoreUint32(&conn.state, connClosed)
797+
conn.cond.Broadcast()
798+
// Free the resources.
799+
if conn.shutdownWatcher != nil {
800+
conn.mutex.Unlock()
801+
conn.shutdownWatcher.Unregister()
802+
conn.mutex.Lock()
803+
conn.shutdownWatcher = nil
804+
}
765805
conn.notify(Closed)
766806
}
767807
} else {
768808
atomic.StoreUint32(&conn.state, connDisconnected)
809+
conn.cond.Broadcast()
769810
conn.notify(Disconnected)
770811
}
771812
if conn.c != nil {
@@ -797,6 +838,7 @@ func (conn *Connection) reconnect(neterr error, c net.Conn) {
797838
} else {
798839
conn.closeConnection(neterr, true)
799840
}
841+
conn.cond.Broadcast()
800842
}
801843

802844
func (conn *Connection) lockShards() {
@@ -1026,6 +1068,15 @@ func (conn *Connection) newFuture(ctx context.Context) (fut *Future) {
10261068
fut.done = nil
10271069
shard.rmut.Unlock()
10281070
return
1071+
case connShutdown:
1072+
fut.err = ClientError{
1073+
ErrConnectionShutdown,
1074+
"server shutdown in progress",
1075+
}
1076+
fut.ready = nil
1077+
fut.done = nil
1078+
shard.rmut.Unlock()
1079+
return
10291080
}
10301081
pos := (fut.requestId / conn.opts.Concurrency) & (requestsMap - 1)
10311082
if ctx != nil {
@@ -1086,6 +1137,7 @@ func (conn *Connection) send(req Request, streamId uint64) *Future {
10861137
if fut.ready == nil {
10871138
return fut
10881139
}
1140+
10891141
if req.Ctx() != nil {
10901142
select {
10911143
case <-req.Ctx().Done():
@@ -1094,13 +1146,31 @@ func (conn *Connection) send(req Request, streamId uint64) *Future {
10941146
default:
10951147
}
10961148
}
1149+
1150+
if conn.shutdownWatcher != nil {
1151+
atomic.AddUint32(&(conn.requestCnt), uint32(1))
1152+
go conn.gracefulWait(fut)
1153+
}
1154+
10971155
conn.putFuture(fut, req, streamId)
1156+
10981157
if req.Ctx() != nil {
10991158
go conn.contextWatchdog(fut, req.Ctx())
11001159
}
1160+
11011161
return fut
11021162
}
11031163

1164+
func (conn *Connection) gracefulWait(fut *Future) {
1165+
<-fut.done
1166+
// This is a real advice from Go documentation
1167+
// about how to decrement atomic uint32.
1168+
// https://pkg.go.dev/sync/atomic#AddUint32
1169+
if atomic.AddUint32(&(conn.requestCnt), ^uint32(0)) == 0 {
1170+
conn.cond.Broadcast()
1171+
}
1172+
}
1173+
11041174
func (conn *Connection) putFuture(fut *Future, req Request, streamId uint64) {
11051175
shardn := fut.requestId & (conn.opts.Concurrency - 1)
11061176
shard := &conn.shard[shardn]
@@ -1458,6 +1528,15 @@ func subscribeWatchChannel(conn *Connection, key string) (chan watchState, error
14581528
return st, nil
14591529
}
14601530

1531+
func (conn *Connection) isWatchersRequired() bool {
1532+
for _, feature := range conn.opts.RequiredProtocolInfo.Features {
1533+
if feature == WatchersFeature {
1534+
return true
1535+
}
1536+
}
1537+
return false
1538+
}
1539+
14611540
// NewWatcher creates a new Watcher object for the connection.
14621541
//
14631542
// You need to require WatchersFeature to use watchers, see examples for the
@@ -1496,15 +1575,7 @@ func (conn *Connection) NewWatcher(key string, callback WatchCallback) (Watcher,
14961575
// asynchronous. We do not expect any response from a Tarantool instance
14971576
// That's why we can't just check the Tarantool response for an unsupported
14981577
// request error.
1499-
watchersRequired := false
1500-
for _, feature := range conn.opts.RequiredProtocolInfo.Features {
1501-
if feature == WatchersFeature {
1502-
watchersRequired = true
1503-
break
1504-
}
1505-
}
1506-
1507-
if !watchersRequired {
1578+
if !conn.isWatchersRequired() {
15081579
err := fmt.Errorf("the feature %s must be required by connection "+
15091580
"options to create a watcher", WatchersFeature)
15101581
return nil, err
@@ -1563,7 +1634,11 @@ func (conn *Connection) NewWatcher(key string, callback WatchCallback) (Watcher,
15631634

15641635
if state.cnt == 0 {
15651636
// The last one sends IPROTO_UNWATCH.
1566-
conn.Do(newUnwatchRequest(key)).Get()
1637+
if !conn.ClosedNow() {
1638+
// conn.ClosedNow() check is a workaround for calling
1639+
// Unregister from connectionClose().
1640+
conn.Do(newUnwatchRequest(key)).Get()
1641+
}
15671642
conn.watchMap.Delete(key)
15681643
close(state.unready)
15691644
}
@@ -1666,3 +1741,52 @@ func (conn *Connection) ServerProtocolInfo() ProtocolInfo {
16661741
func (conn *Connection) ClientProtocolInfo() ProtocolInfo {
16671742
return clientProtocolInfo.Clone()
16681743
}
1744+
1745+
func shutdownEventCallback(event WatchEvent) {
1746+
// Receives "true" on server shutdown.
1747+
// See https://www.tarantool.io/en/doc/latest/dev_guide/internals/iproto/graceful_shutdown/
1748+
// step 2.
1749+
val, ok := event.Value.(bool)
1750+
if ok && val {
1751+
go event.Conn.processShutdown()
1752+
}
1753+
}
1754+
1755+
func (conn *Connection) processShutdown() {
1756+
// Forbid state changes.
1757+
conn.mutex.Lock()
1758+
defer conn.mutex.Unlock()
1759+
1760+
atomic.StoreUint32(&(conn.state), connShutdown)
1761+
conn.notify(Shutdown)
1762+
1763+
c := conn.c
1764+
for (atomic.LoadUint32(&(conn.state)) == connShutdown) &&
1765+
(atomic.LoadUint32(&(conn.requestCnt)) != 0) &&
1766+
(c == conn.c) {
1767+
// Use cond var on conn.mutex since request execution may
1768+
// call reconnect(). It is ok if state changes as part of
1769+
// reconnect since Tarantool server won't allow to reconnect
1770+
// in the middle of shutting down.
1771+
conn.cond.Wait()
1772+
}
1773+
// Do not unregister task explicitly here since connection teardown
1774+
// has the same effect. To clean up connection resources,
1775+
// unregister on full close.
1776+
1777+
if (atomic.LoadUint32(&(conn.state)) == connShutdown) &&
1778+
(c == conn.c) {
1779+
// Start to reconnect based on common rules, same as in net.box.
1780+
// Reconnect also closes the connection: server waits until all
1781+
// subscribed connections are terminated.
1782+
// See https://www.tarantool.io/en/doc/latest/dev_guide/internals/iproto/graceful_shutdown/
1783+
// step 3.
1784+
conn.mutex.Unlock()
1785+
conn.reconnect(
1786+
ClientError{
1787+
ErrConnectionClosed,
1788+
"connection closed after server shutdown",
1789+
}, conn.c)
1790+
conn.mutex.Lock()
1791+
}
1792+
}

errors.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ const (
5555
ErrProtocolError = 0x4000 + iota
5656
ErrTimeouted = 0x4000 + iota
5757
ErrRateLimited = 0x4000 + iota
58+
ErrConnectionShutdown = 0x4000 + iota
5859
)
5960

6061
// Tarantool server error codes.

0 commit comments

Comments
 (0)