Race condition in TarantoolClientImpl

- Avoid a possible race between reading, writing and reconnecting
threads when a reconnection process is started.
It might have happened that the lagged thread (reading or writing) could
reset the state to RECONNECT after the reconnecting thread has already
started and set the state to 0. As a result, all next attempts to
reconnect will never happen. Now the reconnect thread holds on the state
as long as it is required.

- Avoid another possible race between reading and writing threads when
they are started during the reconnection process.
It might have happened that one of the threads crashed when it was
starting and another slightly lagged thread set up its flag. It could
have led that the reconnecting thread saw RECONNECT + R/W state instead
of pure RECONNECT. Again, this case broke down all next reconnection
attempts. Now reading and writing threads take into account whether
RECONNECT state is already set or not.

- Replace LockSupport with ReentrantLock.Condition for a thread to be
suspended and woken up. Our cluster tests and standalone demo app show
that LockSupport is not a safe memory barrier as it could be. The
reconnect thread relies on a visibility guarantee between park-unpark
invocations which, actually, sometimes doesn't work. Also, according to
java-docs LockSupport is more like an internal component to build
high-level blocking primitives. It is not recommended using this class
directly. It was replaced by ReentrantLock.Condition primitive based
on LockSupport but which has proper LockSupport usage inside.

Fixes: #142
Affects: #34, #136

Author: nicktorwald

src/main/java/org/tarantool/TarantoolClientImpl.java

@@ -22,7 +22,6 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.Condition;
-import java.util.concurrent.locks.LockSupport;
 import java.util.concurrent.locks.ReentrantLock;
 
 
@@ -63,17 +62,19 @@ public class TarantoolClientImpl extends TarantoolBase<Future<?>> implements Tar
      */
     protected TarantoolClientStats stats;
     protected StateHelper state = new StateHelper(StateHelper.RECONNECT);
-    protected Thread reader;
-    protected Thread writer;
+    protected volatile Thread reader;
+    protected volatile Thread writer;
 
     protected Thread connector = new Thread(new Runnable() {
         @Override
         public void run() {
             while (!Thread.currentThread().isInterrupted()) {
-                if (state.compareAndSet(StateHelper.RECONNECT, 0)) {
-                    reconnect(0, thumbstone);
+                reconnect(0, thumbstone);
+                try {
+                    state.awaitReconnection();
+                } catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
                 }
-                LockSupport.park(state);
             }
         }
     });
@@ -139,16 +140,13 @@ protected void reconnect(int retry, Throwable lastError) {
     protected void connect(final SocketChannel channel) throws Exception {
         try {
             TarantoolGreeting greeting = ProtoUtils.connect(channel,
-                config.username, config.password);
+                    config.username, config.password);
             this.serverVersion = greeting.getServerVersion();
         } catch (IOException e) {
-            try {
-                channel.close();
-            } catch (IOException ignored) {
-            }
-
+            closeChannel(channel);
             throw new CommunicationException("Couldn't connect to tarantool", e);
         }
+
         channel.configureBlocking(false);
         this.channel = channel;
         this.readChannel = new ReadableViaSelectorChannel(channel);
@@ -174,8 +172,21 @@ public void run() {
                         readThread();
                     } finally {
                         state.release(StateHelper.READING);
-                        if (state.compareAndSet(0, StateHelper.RECONNECT))
-                            LockSupport.unpark(connector);
+                        // Skip the old gen. attempt to reconnect
+                        //
+                        // there're two cases when a read thread is here
+                        // 1. it's a new generation thread inside/outside
+                        //    a reconnection process (currentThread == reader)
+                        // 2. It's an old generation thread inside
+                        //    a reconnection process (currentThread != reader)
+                        // NOTE: reader thread checks the statuses in reverse
+                        //       order the connector thread changes them. It
+                        //       makes this thread be sure that reference to
+                        //       it is not outdated when the state is UNINITIALIZED
+                        if (state.getState() == StateHelper.UNINITIALIZED
+                                && Thread.currentThread() == reader) {
+                            state.trySignalForReconnection();
+                        }
                     }
                 }
             }
@@ -189,13 +200,33 @@ public void run() {
                         writeThread();
                     } finally {
                         state.release(StateHelper.WRITING);
-                        if (state.compareAndSet(0, StateHelper.RECONNECT))
-                            LockSupport.unpark(connector);
+                        // Skip the old gen. attempt to reconnect
+                        //
+                        // there're two cases when a write thread is here
+                        // 1. it's a new generation thread inside/outside
+                        //    a reconnection process (currentThread == writer)
+                        // 2. It's an old generation thread inside
+                        //    a reconnection process (currentThread != writer)
+                        // NOTE: writer thread checks the statuses in reverse
+                        //       order the connector thread changes them. It
+                        //       makes this thread be sure that reference to
+                        //       it is not outdated when the state is UNINITIALIZED
+                        if (state.getState() == StateHelper.UNINITIALIZED
+                                && Thread.currentThread() == writer) {
+                            state.trySignalForReconnection();
+                        }
                     }
                 }
             }
         });
 
+        // reconnection preparation is done
+        // before reconnection the state will be released
+        // reader/writer threads have been replaced by new ones
+        // it's required to be sure that old r/w threads see correct
+        // client's r/w references
+        state.release(StateHelper.RECONNECT);
+
         configureThreads(threadName);
         reader.start();
         writer.start();
@@ -337,25 +368,21 @@ private boolean directWrite(ByteBuffer buffer) throws InterruptedException, IOEx
     }
 
     protected void readThread() {
-        try {
-            while (!Thread.currentThread().isInterrupted()) {
-                try {
-                    TarantoolPacket packet = ProtoUtils.readPacket(readChannel);
+        while (!Thread.currentThread().isInterrupted()) {
+            try {
+                TarantoolPacket packet = ProtoUtils.readPacket(readChannel);
 
-                    Map<Integer, Object> headers = packet.getHeaders();
+                Map<Integer, Object> headers = packet.getHeaders();
 
-                    Long syncId = (Long) headers.get(Key.SYNC.getId());
-                    TarantoolOp<?> future = futures.remove(syncId);
-                    stats.received++;
-                    wait.decrementAndGet();
-                    complete(packet, future);
-                } catch (Exception e) {
-                    die("Cant read answer", e);
-                    return;
-                }
+                Long syncId = (Long) headers.get(Key.SYNC.getId());
+                TarantoolOp<?> future = futures.remove(syncId);
+                stats.received++;
+                wait.decrementAndGet();
+                complete(packet, future);
+            } catch (Exception e) {
+                die("Cant read answer", e);
+                return;
             }
-        } catch (Exception e) {
-            die("Cant init thread", e);
         }
     }
 
@@ -498,7 +525,7 @@ public TarantoolClientOps<Integer, List<?>, Object, List<?>> syncOps() {
 
     @Override
     public TarantoolClientOps<Integer, List<?>, Object, Future<List<?>>> asyncOps() {
-        return (TarantoolClientOps)this;
+        return (TarantoolClientOps) this;
     }
 
     @Override
@@ -514,7 +541,7 @@ public TarantoolClientOps<Integer, List<?>, Object, Long> fireAndForgetOps() {
 
     @Override
     public TarantoolSQLOps<Object, Long, List<Map<String, Object>>> sqlSyncOps() {
-        return new TarantoolSQLOps<Object, Long, List<Map<String,Object>>>() {
+        return new TarantoolSQLOps<Object, Long, List<Map<String, Object>>>() {
 
             @Override
             public Long update(String sql, Object... bind) {
@@ -530,7 +557,7 @@ public List<Map<String, Object>> query(String sql, Object... bind) {
 
     @Override
     public TarantoolSQLOps<Object, Future<Long>, Future<List<Map<String, Object>>>> sqlAsyncOps() {
-        return new TarantoolSQLOps<Object, Future<Long>, Future<List<Map<String,Object>>>>() {
+        return new TarantoolSQLOps<Object, Future<Long>, Future<List<Map<String, Object>>>>() {
             @Override
             public Future<Long> update(String sql, Object... bind) {
                 return (Future<Long>) exec(Code.EXECUTE, Key.SQL_TEXT, sql, Key.SQL_BIND, bind);
@@ -618,6 +645,7 @@ public TarantoolClientStats getStats() {
      * Manages state changes.
      */
     protected final class StateHelper {
+        static final int UNINITIALIZED = 0;
         static final int READING = 1;
         static final int WRITING = 2;
         static final int ALIVE = READING | WRITING;
@@ -627,10 +655,22 @@ protected final class StateHelper {
         private final AtomicInteger state;
 
         private final AtomicReference<CountDownLatch> nextAliveLatch =
-            new AtomicReference<CountDownLatch>(new CountDownLatch(1));
+                new AtomicReference<>(new CountDownLatch(1));
 
         private final CountDownLatch closedLatch = new CountDownLatch(1);
 
+        /**
+         * The condition variable to signal a reconnection is needed from reader /
+         * writer threads and waiting for that signal from the reconnection thread.
+         *
+         * The lock variable to access this condition.
+         *
+         * @see #awaitReconnection()
+         * @see #trySignalForReconnection()
+         */
+        protected final ReentrantLock connectorLock = new ReentrantLock();
+        protected final Condition reconnectRequired = connectorLock.newCondition();
+
         protected StateHelper(int state) {
             this.state = new AtomicInteger(state);
         }
@@ -639,35 +679,60 @@ protected int getState() {
             return state.get();
         }
 
+        /**
+         * Set CLOSED state, drop RECONNECT state.
+         */
         protected boolean close() {
-            for (;;) {
+            for (; ; ) {
                 int st = getState();
+
+                /* CLOSED is the terminal state. */
                 if ((st & CLOSED) == CLOSED)
                     return false;
+
+                /*  Drop RECONNECT, set CLOSED. */
                 if (compareAndSet(st, (st & ~RECONNECT) | CLOSED))
                     return true;
             }
         }
 
+        /**
+         * Move from a current state to a give one.
+         *
+         * Some moves are forbidden.
+         */
         protected boolean acquire(int mask) {
-            for (;;) {
-                int st = getState();
-                if ((st & CLOSED) == CLOSED)
+            for (; ; ) {
+                int currentState = getState();
+
+                /* CLOSED is the terminal state. */
+                if ((currentState & CLOSED) == CLOSED) {
+                    return false;
+                }
+
+                /* Don't move to READING, WRITING or ALIVE from RECONNECT. */
+                if ((currentState & RECONNECT) > mask) {
                     return false;
+                }
 
-                if ((st & mask) != 0)
+                /* Cannot move from a state to the same state. */
+                if ((currentState & mask) != 0) {
                     throw new IllegalStateException("State is already " + mask);
+                }
 
-                if (compareAndSet(st, st | mask))
+                /* Set acquired state. */
+                if (compareAndSet(currentState, currentState | mask)) {
                     return true;
+                }
             }
         }
 
         protected void release(int mask) {
-            for (;;) {
+            for (; ; ) {
                 int st = getState();
-                if (compareAndSet(st, st & ~mask))
+                if (compareAndSet(st, st & ~mask)) {
                     return;
+                }
             }
         }
 
@@ -686,10 +751,18 @@ protected boolean compareAndSet(int expect, int update) {
             return true;
         }
 
+        /**
+         * Reconnection uses another way to await state via receiving a signal
+         * instead of latches.
+         */
         protected void awaitState(int state) throws InterruptedException {
-            CountDownLatch latch = getStateLatch(state);
-            if (latch != null) {
-                latch.await();
+            if (state == RECONNECT) {
+                awaitReconnection();
+            } else {
+                CountDownLatch latch = getStateLatch(state);
+                if (latch != null) {
+                    latch.await();
+                }
             }
         }
 
@@ -709,10 +782,42 @@ private CountDownLatch getStateLatch(int state) {
                 CountDownLatch latch = nextAliveLatch.get();
                 /* It may happen so that an error is detected but the state is still alive.
                  Wait for the 'next' alive state in such cases. */
-                return  (getState() == ALIVE && thumbstone == null) ? null : latch;
+                return (getState() == ALIVE && thumbstone == null) ? null : latch;
             }
             return null;
         }
+
+        /**
+         * Blocks until a reconnection signal will be received.
+         *
+         * @see #trySignalForReconnection()
+         */
+        private void awaitReconnection() throws InterruptedException {
+            connectorLock.lock();
+            try {
+                while (getState() != StateHelper.RECONNECT) {
+                    reconnectRequired.await();
+                }
+            } finally {
+                connectorLock.unlock();
+            }
+        }
+
+        /**
+         * Signals to the connector that reconnection process can be performed.
+         *
+         * @see #awaitReconnection()
+         */
+        private void trySignalForReconnection() {
+            if (compareAndSet(StateHelper.UNINITIALIZED, StateHelper.RECONNECT)) {
+                connectorLock.lock();
+                try {
+                    reconnectRequired.signal();
+                } finally {
+                    connectorLock.unlock();
+                }
+            }
+        }
     }
 
     protected static class TarantoolOp<V> extends CompletableFuture<V> {

src/test/java/org/tarantool/ClientReconnectIT.java

@@ -20,7 +20,6 @@
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -227,13 +226,13 @@ public void run() {
     public void testLongParallelCloseReconnects() {
         int numThreads = 4;
         int numClients = 4;
-        int timeBudget = 30*1000;
+        int timeBudget = 30 * 1000;
 
         SocketChannelProvider provider = new TestSocketChannelProvider(host,
             port, RESTART_TIMEOUT).setSoLinger(0);
 
         final AtomicReferenceArray<TarantoolClient> clients =
-            new AtomicReferenceArray<TarantoolClient>(numClients);
+                new AtomicReferenceArray<>(numClients);
 
         for (int idx = 0; idx < clients.length(); idx++) {
             clients.set(idx, makeClient(provider));
@@ -301,7 +300,7 @@ public void run() {
 
         // Wait for all threads to finish.
         try {
-            assertTrue(latch.await(RESTART_TIMEOUT, TimeUnit.MILLISECONDS));
+            assertTrue(latch.await(RESTART_TIMEOUT * 2, TimeUnit.MILLISECONDS));
         } catch (InterruptedException e) {
             fail(e);
         }