Skip to content

Commit e68c7b9

Browse files
committed
HDFS-10962. TestRequestHedgingProxyProvider is flaky.
1 parent 00160f7 commit e68c7b9

File tree

2 files changed

+28
-15
lines changed

2 files changed

+28
-15
lines changed

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -31,14 +31,14 @@
3131
import java.util.concurrent.Executors;
3232
import java.util.concurrent.Future;
3333

34-
import org.apache.commons.logging.Log;
35-
import org.apache.commons.logging.LogFactory;
3634
import org.apache.hadoop.conf.Configuration;
3735
import org.apache.hadoop.ipc.RemoteException;
3836
import org.apache.hadoop.ipc.StandbyException;
3937

4038
import com.google.common.annotations.VisibleForTesting;
4139
import org.apache.hadoop.io.retry.MultiException;
40+
import org.slf4j.Logger;
41+
import org.slf4j.LoggerFactory;
4242

4343
/**
4444
* A FailoverProxyProvider implementation that technically does not "failover"
@@ -51,8 +51,8 @@
5151
public class RequestHedgingProxyProvider<T> extends
5252
ConfiguredFailoverProxyProvider<T> {
5353

54-
private static final Log LOG =
55-
LogFactory.getLog(RequestHedgingProxyProvider.class);
54+
public static final Logger LOG =
55+
LoggerFactory.getLogger(RequestHedgingProxyProvider.class);
5656

5757
class RequestHedgingInvocationHandler implements InvocationHandler {
5858

@@ -100,6 +100,8 @@ public RequestHedgingInvocationHandler(
100100
Callable<Object> c = new Callable<Object>() {
101101
@Override
102102
public Object call() throws Exception {
103+
LOG.trace("Invoking method {} on proxy {}", method,
104+
pEntry.getValue().proxyInfo);
103105
return method.invoke(pEntry.getValue().proxy, args);
104106
}
105107
};
@@ -114,15 +116,14 @@ public Object call() throws Exception {
114116
try {
115117
retVal = callResultFuture.get();
116118
successfulProxy = proxyMap.get(callResultFuture);
117-
if (LOG.isDebugEnabled()) {
118-
LOG.debug("Invocation successful on ["
119-
+ successfulProxy.proxyInfo + "]");
120-
}
119+
LOG.debug("Invocation successful on [{}]",
120+
successfulProxy.proxyInfo);
121121
return retVal;
122122
} catch (Exception ex) {
123123
ProxyInfo<T> tProxyInfo = proxyMap.get(callResultFuture);
124124
logProxyException(ex, tProxyInfo.proxyInfo);
125125
badResults.put(tProxyInfo.proxyInfo, ex);
126+
LOG.trace("Unsuccessful invocation on [{}]", tProxyInfo.proxyInfo);
126127
numAttempts--;
127128
}
128129
}
@@ -136,6 +137,7 @@ public Object call() throws Exception {
136137
}
137138
} finally {
138139
if (executor != null) {
140+
LOG.trace("Shutting down threadpool executor");
139141
executor.shutdownNow();
140142
}
141143
}
@@ -193,12 +195,9 @@ public synchronized void performFailover(T currentProxy) {
193195
*/
194196
private void logProxyException(Exception ex, String proxyInfo) {
195197
if (isStandbyException(ex)) {
196-
if (LOG.isDebugEnabled()) {
197-
LOG.debug("Invocation returned standby exception on [" +
198-
proxyInfo + "]");
199-
}
198+
LOG.debug("Invocation returned standby exception on [{}]", proxyInfo);
200199
} else {
201-
LOG.warn("Invocation returned exception on [" + proxyInfo + "]");
200+
LOG.warn("Invocation returned exception on [{}]", proxyInfo);
202201
}
203202
}
204203

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,12 @@
3131
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
3232
import org.apache.hadoop.io.retry.MultiException;
3333
import org.apache.hadoop.security.UserGroupInformation;
34+
import org.apache.hadoop.test.GenericTestUtils;
3435
import org.apache.hadoop.util.Time;
36+
import org.apache.log4j.Level;
3537
import org.junit.Assert;
3638
import org.junit.Before;
39+
import org.junit.BeforeClass;
3740
import org.junit.Test;
3841
import org.mockito.Mockito;
3942
import org.mockito.invocation.InvocationOnMock;
@@ -47,6 +50,11 @@ public class TestRequestHedgingProxyProvider {
4750
private URI nnUri;
4851
private String ns;
4952

53+
@BeforeClass
54+
public static void setupClass() throws Exception {
55+
GenericTestUtils.setLogLevel(RequestHedgingProxyProvider.LOG, Level.TRACE);
56+
}
57+
5058
@Before
5159
public void setup() throws URISyntaxException {
5260
ns = "mycluster-" + Time.monotonicNow();
@@ -66,13 +74,19 @@ public void setup() throws URISyntaxException {
6674
@Test
6775
public void testHedgingWhenOneFails() throws Exception {
6876
final NamenodeProtocols goodMock = Mockito.mock(NamenodeProtocols.class);
69-
Mockito.when(goodMock.getStats()).thenReturn(new long[] {1});
77+
Mockito.when(goodMock.getStats()).thenAnswer(new Answer<long[]>() {
78+
@Override
79+
public long[] answer(InvocationOnMock invocation) throws Throwable {
80+
Thread.sleep(1000);
81+
return new long[]{1};
82+
}
83+
});
7084
final NamenodeProtocols badMock = Mockito.mock(NamenodeProtocols.class);
7185
Mockito.when(badMock.getStats()).thenThrow(new IOException("Bad mock !!"));
7286

7387
RequestHedgingProxyProvider<NamenodeProtocols> provider =
7488
new RequestHedgingProxyProvider<>(conf, nnUri, NamenodeProtocols.class,
75-
createFactory(goodMock, badMock));
89+
createFactory(badMock, goodMock));
7690
long[] stats = provider.getProxy().proxy.getStats();
7791
Assert.assertTrue(stats.length == 1);
7892
Mockito.verify(badMock).getStats();

0 commit comments

Comments
 (0)