Skip to content

Commit 0b507d2

Browse files
adoroszlaibshashikant
adoroszlai
authored and
bshashikant
committed
HDDS-1908. TestMultiBlockWritesWithDnFailures is failing (#1282)
1 parent 454420e commit 0b507d2

File tree

2 files changed

+67
-74
lines changed

2 files changed

+67
-74
lines changed

‎hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java‎

Lines changed: 3 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
4343
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
4444
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
45+
import org.junit.After;
4546
import org.junit.Assert;
4647
import org.junit.Test;
4748

@@ -71,7 +72,6 @@ public class TestFailureHandlingByClient {
7172
private String volumeName;
7273
private String bucketName;
7374
private String keyString;
74-
private int maxRetries;
7575

7676
/**
7777
* Create a MiniDFSCluster for testing.
@@ -82,7 +82,6 @@ public class TestFailureHandlingByClient {
8282
*/
8383
private void init() throws Exception {
8484
conf = new OzoneConfiguration();
85-
maxRetries = 100;
8685
chunkSize = (int) OzoneConsts.MB;
8786
blockSize = 4 * chunkSize;
8887
conf.setTimeDuration(OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT, 5,
@@ -125,7 +124,8 @@ private void startCluster() throws Exception {
125124
/**
126125
* Shutdown MiniDFSCluster.
127126
*/
128-
private void shutdown() {
127+
@After
128+
public void shutdown() {
129129
if (cluster != null) {
130130
cluster.shutdown();
131131
}
@@ -170,61 +170,6 @@ public void testBlockWritesWithDnFailures() throws Exception {
170170
OmKeyInfo keyInfo = cluster.getOzoneManager().lookupKey(keyArgs);
171171
Assert.assertEquals(data.length, keyInfo.getDataSize());
172172
validateData(keyName, data);
173-
shutdown();
174-
}
175-
176-
177-
@Test
178-
public void testMultiBlockWritesWithIntermittentDnFailures()
179-
throws Exception {
180-
startCluster();
181-
String keyName = UUID.randomUUID().toString();
182-
OzoneOutputStream key =
183-
createKey(keyName, ReplicationType.RATIS, 6 * blockSize);
184-
String data = ContainerTestHelper
185-
.getFixedLengthString(keyString, blockSize + chunkSize);
186-
key.write(data.getBytes());
187-
188-
// get the name of a valid container
189-
Assert.assertTrue(key.getOutputStream() instanceof KeyOutputStream);
190-
KeyOutputStream keyOutputStream =
191-
(KeyOutputStream) key.getOutputStream();
192-
List<BlockOutputStreamEntry> streamEntryList =
193-
keyOutputStream.getStreamEntries();
194-
195-
// Assert that 6 block will be preallocated
196-
Assert.assertEquals(6, streamEntryList.size());
197-
key.write(data.getBytes());
198-
key.flush();
199-
long containerId = streamEntryList.get(0).getBlockID().getContainerID();
200-
BlockID blockId = streamEntryList.get(0).getBlockID();
201-
ContainerInfo container =
202-
cluster.getStorageContainerManager().getContainerManager()
203-
.getContainer(ContainerID.valueof(containerId));
204-
Pipeline pipeline =
205-
cluster.getStorageContainerManager().getPipelineManager()
206-
.getPipeline(container.getPipelineID());
207-
List<DatanodeDetails> datanodes = pipeline.getNodes();
208-
cluster.shutdownHddsDatanode(datanodes.get(0));
209-
210-
// The write will fail but exception will be handled and length will be
211-
// updated correctly in OzoneManager once the steam is closed
212-
key.write(data.getBytes());
213-
214-
// shutdown the second datanode
215-
cluster.shutdownHddsDatanode(datanodes.get(1));
216-
key.write(data.getBytes());
217-
key.close();
218-
OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName)
219-
.setBucketName(bucketName).setType(HddsProtos.ReplicationType.RATIS)
220-
.setFactor(HddsProtos.ReplicationFactor.THREE).setKeyName(keyName)
221-
.setRefreshPipeline(true)
222-
.build();
223-
OmKeyInfo keyInfo = cluster.getOzoneManager().lookupKey(keyArgs);
224-
Assert.assertEquals(4 * data.getBytes().length, keyInfo.getDataSize());
225-
validateData(keyName,
226-
data.concat(data).concat(data).concat(data).getBytes());
227-
shutdown();
228173
}
229174

230175
@Test
@@ -270,7 +215,6 @@ public void testWriteSmallFile() throws Exception {
270215
.getBlockID(), blockId);
271216
Assert.assertEquals(data.getBytes().length, keyInfo.getDataSize());
272217
validateData(keyName, data.getBytes());
273-
shutdown();
274218
}
275219

276220

@@ -331,7 +275,6 @@ public void testContainerExclusionWithClosedContainerException()
331275
.getBlockID(), blockId);
332276
Assert.assertEquals(2 * data.getBytes().length, keyInfo.getDataSize());
333277
validateData(keyName, data.concat(data).getBytes());
334-
shutdown();
335278
}
336279

337280
@Test
@@ -394,7 +337,6 @@ public void testDatanodeExclusionWithMajorityCommit() throws Exception {
394337
.getBlockID(), blockId);
395338
Assert.assertEquals(3 * data.getBytes().length, keyInfo.getDataSize());
396339
validateData(keyName, data.concat(data).concat(data).getBytes());
397-
shutdown();
398340
}
399341

400342

@@ -458,7 +400,6 @@ public void testPipelineExclusionWithPipelineFailure() throws Exception {
458400
.getBlockID(), blockId);
459401
Assert.assertEquals(3 * data.getBytes().length, keyInfo.getDataSize());
460402
validateData(keyName, data.concat(data).concat(data).getBytes());
461-
shutdown();
462403
}
463404

464405
private OzoneOutputStream createKey(String keyName, ReplicationType type,

‎hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestMultiBlockWritesWithDnFailures.java‎

Lines changed: 64 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package org.apache.hadoop.ozone.client.rpc;
1919

20+
import org.apache.hadoop.hdds.client.BlockID;
2021
import org.apache.hadoop.hdds.client.ReplicationType;
2122
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
2223
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -30,12 +31,14 @@
3031
import org.apache.hadoop.ozone.client.ObjectStore;
3132
import org.apache.hadoop.ozone.client.OzoneClient;
3233
import org.apache.hadoop.ozone.client.OzoneClientFactory;
34+
import org.apache.hadoop.ozone.client.io.BlockOutputStreamEntry;
3335
import org.apache.hadoop.ozone.client.io.KeyOutputStream;
3436
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
3537
import org.apache.hadoop.ozone.container.ContainerTestHelper;
3638
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
3739
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
3840
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
41+
import org.junit.After;
3942
import org.junit.Assert;
4043
import org.junit.Test;
4144

@@ -61,7 +64,6 @@ public class TestMultiBlockWritesWithDnFailures {
6164
private String volumeName;
6265
private String bucketName;
6366
private String keyString;
64-
private int maxRetries;
6567

6668
/**
6769
* Create a MiniDFSCluster for testing.
@@ -70,23 +72,25 @@ public class TestMultiBlockWritesWithDnFailures {
7072
*
7173
* @throws IOException
7274
*/
73-
private void init() throws Exception {
75+
private void startCluster(int datanodes) throws Exception {
7476
conf = new OzoneConfiguration();
75-
maxRetries = 100;
7677
chunkSize = (int) OzoneConsts.MB;
7778
blockSize = 4 * chunkSize;
7879
conf.setTimeDuration(OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT, 5,
7980
TimeUnit.SECONDS);
8081
conf.setTimeDuration(HDDS_SCM_WATCHER_TIMEOUT, 1000, TimeUnit.MILLISECONDS);
8182
conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 100, TimeUnit.SECONDS);
82-
conf.setInt(OzoneConfigKeys.DFS_RATIS_CLIENT_REQUEST_MAX_RETRIES_KEY, 5);
83+
conf.setInt(OzoneConfigKeys.DFS_RATIS_CLIENT_REQUEST_MAX_RETRIES_KEY, 10);
8384
conf.setTimeDuration(
8485
OzoneConfigKeys.DFS_RATIS_CLIENT_REQUEST_RETRY_INTERVAL_KEY,
8586
1, TimeUnit.SECONDS);
87+
conf.setTimeDuration(
88+
OzoneConfigKeys.DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY,
89+
1, TimeUnit.SECONDS);
8690

8791
conf.setQuietMode(false);
8892
cluster = MiniOzoneCluster.newBuilder(conf)
89-
.setNumDatanodes(6).build();
93+
.setNumDatanodes(datanodes).build();
9094
cluster.waitForClusterToBeReady();
9195
//the easiest way to create an open container is creating a key
9296
client = OzoneClientFactory.getClient(conf);
@@ -98,22 +102,19 @@ private void init() throws Exception {
98102
objectStore.getVolume(volumeName).createBucket(bucketName);
99103
}
100104

101-
private void startCluster() throws Exception {
102-
init();
103-
}
104-
105105
/**
106106
* Shutdown MiniDFSCluster.
107107
*/
108-
private void shutdown() {
108+
@After
109+
public void shutdown() {
109110
if (cluster != null) {
110111
cluster.shutdown();
111112
}
112113
}
113114

114115
@Test
115116
public void testMultiBlockWritesWithDnFailures() throws Exception {
116-
startCluster();
117+
startCluster(6);
117118
String keyName = "ratis3";
118119
OzoneOutputStream key = createKey(keyName, ReplicationType.RATIS, 0);
119120
String data =
@@ -151,7 +152,58 @@ public void testMultiBlockWritesWithDnFailures() throws Exception {
151152
OmKeyInfo keyInfo = cluster.getOzoneManager().lookupKey(keyArgs);
152153
Assert.assertEquals(2 * data.getBytes().length, keyInfo.getDataSize());
153154
validateData(keyName, data.concat(data).getBytes());
154-
shutdown();
155+
}
156+
157+
@Test
158+
public void testMultiBlockWritesWithIntermittentDnFailures()
159+
throws Exception {
160+
startCluster(10);
161+
String keyName = UUID.randomUUID().toString();
162+
OzoneOutputStream key =
163+
createKey(keyName, ReplicationType.RATIS, 6 * blockSize);
164+
String data = ContainerTestHelper
165+
.getFixedLengthString(keyString, blockSize + chunkSize);
166+
key.write(data.getBytes());
167+
168+
// get the name of a valid container
169+
Assert.assertTrue(key.getOutputStream() instanceof KeyOutputStream);
170+
KeyOutputStream keyOutputStream =
171+
(KeyOutputStream) key.getOutputStream();
172+
List<BlockOutputStreamEntry> streamEntryList =
173+
keyOutputStream.getStreamEntries();
174+
175+
// Assert that 6 block will be preallocated
176+
Assert.assertEquals(6, streamEntryList.size());
177+
key.write(data.getBytes());
178+
key.flush();
179+
long containerId = streamEntryList.get(0).getBlockID().getContainerID();
180+
BlockID blockId = streamEntryList.get(0).getBlockID();
181+
ContainerInfo container =
182+
cluster.getStorageContainerManager().getContainerManager()
183+
.getContainer(ContainerID.valueof(containerId));
184+
Pipeline pipeline =
185+
cluster.getStorageContainerManager().getPipelineManager()
186+
.getPipeline(container.getPipelineID());
187+
List<DatanodeDetails> datanodes = pipeline.getNodes();
188+
cluster.shutdownHddsDatanode(datanodes.get(0));
189+
190+
// The write will fail but exception will be handled and length will be
191+
// updated correctly in OzoneManager once the steam is closed
192+
key.write(data.getBytes());
193+
194+
// shutdown the second datanode
195+
cluster.shutdownHddsDatanode(datanodes.get(1));
196+
key.write(data.getBytes());
197+
key.close();
198+
OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName)
199+
.setBucketName(bucketName).setType(HddsProtos.ReplicationType.RATIS)
200+
.setFactor(HddsProtos.ReplicationFactor.THREE).setKeyName(keyName)
201+
.setRefreshPipeline(true)
202+
.build();
203+
OmKeyInfo keyInfo = cluster.getOzoneManager().lookupKey(keyArgs);
204+
Assert.assertEquals(4 * data.getBytes().length, keyInfo.getDataSize());
205+
validateData(keyName,
206+
data.concat(data).concat(data).concat(data).getBytes());
155207
}
156208

157209
private OzoneOutputStream createKey(String keyName, ReplicationType type,

0 commit comments

Comments
 (0)