Skip to content

Commit 09dcc8c

Browse files
slfan1989slfan1989
authored
Merge branch 'trunk' into YARN-11158-V3
2 parents 45479a9 + 5d6ab15 commit 09dcc8c

File tree

41 files changed

+1067
-233
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+1067
-233
lines changed

‎hadoop-dist/pom.xml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,11 +41,21 @@
4141
<artifactId>hadoop-hdfs-client</artifactId>
4242
<scope>provided</scope>
4343
</dependency>
44+
<dependency>
45+
<groupId>org.apache.hadoop</groupId>
46+
<artifactId>hadoop-hdfs-native-client</artifactId>
47+
<scope>provided</scope>
48+
</dependency>
4449
<dependency>
4550
<groupId>org.apache.hadoop</groupId>
4651
<artifactId>hadoop-mapreduce-client-app</artifactId>
4752
<scope>provided</scope>
4853
</dependency>
54+
<dependency>
55+
<groupId>org.apache.hadoop</groupId>
56+
<artifactId>hadoop-mapreduce-client-nativetask</artifactId>
57+
<scope>provided</scope>
58+
</dependency>
4959
<dependency>
5060
<groupId>org.apache.hadoop</groupId>
5161
<artifactId>hadoop-yarn-api</artifactId>

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminBackoffMonitor.java‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.apache.hadoop.hdfs.server.namenode.INodeId;
2525
import org.apache.hadoop.hdfs.util.LightWeightHashSet;
2626
import org.apache.hadoop.hdfs.util.LightWeightLinkedSet;
27+
import org.apache.hadoop.classification.VisibleForTesting;
2728
import org.slf4j.Logger;
2829
import org.slf4j.LoggerFactory;
2930
import java.util.HashMap;
@@ -70,10 +71,10 @@ public class DatanodeAdminBackoffMonitor extends DatanodeAdminMonitorBase
7071
outOfServiceNodeBlocks = new HashMap<>();
7172

7273
/**
73-
* The numbe of blocks to process when moving blocks to pendingReplication
74+
* The number of blocks to process when moving blocks to pendingReplication
7475
* before releasing and reclaiming the namenode lock.
7576
*/
76-
private int blocksPerLock;
77+
private volatile int blocksPerLock;
7778

7879
/**
7980
* The number of blocks that have been checked on this tick.
@@ -82,7 +83,7 @@ public class DatanodeAdminBackoffMonitor extends DatanodeAdminMonitorBase
8283
/**
8384
* The maximum number of blocks to hold in PendingRep at any time.
8485
*/
85-
private int pendingRepLimit;
86+
private volatile int pendingRepLimit;
8687

8788
/**
8889
* The list of blocks which have been placed onto the replication queue
@@ -801,6 +802,26 @@ private boolean isBlockReplicatedOk(DatanodeDescriptor datanode,
801802
return false;
802803
}
803804

805+
@VisibleForTesting
806+
@Override
807+
public int getPendingRepLimit() {
808+
return pendingRepLimit;
809+
}
810+
811+
public void setPendingRepLimit(int pendingRepLimit) {
812+
this.pendingRepLimit = pendingRepLimit;
813+
}
814+
815+
@VisibleForTesting
816+
@Override
817+
public int getBlocksPerLock() {
818+
return blocksPerLock;
819+
}
820+
821+
public void setBlocksPerLock(int blocksPerLock) {
822+
this.blocksPerLock = blocksPerLock;
823+
}
824+
804825
static class BlockStats {
805826
private LightWeightHashSet<Long> openFiles =
806827
new LightWeightLinkedSet<>();

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.apache.hadoop.hdfs.util.LightWeightHashSet;
2828
import org.apache.hadoop.hdfs.util.LightWeightLinkedSet;
2929
import org.apache.hadoop.util.ChunkedArrayList;
30+
import org.apache.hadoop.classification.VisibleForTesting;
3031
import org.slf4j.Logger;
3132
import org.slf4j.LoggerFactory;
3233

@@ -137,6 +138,28 @@ public int getNumNodesChecked() {
137138
return numNodesChecked;
138139
}
139140

141+
@VisibleForTesting
142+
@Override
143+
public int getPendingRepLimit() {
144+
return 0;
145+
}
146+
147+
@Override
148+
public void setPendingRepLimit(int pendingRepLimit) {
149+
// nothing.
150+
}
151+
152+
@VisibleForTesting
153+
@Override
154+
public int getBlocksPerLock() {
155+
return 0;
156+
}
157+
158+
@Override
159+
public void setBlocksPerLock(int blocksPerLock) {
160+
// nothing.
161+
}
162+
140163
@Override
141164
public void run() {
142165
LOG.debug("DatanodeAdminMonitor is running.");

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminManager.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -419,4 +419,30 @@ void runMonitorForTest() throws ExecutionException, InterruptedException {
419419
executor.submit(monitor).get();
420420
}
421421

422+
public void refreshPendingRepLimit(int pendingRepLimit, String key) {
423+
ensurePositiveInt(pendingRepLimit, key);
424+
this.monitor.setPendingRepLimit(pendingRepLimit);
425+
}
426+
427+
@VisibleForTesting
428+
public int getPendingRepLimit() {
429+
return this.monitor.getPendingRepLimit();
430+
}
431+
432+
public void refreshBlocksPerLock(int blocksPerLock, String key) {
433+
ensurePositiveInt(blocksPerLock, key);
434+
this.monitor.setBlocksPerLock(blocksPerLock);
435+
}
436+
437+
@VisibleForTesting
438+
public int getBlocksPerLock() {
439+
return this.monitor.getBlocksPerLock();
440+
}
441+
442+
private void ensurePositiveInt(int val, String key) {
443+
Preconditions.checkArgument(
444+
(val > 0),
445+
key + " = '" + val + "' is invalid. " +
446+
"It should be a positive, non-zero integer value.");
447+
}
422448
}

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminMonitorInterface.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,4 +37,12 @@ public interface DatanodeAdminMonitorInterface extends Runnable {
3737
void setBlockManager(BlockManager bm);
3838
void setDatanodeAdminManager(DatanodeAdminManager dnm);
3939
void setNameSystem(Namesystem ns);
40+
41+
int getPendingRepLimit();
42+
43+
void setPendingRepLimit(int pendingRepLimit);
44+
45+
int getBlocksPerLock();
46+
47+
void setBlocksPerLock(int blocksPerLock);
4048
}

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,10 @@
205205
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_DEFAULT;
206206
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY;
207207
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_DEFAULT;
208+
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT;
209+
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT_DEFAULT;
210+
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK;
211+
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK_DEFAULT;
208212

209213
import static org.apache.hadoop.util.ExitUtil.terminate;
210214
import static org.apache.hadoop.util.ToolRunner.confirmPrompt;
@@ -353,7 +357,9 @@ public enum OperationCategory {
353357
DFS_BLOCK_INVALIDATE_LIMIT_KEY,
354358
DFS_DATANODE_PEER_STATS_ENABLED_KEY,
355359
DFS_DATANODE_MAX_NODES_TO_REPORT_KEY,
356-
DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY));
360+
DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY,
361+
DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT,
362+
DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK));
357363

358364
private static final String USAGE = "Usage: hdfs namenode ["
359365
+ StartupOption.BACKUP.getName() + "] | \n\t["
@@ -2321,6 +2327,10 @@ protected String reconfigurePropertyImpl(String property, String newVal)
23212327
return reconfigureSlowNodesParameters(datanodeManager, property, newVal);
23222328
} else if (property.equals(DFS_BLOCK_INVALIDATE_LIMIT_KEY)) {
23232329
return reconfigureBlockInvalidateLimit(datanodeManager, property, newVal);
2330+
} else if (property.equals(DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT) ||
2331+
(property.equals(DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK))) {
2332+
return reconfigureDecommissionBackoffMonitorParameters(datanodeManager, property,
2333+
newVal);
23242334
} else {
23252335
throw new ReconfigurationException(property, newVal, getConf().get(
23262336
property));
@@ -2601,6 +2611,34 @@ private String reconfigureBlockInvalidateLimit(final DatanodeManager datanodeMan
26012611
}
26022612
}
26032613

2614+
private String reconfigureDecommissionBackoffMonitorParameters(
2615+
final DatanodeManager datanodeManager, final String property, final String newVal)
2616+
throws ReconfigurationException {
2617+
String newSetting = null;
2618+
try {
2619+
if (property.equals(DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT)) {
2620+
int pendingRepLimit = (newVal == null ?
2621+
DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT_DEFAULT :
2622+
Integer.parseInt(newVal));
2623+
datanodeManager.getDatanodeAdminManager().refreshPendingRepLimit(pendingRepLimit,
2624+
DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT);
2625+
newSetting = String.valueOf(datanodeManager.getDatanodeAdminManager().getPendingRepLimit());
2626+
} else if (property.equals(
2627+
DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK)) {
2628+
int blocksPerLock = (newVal == null ?
2629+
DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK_DEFAULT :
2630+
Integer.parseInt(newVal));
2631+
datanodeManager.getDatanodeAdminManager().refreshBlocksPerLock(blocksPerLock,
2632+
DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK);
2633+
newSetting = String.valueOf(datanodeManager.getDatanodeAdminManager().getBlocksPerLock());
2634+
}
2635+
LOG.info("RECONFIGURE* changed reconfigureDecommissionBackoffMonitorParameters {} to {}",
2636+
property, newSetting);
2637+
return newSetting;
2638+
} catch (IllegalArgumentException e) {
2639+
throw new ReconfigurationException(property, newVal, getConf().get(property), e);
2640+
}
2641+
}
26042642

26052643
@Override // ReconfigurableBase
26062644
protected Configuration getNewConf() {

‎hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
import java.util.ArrayList;
2323
import java.util.List;
2424

25+
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeAdminBackoffMonitor;
26+
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeAdminMonitorInterface;
2527
import org.junit.Test;
2628
import org.junit.Before;
2729
import org.junit.After;
@@ -62,6 +64,8 @@
6264
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY;
6365
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY;
6466
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY;
67+
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT;
68+
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK;
6569
import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_BACKOFF_ENABLE_DEFAULT;
6670

6771
public class TestNameNodeReconfigure {
@@ -567,6 +571,87 @@ private List<Boolean> validatePeerReport(String jsonReport) {
567571
return containReport;
568572
}
569573

574+
@Test
575+
public void testReconfigureDecommissionBackoffMonitorParameters()
576+
throws ReconfigurationException, IOException {
577+
Configuration conf = new HdfsConfiguration();
578+
conf.setClass(DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_MONITOR_CLASS,
579+
DatanodeAdminBackoffMonitor.class, DatanodeAdminMonitorInterface.class);
580+
int defaultPendingRepLimit = 1000;
581+
conf.setInt(DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT, defaultPendingRepLimit);
582+
int defaultBlocksPerLock = 1000;
583+
conf.setInt(DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK,
584+
defaultBlocksPerLock);
585+
586+
try (MiniDFSCluster newCluster = new MiniDFSCluster.Builder(conf).build()) {
587+
newCluster.waitActive();
588+
final NameNode nameNode = newCluster.getNameNode();
589+
final DatanodeManager datanodeManager = nameNode.namesystem
590+
.getBlockManager().getDatanodeManager();
591+
592+
// verify defaultPendingRepLimit.
593+
assertEquals(datanodeManager.getDatanodeAdminManager().getPendingRepLimit(),
594+
defaultPendingRepLimit);
595+
596+
// try invalid pendingRepLimit.
597+
try {
598+
nameNode.reconfigureProperty(DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT,
599+
"non-numeric");
600+
fail("Should not reach here");
601+
} catch (ReconfigurationException e) {
602+
assertEquals("Could not change property " +
603+
"dfs.namenode.decommission.backoff.monitor.pending.limit from '" +
604+
defaultPendingRepLimit + "' to 'non-numeric'", e.getMessage());
605+
}
606+
607+
try {
608+
nameNode.reconfigureProperty(DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT,
609+
"-1");
610+
fail("Should not reach here");
611+
} catch (ReconfigurationException e) {
612+
assertEquals("Could not change property " +
613+
"dfs.namenode.decommission.backoff.monitor.pending.limit from '" +
614+
defaultPendingRepLimit + "' to '-1'", e.getMessage());
615+
}
616+
617+
// try correct pendingRepLimit.
618+
nameNode.reconfigureProperty(DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT,
619+
"20000");
620+
assertEquals(datanodeManager.getDatanodeAdminManager().getPendingRepLimit(), 20000);
621+
622+
// verify defaultBlocksPerLock.
623+
assertEquals(datanodeManager.getDatanodeAdminManager().getBlocksPerLock(),
624+
defaultBlocksPerLock);
625+
626+
// try invalid blocksPerLock.
627+
try {
628+
nameNode.reconfigureProperty(
629+
DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK,
630+
"non-numeric");
631+
fail("Should not reach here");
632+
} catch (ReconfigurationException e) {
633+
assertEquals("Could not change property " +
634+
"dfs.namenode.decommission.backoff.monitor.pending.blocks.per.lock from '" +
635+
defaultBlocksPerLock + "' to 'non-numeric'", e.getMessage());
636+
}
637+
638+
try {
639+
nameNode.reconfigureProperty(
640+
DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK, "-1");
641+
fail("Should not reach here");
642+
} catch (ReconfigurationException e) {
643+
assertEquals("Could not change property " +
644+
"dfs.namenode.decommission.backoff.monitor.pending.blocks.per.lock from '" +
645+
defaultBlocksPerLock + "' to '-1'", e.getMessage());
646+
}
647+
648+
// try correct blocksPerLock.
649+
nameNode.reconfigureProperty(
650+
DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK, "10000");
651+
assertEquals(datanodeManager.getDatanodeAdminManager().getBlocksPerLock(), 10000);
652+
}
653+
}
654+
570655
@After
571656
public void shutDown() throws IOException {
572657
if (cluster != null) {

‎hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@
4343
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY;
4444
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY;
4545
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY;
46+
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT;
47+
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK;
4648

4749
import org.apache.commons.io.FileUtils;
4850
import org.apache.commons.text.TextStringBuilder;
@@ -438,7 +440,7 @@ public void testNameNodeGetReconfigurableProperties() throws IOException, Interr
438440
final List<String> outs = Lists.newArrayList();
439441
final List<String> errs = Lists.newArrayList();
440442
getReconfigurableProperties("namenode", address, outs, errs);
441-
assertEquals(20, outs.size());
443+
assertEquals(22, outs.size());
442444
assertTrue(outs.get(0).contains("Reconfigurable properties:"));
443445
assertEquals(DFS_BLOCK_INVALIDATE_LIMIT_KEY, outs.get(1));
444446
assertEquals(DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY, outs.get(2));
@@ -449,8 +451,10 @@ public void testNameNodeGetReconfigurableProperties() throws IOException, Interr
449451
assertEquals(DFS_IMAGE_PARALLEL_LOAD_KEY, outs.get(7));
450452
assertEquals(DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY, outs.get(8));
451453
assertEquals(DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY, outs.get(9));
452-
assertEquals(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, outs.get(10));
453-
assertEquals(DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY, outs.get(11));
454+
assertEquals(DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK, outs.get(10));
455+
assertEquals(DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT, outs.get(11));
456+
assertEquals(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, outs.get(12));
457+
assertEquals(DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY, outs.get(13));
454458
assertEquals(errs.size(), 0);
455459
}
456460

0 commit comments

Comments
 (0)