Skip to content

Commit e7d2a31

Browse files
lnbest0707lnbest0707
authored
Merge branch 'apache:trunk' into YARN-11391 
2 parents 23002be + 5022003 commit e7d2a31

File tree

8 files changed

+213
-9
lines changed

8 files changed

+213
-9
lines changed

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -628,6 +628,12 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
628628
public static final String DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_KEY =
629629
"dfs.namenode.read-lock-reporting-threshold-ms";
630630
public static final long DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_DEFAULT = 5000L;
631+
632+
public static final String DFS_NAMENODE_ACCESS_CONTROL_ENFORCER_REPORTING_THRESHOLD_MS_KEY
633+
= "dfs.namenode.access-control-enforcer-reporting-threshold-ms";
634+
public static final long DFS_NAMENODE_ACCESS_CONTROL_ENFORCER_REPORTING_THRESHOLD_MS_DEFAULT
635+
= 1000L;
636+
631637
// Threshold for how long the lock warnings must be suppressed
632638
public static final String DFS_LOCK_SUPPRESS_WARNING_INTERVAL_KEY =
633639
"dfs.lock.suppress.warning.interval";

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,8 @@
8787
import static org.apache.hadoop.fs.CommonConfigurationKeys.FS_PROTECTED_DIRECTORIES;
8888
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_DEFAULT;
8989
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY;
90+
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ACCESS_CONTROL_ENFORCER_REPORTING_THRESHOLD_MS_DEFAULT;
91+
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ACCESS_CONTROL_ENFORCER_REPORTING_THRESHOLD_MS_KEY;
9092
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_QUOTA_BY_STORAGETYPE_ENABLED_DEFAULT;
9193
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_QUOTA_BY_STORAGETYPE_ENABLED_KEY;
9294
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PROTECTED_SUBDIRECTORIES_ENABLE;
@@ -181,6 +183,8 @@ private static INodeDirectory createRoot(FSNamesystem namesystem) {
181183
* ACL-related operations.
182184
*/
183185
private final boolean aclsEnabled;
186+
/** Threshold to print a warning. */
187+
private final long accessControlEnforcerReportingThresholdMs;
184188
/**
185189
* Support for POSIX ACL inheritance. Not final for testing purpose.
186190
*/
@@ -388,6 +392,10 @@ public enum DirOp {
388392
DFS_PROTECTED_SUBDIRECTORIES_ENABLE,
389393
DFS_PROTECTED_SUBDIRECTORIES_ENABLE_DEFAULT);
390394

395+
this.accessControlEnforcerReportingThresholdMs = conf.getLong(
396+
DFS_NAMENODE_ACCESS_CONTROL_ENFORCER_REPORTING_THRESHOLD_MS_KEY,
397+
DFS_NAMENODE_ACCESS_CONTROL_ENFORCER_REPORTING_THRESHOLD_MS_DEFAULT);
398+
391399
Preconditions.checkArgument(this.inodeXAttrsLimit >= 0,
392400
"Cannot set a negative limit on the number of xattrs per inode (%s).",
393401
DFSConfigKeys.DFS_NAMENODE_MAX_XATTRS_PER_INODE_KEY);
@@ -1869,7 +1877,8 @@ FSPermissionChecker getPermissionChecker(String fsOwner, String superGroup,
18691877
UserGroupInformation ugi) throws AccessControlException {
18701878
return new FSPermissionChecker(
18711879
fsOwner, superGroup, ugi, getUserFilteredAttributeProvider(ugi),
1872-
useAuthorizationWithContextAPI);
1880+
useAuthorizationWithContextAPI,
1881+
accessControlEnforcerReportingThresholdMs);
18731882
}
18741883

18751884
void checkOwner(FSPermissionChecker pc, INodesInPath iip)

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java‎

Lines changed: 102 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,13 @@
2121
import java.util.ArrayList;
2222
import java.util.Collection;
2323
import java.util.List;
24+
import java.util.Optional;
2425
import java.util.Stack;
26+
import java.util.function.LongFunction;
2527

2628
import org.apache.hadoop.util.Preconditions;
2729
import org.apache.hadoop.ipc.CallerContext;
30+
import org.apache.hadoop.util.Time;
2831
import org.slf4j.Logger;
2932
import org.slf4j.LoggerFactory;
3033
import org.apache.hadoop.fs.FSExceptionMessages;
@@ -87,19 +90,21 @@ private String toAccessControlString(INodeAttributes inodeAttrib,
8790
private final boolean isSuper;
8891
private final INodeAttributeProvider attributeProvider;
8992
private final boolean authorizeWithContext;
93+
private final long accessControlEnforcerReportingThresholdMs;
9094

9195
private static ThreadLocal<String> operationType = new ThreadLocal<>();
9296

9397
protected FSPermissionChecker(String fsOwner, String supergroup,
9498
UserGroupInformation callerUgi,
9599
INodeAttributeProvider attributeProvider) {
96-
this(fsOwner, supergroup, callerUgi, attributeProvider, false);
100+
this(fsOwner, supergroup, callerUgi, attributeProvider, false, 0);
97101
}
98102

99103
protected FSPermissionChecker(String fsOwner, String supergroup,
100104
UserGroupInformation callerUgi,
101105
INodeAttributeProvider attributeProvider,
102-
boolean useAuthorizationWithContextAPI) {
106+
boolean useAuthorizationWithContextAPI,
107+
long accessControlEnforcerReportingThresholdMs) {
103108
this.fsOwner = fsOwner;
104109
this.supergroup = supergroup;
105110
this.callerUgi = callerUgi;
@@ -117,6 +122,38 @@ protected FSPermissionChecker(String fsOwner, String supergroup,
117122
} else {
118123
authorizeWithContext = useAuthorizationWithContextAPI;
119124
}
125+
this.accessControlEnforcerReportingThresholdMs
126+
= accessControlEnforcerReportingThresholdMs;
127+
}
128+
129+
private String checkAccessControlEnforcerSlowness(
130+
long elapsedMs, AccessControlEnforcer ace,
131+
boolean checkSuperuser, AuthorizationContext context) {
132+
return checkAccessControlEnforcerSlowness(elapsedMs,
133+
accessControlEnforcerReportingThresholdMs, ace.getClass(), checkSuperuser,
134+
context.getPath(), context.getOperationName(),
135+
context.getCallerContext());
136+
}
137+
138+
/** @return the warning message if there is any. */
139+
static String checkAccessControlEnforcerSlowness(
140+
long elapsedMs, long thresholdMs, Class<? extends AccessControlEnforcer> clazz,
141+
boolean checkSuperuser, String path, String op, Object caller) {
142+
if (!LOG.isWarnEnabled()) {
143+
return null;
144+
}
145+
if (thresholdMs <= 0) {
146+
return null;
147+
}
148+
if (elapsedMs > thresholdMs) {
149+
final String message = clazz + " ran for "
150+
+ elapsedMs + "ms (threshold=" + thresholdMs + "ms) to check "
151+
+ (checkSuperuser ? "superuser" : "permission")
152+
+ " on " + path + " for " + op + " from caller " + caller;
153+
LOG.warn(message, new Throwable("TRACE"));
154+
return message;
155+
}
156+
return null;
120157
}
121158

122159
public static void setOperationType(String opType) {
@@ -139,9 +176,70 @@ public INodeAttributeProvider getAttributesProvider() {
139176
return attributeProvider;
140177
}
141178

179+
@FunctionalInterface
180+
interface CheckPermission {
181+
void run() throws AccessControlException;
182+
}
183+
184+
static String runCheckPermission(CheckPermission checker,
185+
LongFunction<String> checkElapsedMs) throws AccessControlException {
186+
final String message;
187+
final long start = Time.monotonicNow();
188+
try {
189+
checker.run();
190+
} finally {
191+
final long end = Time.monotonicNow();
192+
message = checkElapsedMs.apply(end - start);
193+
}
194+
return message;
195+
}
196+
142197
private AccessControlEnforcer getAccessControlEnforcer() {
143-
return (attributeProvider != null)
144-
? attributeProvider.getExternalAccessControlEnforcer(this) : this;
198+
final AccessControlEnforcer e = Optional.ofNullable(attributeProvider)
199+
.map(p -> p.getExternalAccessControlEnforcer(this))
200+
.orElse(this);
201+
if (e == this) {
202+
return this;
203+
}
204+
// For an external AccessControlEnforcer, check for slowness.
205+
return new AccessControlEnforcer() {
206+
@Override
207+
public void checkPermission(
208+
String filesystemOwner, String superGroup, UserGroupInformation ugi,
209+
INodeAttributes[] inodeAttrs, INode[] inodes, byte[][] pathByNameArr,
210+
int snapshotId, String path, int ancestorIndex, boolean doCheckOwner,
211+
FsAction ancestorAccess, FsAction parentAccess, FsAction access,
212+
FsAction subAccess, boolean ignoreEmptyDir)
213+
throws AccessControlException {
214+
runCheckPermission(
215+
() -> e.checkPermission(filesystemOwner, superGroup, ugi,
216+
inodeAttrs, inodes, pathByNameArr, snapshotId, path,
217+
ancestorIndex, doCheckOwner, ancestorAccess, parentAccess,
218+
access, subAccess, ignoreEmptyDir),
219+
elapsedMs -> checkAccessControlEnforcerSlowness(elapsedMs,
220+
accessControlEnforcerReportingThresholdMs,
221+
e.getClass(), false, path, operationType.get(),
222+
CallerContext.getCurrent()));
223+
}
224+
225+
@Override
226+
public void checkPermissionWithContext(AuthorizationContext context)
227+
throws AccessControlException {
228+
runCheckPermission(
229+
() -> e.checkPermissionWithContext(context),
230+
elapsedMs -> checkAccessControlEnforcerSlowness(elapsedMs,
231+
e, false, context));
232+
}
233+
234+
@Override
235+
public void checkSuperUserPermissionWithContext(
236+
AuthorizationContext context) throws AccessControlException {
237+
runCheckPermission(
238+
() -> e.checkSuperUserPermissionWithContext(context),
239+
elapsedMs -> checkAccessControlEnforcerSlowness(elapsedMs,
240+
e, true, context));
241+
}
242+
};
145243
}
146244

147245
private AuthorizationContext getAuthorizationContextForSuperUser(

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -320,6 +320,10 @@ public void blockIdCK(String blockId) {
320320
}
321321
out.println("No. of corrupted Replica: " +
322322
numberReplicas.corruptReplicas());
323+
// for striped blocks only and number of redundant internal block replicas.
324+
if (blockInfo.isStriped()) {
325+
out.println("No. of redundant Replica: " + numberReplicas.redundantInternalBlocks());
326+
}
323327
//record datanodes that have corrupted block replica
324328
Collection<DatanodeDescriptor> corruptionRecord = null;
325329
if (blockManager.getCorruptReplicas(block) != null) {

‎hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5457,8 +5457,6 @@
54575457
</property>
54585458

54595459
<property>
5460-
<name>dfs.storage.policy.satisfier.enabled</name>
5461-
<value>false</value>
54625460
<name>dfs.storage.policy.satisfier.mode</name>
54635461
<value>none</value>
54645462
<description>

‎hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuthorizationContext.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ public void testLegacyAPI() throws IOException {
103103
thenReturn(mockEnforcer);
104104

105105
FSPermissionChecker checker = new FSPermissionChecker(
106-
fsOwner, superGroup, ugi, mockINodeAttributeProvider, false);
106+
fsOwner, superGroup, ugi, mockINodeAttributeProvider, false, 0);
107107

108108
when(iip.getPathSnapshotId()).thenReturn(snapshotId);
109109
when(iip.getINodesArray()).thenReturn(inodes);
@@ -128,7 +128,7 @@ public void testCheckPermissionWithContextAPI() throws IOException {
128128

129129
// force it to use the new, checkPermissionWithContext API.
130130
FSPermissionChecker checker = new FSPermissionChecker(
131-
fsOwner, superGroup, ugi, mockINodeAttributeProvider, true);
131+
fsOwner, superGroup, ugi, mockINodeAttributeProvider, true, 0);
132132

133133
String operationName = "abc";
134134
FSPermissionChecker.setOperationType(operationName);

‎hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSPermissionChecker.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040

4141
import java.io.IOException;
4242
import java.util.Arrays;
43+
import java.util.function.LongFunction;
4344

4445
import org.apache.hadoop.conf.Configuration;
4546
import org.apache.hadoop.fs.Path;
@@ -52,6 +53,7 @@
5253
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
5354
import org.apache.hadoop.security.AccessControlException;
5455
import org.apache.hadoop.security.UserGroupInformation;
56+
import org.junit.Assert;
5557
import org.junit.Before;
5658
import org.junit.Test;
5759
import org.mockito.invocation.InvocationOnMock;
@@ -446,4 +448,29 @@ private static INodeFile createINodeFile(INodeDirectory parent, String name,
446448
parent.addChild(inodeFile);
447449
return inodeFile;
448450
}
451+
452+
@Test
453+
public void testCheckAccessControlEnforcerSlowness() throws Exception {
454+
final long thresholdMs = 10;
455+
final LongFunction<String> checkAccessControlEnforcerSlowness =
456+
elapsedMs -> FSPermissionChecker.checkAccessControlEnforcerSlowness(
457+
elapsedMs, thresholdMs, INodeAttributeProvider.AccessControlEnforcer.class,
458+
false, "/foo", "mkdir", "client");
459+
460+
final String m1 = FSPermissionChecker.runCheckPermission(
461+
() -> FSPermissionChecker.LOG.info("Fast runner"),
462+
checkAccessControlEnforcerSlowness);
463+
Assert.assertNull(m1);
464+
465+
final String m2 = FSPermissionChecker.runCheckPermission(() -> {
466+
FSPermissionChecker.LOG.info("Slow runner");
467+
try {
468+
Thread.sleep(20);
469+
} catch (InterruptedException e) {
470+
Thread.currentThread().interrupt();
471+
throw new IllegalStateException(e);
472+
}
473+
}, checkAccessControlEnforcerSlowness);
474+
Assert.assertNotNull(m2);
475+
}
449476
}

‎hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2450,6 +2450,68 @@ public void testFsckMissingECFile() throws Exception {
24502450
assertTrue(outStr.contains("has 1 CORRUPT blocks"));
24512451
}
24522452

2453+
@Test
2454+
public void testFsckECBlockIdRedundantInternalBlocks() throws Exception {
2455+
final int dataBlocks = StripedFileTestUtil.getDefaultECPolicy().getNumDataUnits();
2456+
final int parityBlocks = StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits();
2457+
final int cellSize = StripedFileTestUtil.getDefaultECPolicy().getCellSize();
2458+
final short groupSize = (short) (dataBlocks + parityBlocks);
2459+
final File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
2460+
final Path dirPath = new Path("/ec_dir");
2461+
final Path filePath = new Path(dirPath, "file");
2462+
2463+
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
2464+
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir).numDataNodes(groupSize + 1).build();
2465+
cluster.waitActive();
2466+
2467+
DistributedFileSystem fs = cluster.getFileSystem();
2468+
fs.enableErasureCodingPolicy(
2469+
StripedFileTestUtil.getDefaultECPolicy().getName());
2470+
2471+
try {
2472+
fs.mkdirs(dirPath);
2473+
fs.setErasureCodingPolicy(dirPath, StripedFileTestUtil.getDefaultECPolicy().getName());
2474+
DFSTestUtil.createFile(fs, filePath, cellSize * dataBlocks * 2, (short) 1, 0L);
2475+
LocatedBlocks blks = fs.getClient().getLocatedBlocks(filePath.toString(), 0);
2476+
LocatedStripedBlock block = (LocatedStripedBlock) blks.getLastLocatedBlock();
2477+
Assert.assertEquals(groupSize, block.getLocations().length);
2478+
2479+
//general test.
2480+
String runFsckResult = runFsck(conf, 0, true, "/",
2481+
"-blockId", block.getBlock().getBlockName());
2482+
assertTrue(runFsckResult.contains(block.getBlock().getBlockName()));
2483+
assertTrue(runFsckResult.contains("No. of Expected Replica: " + groupSize));
2484+
assertTrue(runFsckResult.contains("No. of live Replica: " + groupSize));
2485+
assertTrue(runFsckResult.contains("No. of redundant Replica: " + 0));
2486+
2487+
// stop a dn.
2488+
DatanodeInfo dnToStop = block.getLocations()[0];
2489+
MiniDFSCluster.DataNodeProperties dnProp = cluster.stopDataNode(dnToStop.getXferAddr());
2490+
cluster.setDataNodeDead(dnToStop);
2491+
2492+
// wait for reconstruction to happen.
2493+
DFSTestUtil.waitForReplication(fs, filePath, groupSize, 15 * 1000);
2494+
2495+
// bring the dn back: 10 internal blocks now.
2496+
cluster.restartDataNode(dnProp);
2497+
cluster.waitActive();
2498+
2499+
blks = fs.getClient().getLocatedBlocks(filePath.toString(), 0);
2500+
block = (LocatedStripedBlock) blks.getLastLocatedBlock();
2501+
Assert.assertEquals(groupSize + 1, block.getLocations().length);
2502+
2503+
//general test, number of redundant internal block replicas.
2504+
runFsckResult = runFsck(conf, 0, true, "/",
2505+
"-blockId", block.getBlock().getBlockName());
2506+
assertTrue(runFsckResult.contains(block.getBlock().getBlockName()));
2507+
assertTrue(runFsckResult.contains("No. of Expected Replica: " + groupSize));
2508+
assertTrue(runFsckResult.contains("No. of live Replica: " + groupSize));
2509+
assertTrue(runFsckResult.contains("No. of redundant Replica: " + 1));
2510+
} finally {
2511+
cluster.shutdown();
2512+
}
2513+
}
2514+
24532515
private void waitForUnrecoverableBlockGroup(Configuration configuration)
24542516
throws TimeoutException, InterruptedException {
24552517
GenericTestUtils.waitFor(new Supplier<Boolean>() {

0 commit comments

Comments
 (0)