Skip to content

Commit d1daf26

Browse files
authored
Fixes HDFS-17181 by routing all CREATE requests to the BlockManager (#6108)
1 parent 1336c36 commit d1daf26

File tree

1 file changed

+29
-23
lines changed

1 file changed

+29
-23
lines changed

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java

Lines changed: 29 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import java.util.HashSet;
3737
import java.util.List;
3838
import java.util.Map;
39+
import java.util.Set;
3940
import java.util.concurrent.ExecutionException;
4041

4142
import javax.servlet.ServletContext;
@@ -278,6 +279,9 @@ protected void queueExternalCall(ExternalCall call)
278279
namenode.queueExternalCall(call);
279280
}
280281

282+
/**
283+
* Chooses a Datanode to redirect a request to.
284+
*/
281285
@VisibleForTesting
282286
static DatanodeInfo chooseDatanode(final NameNode namenode,
283287
final String path, final HttpOpParam.Op op, final long openOffset,
@@ -288,18 +292,18 @@ static DatanodeInfo chooseDatanode(final NameNode namenode,
288292
throw new IOException("Namesystem has not been initialized yet.");
289293
}
290294
final BlockManager bm = fsn.getBlockManager();
291-
292-
HashSet<Node> excludes = new HashSet<Node>();
295+
296+
Set<Node> excludes = new HashSet<>();
293297
if (excludeDatanodes != null) {
294298
for (String host : StringUtils
295299
.getTrimmedStringCollection(excludeDatanodes)) {
296-
int idx = host.indexOf(":");
300+
int idx = host.indexOf(':');
297301
Node excludeNode = null;
298-
if (idx != -1) {
299-
excludeNode = bm.getDatanodeManager().getDatanodeByXferAddr(
300-
host.substring(0, idx), Integer.parseInt(host.substring(idx + 1)));
301-
} else {
302+
if (idx == -1) {
302303
excludeNode = bm.getDatanodeManager().getDatanodeByHost(host);
304+
} else {
305+
excludeNode = bm.getDatanodeManager().getDatanodeByXferAddr(
306+
host.substring(0, idx), Integer.parseInt(host.substring(idx + 1)));
303307
}
304308

305309
if (excludeNode != null) {
@@ -311,25 +315,15 @@ static DatanodeInfo chooseDatanode(final NameNode namenode,
311315
}
312316
}
313317

314-
if (op == PutOpParam.Op.CREATE) {
315-
//choose a datanode near to client
316-
final DatanodeDescriptor clientNode = bm.getDatanodeManager(
317-
).getDatanodeByHost(remoteAddr);
318-
if (clientNode != null) {
319-
final DatanodeStorageInfo[] storages = bm.chooseTarget4WebHDFS(
320-
path, clientNode, excludes, blocksize);
321-
if (storages.length > 0) {
322-
return storages[0].getDatanodeDescriptor();
323-
}
324-
}
325-
} else if (op == GetOpParam.Op.OPEN
318+
// For these operations choose a datanode containing a replica
319+
if (op == GetOpParam.Op.OPEN
326320
|| op == GetOpParam.Op.GETFILECHECKSUM
327321
|| op == PostOpParam.Op.APPEND) {
328-
//choose a datanode containing a replica
329322
final NamenodeProtocols np = getRPCServer(namenode);
330323
if (status == null) {
331324
throw new FileNotFoundException("File " + path + " not found.");
332325
}
326+
333327
final long len = status.getLen();
334328
if (op == GetOpParam.Op.OPEN) {
335329
if (openOffset < 0L || (openOffset >= len && len > 0)) {
@@ -344,10 +338,22 @@ static DatanodeInfo chooseDatanode(final NameNode namenode,
344338
final int count = locations.locatedBlockCount();
345339
if (count > 0) {
346340
return bestNode(locations.get(0).getLocations(), excludes);
341+
} else {
342+
throw new IOException("Block could not be located. Path=" + path + ", offset=" + offset);
347343
}
348344
}
349345
}
350346

347+
// All other operations don't affect a specific node so let the BlockManager pick a target
348+
DatanodeDescriptor clientNode = bm.getDatanodeManager(
349+
).getDatanodeByHost(remoteAddr);
350+
351+
DatanodeStorageInfo[] storages = bm.chooseTarget4WebHDFS(
352+
path, clientNode, excludes, blocksize);
353+
if (storages.length > 0) {
354+
return storages[0].getDatanodeDescriptor();
355+
}
356+
351357
return (DatanodeDescriptor)bm.getDatanodeManager().getNetworkTopology(
352358
).chooseRandom(NodeBase.ROOT, excludes);
353359
}
@@ -358,13 +364,13 @@ static DatanodeInfo chooseDatanode(final NameNode namenode,
358364
* to return the first element of the node here.
359365
*/
360366
protected static DatanodeInfo bestNode(DatanodeInfo[] nodes,
361-
HashSet<Node> excludes) throws IOException {
367+
Set<Node> excludes) throws IOException {
362368
for (DatanodeInfo dn: nodes) {
363-
if (false == dn.isDecommissioned() && false == excludes.contains(dn)) {
369+
if (!dn.isDecommissioned() && !excludes.contains(dn)) {
364370
return dn;
365371
}
366372
}
367-
throw new IOException("No active nodes contain this block");
373+
throw new IOException("No active and not excluded nodes contain this block");
368374
}
369375

370376
public long renewDelegationToken(Token<DelegationTokenIdentifier> token)

0 commit comments

Comments
 (0)