From b6957d74806004b85674016af809acd72c32c42c Mon Sep 17 00:00:00 2001 From: zhangshuyan Date: Sun, 17 Sep 2023 17:40:51 +0800 Subject: [PATCH 1/5] HDFS-17197. Show file replication when listing corrupt files. --- .../hadoop/hdfs/server/namenode/FSNamesystem.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index eb8022dc63451..04313bacac5bb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -6131,15 +6131,17 @@ void releaseBackupNode(NamenodeRegistration registration) static class CorruptFileBlockInfo { final String path; final Block block; + final int replication; - public CorruptFileBlockInfo(String p, Block b) { + public CorruptFileBlockInfo(String p, Block b, int r) { path = p; block = b; + replication = r; } @Override public String toString() { - return block.getBlockName() + "\t" + path; + return block.getBlockName() + "\t" + replication + "\t" + path; } } /** @@ -6195,7 +6197,11 @@ Collection listCorruptFileBlocks(String path, if (inode != null) { String src = inode.getFullPathName(); if (isParentEntry(src, path)) { - corruptFiles.add(new CorruptFileBlockInfo(src, blk)); + int repl = 0; + if (inode.isFile()) { + repl = inode.asFile().getFileReplication(); + } + corruptFiles.add(new CorruptFileBlockInfo(src, blk, repl)); count++; if (count >= maxCorruptFileBlocksReturn) break; From 16e9ef6d27294c8c02121e9209a705fc56e57e31 Mon Sep 17 00:00:00 2001 From: zhangshuyan Date: Mon, 18 Sep 2023 10:15:56 +0800 Subject: [PATCH 2/5] fix code style. --- .../org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 04313bacac5bb..996042f3efc2c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -6131,9 +6131,9 @@ void releaseBackupNode(NamenodeRegistration registration) static class CorruptFileBlockInfo { final String path; final Block block; - final int replication; + private final int replication; - public CorruptFileBlockInfo(String p, Block b, int r) { + CorruptFileBlockInfo(String p, Block b, int r) { path = p; block = b; replication = r; From 77b8819990c5244a6600811030e631b7d914c382 Mon Sep 17 00:00:00 2001 From: zhangshuyan Date: Tue, 19 Sep 2023 20:04:03 +0800 Subject: [PATCH 3/5] feedback review. --- .../apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 996042f3efc2c..20707b2952019 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -6141,7 +6141,8 @@ static class CorruptFileBlockInfo { @Override public String toString() { - return block.getBlockName() + "\t" + replication + "\t" + path; + return block.getBlockName() + "\t" + + (replication == -1 ? "EC" : replication) + "\t" + path; } } /** @@ -6199,7 +6200,11 @@ Collection listCorruptFileBlocks(String path, if (isParentEntry(src, path)) { int repl = 0; if (inode.isFile()) { - repl = inode.asFile().getFileReplication(); + if (inode.asFile().isStriped()) { + repl = -1; + } else { + repl = inode.asFile().getFileReplication(); + } } corruptFiles.add(new CorruptFileBlockInfo(src, blk, repl)); count++; From fced8099a4edc9fac6a2b6ad0ce0f0d97c1c4565 Mon Sep 17 00:00:00 2001 From: zhangshuyan Date: Wed, 20 Sep 2023 12:01:13 +0800 Subject: [PATCH 4/5] refactor. --- .../hdfs/server/namenode/FSNamesystem.java | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 20707b2952019..d410161b04f40 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -96,6 +96,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_DIFF_LISTING_LIMIT_DEFAULT; import static org.apache.hadoop.hdfs.DFSUtil.isParentEntry; +import java.util.Objects; import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.text.CaseUtils; @@ -6132,17 +6133,19 @@ static class CorruptFileBlockInfo { final String path; final Block block; private final int replication; + private final String ecPolicy; - CorruptFileBlockInfo(String p, Block b, int r) { + CorruptFileBlockInfo(String p, Block b, int r, String ec) { path = p; block = b; replication = r; + ecPolicy = ec; } @Override public String toString() { return block.getBlockName() + "\t" + - (replication == -1 ? "EC" : replication) + "\t" + path; + (replication == -1 ? ecPolicy : replication) + "\t" + path; } } /** @@ -6198,15 +6201,21 @@ Collection listCorruptFileBlocks(String path, if (inode != null) { String src = inode.getFullPathName(); if (isParentEntry(src, path)) { - int repl = 0; + int repl = -1; + String ecPolicyName = null; if (inode.isFile()) { if (inode.asFile().isStriped()) { - repl = -1; + ErasureCodingPolicy ecPolicy = + ErasureCodingPolicyManager.getInstance() + .getByID(inode.asFile().getErasureCodingPolicyID()); + if (ecPolicy != null) { + ecPolicyName = ecPolicy.getName(); + } } else { repl = inode.asFile().getFileReplication(); } } - corruptFiles.add(new CorruptFileBlockInfo(src, blk, repl)); + corruptFiles.add(new CorruptFileBlockInfo(src, blk, repl, ecPolicyName)); count++; if (count >= maxCorruptFileBlocksReturn) break; From 3bcf9571f354ba8368ed4a70283af387a83f2c4d Mon Sep 17 00:00:00 2001 From: zhangshuyan Date: Wed, 20 Sep 2023 21:04:38 +0800 Subject: [PATCH 5/5] fix code style. --- .../org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index d410161b04f40..d9b165f96ee0c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -96,7 +96,6 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_DIFF_LISTING_LIMIT_DEFAULT; import static org.apache.hadoop.hdfs.DFSUtil.isParentEntry; -import java.util.Objects; import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.text.CaseUtils;