This is an automated email from the ASF dual-hosted git repository.
tomscut pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 6cc56d40f5c HDFS-17135. Update fsck -blockId to display excess state
info of blocks (#5904)
6cc56d40f5c is described below
commit 6cc56d40f5c433d9dc3475f53060607d68ebf236
Author: huhaiyang <huhaiyang...@126.com>
AuthorDate: Mon Jul 31 09:21:22 2023 +0800
HDFS-17135. Update fsck -blockId to display excess state info of blocks
(#5904)
Reviewed-by: He Xiaoqiao <hexiaoq...@apache.org>
Reviewed-by: Shilun Fan <slfan1...@apache.org>
Signed-off-by: Tao Li <toms...@apache.org>
---
.../hadoop/hdfs/server/namenode/NamenodeFsck.java | 3 ++
.../java/org/apache/hadoop/hdfs/tools/DFSck.java | 2 +
.../hadoop/hdfs/server/namenode/TestFsck.java | 49 ++++++++++++++++++++--
3 files changed, 51 insertions(+), 3 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index dfdb12285fb..bc6df0141c6 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -127,6 +127,7 @@ public class NamenodeFsck implements
DataEncryptionKeyFactory {
"is ENTERING MAINTENANCE";
public static final String IN_MAINTENANCE_STATUS = "is IN MAINTENANCE";
public static final String STALE_STATUS = "is STALE";
+ public static final String EXCESS_STATUS = "is EXCESS";
public static final String NONEXISTENT_STATUS = "does not exist";
public static final String FAILURE_STATUS = "FAILED";
public static final String UNDEFINED = "undefined";
@@ -373,6 +374,8 @@ public class NamenodeFsck implements
DataEncryptionKeyFactory {
out.print(IN_MAINTENANCE_STATUS);
} else if (dn.isStale(this.staleInterval)) {
out.print(STALE_STATUS);
+ } else if (blockManager.isExcess(dn, blockManager.getStoredBlock(block))) {
+ out.print(EXCESS_STATUS);
} else {
out.print(HEALTHY_STATUS);
}
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
index 6587ffcce3c..9f0288ebf3f 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
@@ -396,6 +396,8 @@ public class DFSck extends Configured implements Tool {
errCode = 0;
} else if (lastLine.contains("Incorrect blockId format:")) {
errCode = 0;
+ } else if (lastLine.endsWith(NamenodeFsck.EXCESS_STATUS)) {
+ errCode = 0;
} else if (lastLine.endsWith(NamenodeFsck.DECOMMISSIONED_STATUS)) {
errCode = 2;
} else if (lastLine.endsWith(NamenodeFsck.DECOMMISSIONING_STATUS)) {
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
index 5db4e029cb9..7d6e7980be5 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
@@ -1056,8 +1056,6 @@ public class TestFsck {
assertTrue(fsckOut.contains("(DECOMMISSIONED)"));
assertFalse(fsckOut.contains("(ENTERING MAINTENANCE)"));
assertFalse(fsckOut.contains("(IN MAINTENANCE)"));
-
-
}
/** Test if fsck can return -1 in case of failure.
@@ -1834,6 +1832,51 @@ public class TestFsck {
assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS));
}
+ /**
+ * Test for blockIdCK with block excess.
+ */
+ @Test
+ public void testBlockIdCKExcess() throws Exception {
+ final Configuration configuration = new Configuration();
+ // Disable redundancy monitor check so that excess block can be verified.
+
configuration.setLong(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
5000);
+
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(configuration).
+ numDataNodes(2).build()) {
+ cluster.waitActive();
+ final DistributedFileSystem fs = cluster.getFileSystem();
+
+ // Create file.
+ Path file = new Path("/test");
+ long fileLength = 512;
+ DFSTestUtil.createFile(fs, file, fileLength, (short) 2, 0L);
+ DFSTestUtil.waitReplication(fs, file, (short) 2);
+
+ List<LocatedBlock> locatedBlocks = DFSTestUtil.getAllBlocks(fs, file);
+ assertEquals(1, locatedBlocks.size());
+ String blockName = locatedBlocks.get(0).getBlock().getBlockName();
+
+ // Validate block is HEALTHY.
+ String outStr = runFsck(configuration, 0, true,
+ "/", "-blockId", blockName);
+ assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
+ assertTrue(outStr.contains("No. of Expected Replica: " + 2));
+ assertTrue(outStr.contains("No. of live Replica: " + 2));
+ assertTrue(outStr.contains("No. of excess Replica: " + 0));
+
+ // Make the block on one datanode enter excess state.
+ fs.setReplication(file, (short)1);
+
+ // Validate the one block is EXCESS.
+ outStr = runFsck(configuration, 0, true,
+ "/", "-blockId", blockName);
+ assertTrue(outStr.contains(NamenodeFsck.EXCESS_STATUS));
+ assertTrue(outStr.contains("No. of Expected Replica: " + 1));
+ assertTrue(outStr.contains("No. of live Replica: " + 1));
+ assertTrue(outStr.contains("No. of excess Replica: " + 1));
+ }
+ }
+
private void writeFile(final DistributedFileSystem dfs,
Path dir, String fileName) throws IOException {
Path filePath = new Path(dir.toString() + Path.SEPARATOR + fileName);
@@ -1961,7 +2004,7 @@ public class TestFsck {
// check the replica status should be healthy(0) after decommission
// is done
- String fsckOut = runFsck(conf, 0, true, testFile);
+ runFsck(conf, 0, true, testFile);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
