This is an automated email from the ASF dual-hosted git repository.
zanderxu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 35ce60eadd1 HDFS-16879. EC: Fsck -blockId shows number of redundant
internal block replicas for EC Blocks (#5264)
35ce60eadd1 is described below
commit 35ce60eadd1d890f00c516b57ebb853ce9c95b37
Author: huhaiyang <huhaiyang...@126.com>
AuthorDate: Wed Jan 4 11:38:32 2023 +0800
HDFS-16879. EC: Fsck -blockId shows number of redundant internal block
replicas for EC Blocks (#5264)
---
.../hadoop/hdfs/server/namenode/NamenodeFsck.java | 4 ++
.../hadoop/hdfs/server/namenode/TestFsck.java | 62 ++++++++++++++++++++++
2 files changed, 66 insertions(+)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index 1e8517edeca..050eda628ee 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -320,6 +320,10 @@ public class NamenodeFsck implements
DataEncryptionKeyFactory {
}
out.println("No. of corrupted Replica: " +
numberReplicas.corruptReplicas());
+ // for striped blocks only and number of redundant internal block
replicas.
+ if (blockInfo.isStriped()) {
+ out.println("No. of redundant Replica: " +
numberReplicas.redundantInternalBlocks());
+ }
//record datanodes that have corrupted block replica
Collection<DatanodeDescriptor> corruptionRecord = null;
if (blockManager.getCorruptReplicas(block) != null) {
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
index 420635e012a..60442c6bd04 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
@@ -2450,6 +2450,68 @@ public class TestFsck {
assertTrue(outStr.contains("has 1 CORRUPT blocks"));
}
+ @Test
+ public void testFsckECBlockIdRedundantInternalBlocks() throws Exception {
+ final int dataBlocks =
StripedFileTestUtil.getDefaultECPolicy().getNumDataUnits();
+ final int parityBlocks =
StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits();
+ final int cellSize =
StripedFileTestUtil.getDefaultECPolicy().getCellSize();
+ final short groupSize = (short) (dataBlocks + parityBlocks);
+ final File builderBaseDir = new
File(GenericTestUtils.getRandomizedTempPath());
+ final Path dirPath = new Path("/ec_dir");
+ final Path filePath = new Path(dirPath, "file");
+
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
+ cluster = new MiniDFSCluster.Builder(conf,
builderBaseDir).numDataNodes(groupSize + 1).build();
+ cluster.waitActive();
+
+ DistributedFileSystem fs = cluster.getFileSystem();
+ fs.enableErasureCodingPolicy(
+ StripedFileTestUtil.getDefaultECPolicy().getName());
+
+ try {
+ fs.mkdirs(dirPath);
+ fs.setErasureCodingPolicy(dirPath,
StripedFileTestUtil.getDefaultECPolicy().getName());
+ DFSTestUtil.createFile(fs, filePath, cellSize * dataBlocks * 2, (short)
1, 0L);
+ LocatedBlocks blks =
fs.getClient().getLocatedBlocks(filePath.toString(), 0);
+ LocatedStripedBlock block = (LocatedStripedBlock)
blks.getLastLocatedBlock();
+ Assert.assertEquals(groupSize, block.getLocations().length);
+
+ //general test.
+ String runFsckResult = runFsck(conf, 0, true, "/",
+ "-blockId", block.getBlock().getBlockName());
+ assertTrue(runFsckResult.contains(block.getBlock().getBlockName()));
+ assertTrue(runFsckResult.contains("No. of Expected Replica: " +
groupSize));
+ assertTrue(runFsckResult.contains("No. of live Replica: " + groupSize));
+ assertTrue(runFsckResult.contains("No. of redundant Replica: " + 0));
+
+ // stop a dn.
+ DatanodeInfo dnToStop = block.getLocations()[0];
+ MiniDFSCluster.DataNodeProperties dnProp =
cluster.stopDataNode(dnToStop.getXferAddr());
+ cluster.setDataNodeDead(dnToStop);
+
+ // wait for reconstruction to happen.
+ DFSTestUtil.waitForReplication(fs, filePath, groupSize, 15 * 1000);
+
+ // bring the dn back: 10 internal blocks now.
+ cluster.restartDataNode(dnProp);
+ cluster.waitActive();
+
+ blks = fs.getClient().getLocatedBlocks(filePath.toString(), 0);
+ block = (LocatedStripedBlock) blks.getLastLocatedBlock();
+ Assert.assertEquals(groupSize + 1, block.getLocations().length);
+
+ //general test, number of redundant internal block replicas.
+ runFsckResult = runFsck(conf, 0, true, "/",
+ "-blockId", block.getBlock().getBlockName());
+ assertTrue(runFsckResult.contains(block.getBlock().getBlockName()));
+ assertTrue(runFsckResult.contains("No. of Expected Replica: " +
groupSize));
+ assertTrue(runFsckResult.contains("No. of live Replica: " + groupSize));
+ assertTrue(runFsckResult.contains("No. of redundant Replica: " + 1));
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
private void waitForUnrecoverableBlockGroup(Configuration configuration)
throws TimeoutException, InterruptedException {
GenericTestUtils.waitFor(new Supplier<Boolean>() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
