This is an automated email from the ASF dual-hosted git repository.
hexiaoqiao pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new d8dea6f HDFS-16352. return the real datanode numBlocks in
#getDatanodeStorageReport (#3714). Contributed by liubingxing.
d8dea6f is described below
commit d8dea6f52ada03e099fa7dbb3efe62345ca27588
Author: liubingxing <[email protected]>
AuthorDate: Thu Dec 16 23:31:28 2021 -0600
HDFS-16352. return the real datanode numBlocks in #getDatanodeStorageReport
(#3714). Contributed by liubingxing.
Signed-off-by: He Xiaoqiao <[email protected]>
---
.../apache/hadoop/hdfs/protocol/DatanodeInfo.java | 6 ++---
.../server/blockmanagement/DatanodeManager.java | 3 ++-
.../namenode/TestNameNodeRpcServerMethods.java | 30 ++++++++++++++++++++++
3 files changed, 35 insertions(+), 4 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
index bba90a0..fbe6bcc 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
@@ -698,9 +698,10 @@ public class DatanodeInfo extends DatanodeID implements
Node {
private long nonDfsUsed = 0L;
private long lastBlockReportTime = 0L;
private long lastBlockReportMonotonic = 0L;
- private int numBlocks;
-
+ private int numBlocks = 0;
+ // Please use setNumBlocks explicitly to set numBlocks as this method
doesn't have
+ // sufficient info about numBlocks
public DatanodeInfoBuilder setFrom(DatanodeInfo from) {
this.capacity = from.getCapacity();
this.dfsUsed = from.getDfsUsed();
@@ -717,7 +718,6 @@ public class DatanodeInfo extends DatanodeID implements
Node {
this.upgradeDomain = from.getUpgradeDomain();
this.lastBlockReportTime = from.getLastBlockReportTime();
this.lastBlockReportMonotonic = from.getLastBlockReportMonotonic();
- this.numBlocks = from.getNumBlocks();
setNodeID(from);
return this;
}
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index ef51c6c..cfb1d83 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -2182,7 +2182,8 @@ public class DatanodeManager {
for (int i = 0; i < reports.length; i++) {
final DatanodeDescriptor d = datanodes.get(i);
reports[i] = new DatanodeStorageReport(
- new DatanodeInfoBuilder().setFrom(d).build(), d.getStorageReports());
+ new
DatanodeInfoBuilder().setFrom(d).setNumBlocks(d.numBlocks()).build(),
+ d.getStorageReports());
}
return reports;
}
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServerMethods.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServerMethods.java
index a32e218..50740bd 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServerMethods.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServerMethods.java
@@ -20,9 +20,14 @@ package org.apache.hadoop.hdfs.server.namenode;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnresolvedLinkException;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.test.GenericTestUtils;
@@ -31,6 +36,8 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+
public class TestNameNodeRpcServerMethods {
private static NamenodeProtocols nnRpc;
private static Configuration conf;
@@ -83,4 +90,27 @@ public class TestNameNodeRpcServerMethods {
}
+ @Test
+ public void testGetDatanodeStorageReportWithNumBLocksNotZero() throws
Exception {
+ int buffSize = 1024;
+ long blockSize = 1024 * 1024;
+ String file = "/testFile";
+ DistributedFileSystem dfs = cluster.getFileSystem();
+ FSDataOutputStream outputStream = dfs.create(
+ new Path(file), true, buffSize, (short)1, blockSize);
+ byte[] outBuffer = new byte[buffSize];
+ for (int i = 0; i < buffSize; i++) {
+ outBuffer[i] = (byte) (i & 0x00ff);
+ }
+ outputStream.write(outBuffer);
+ outputStream.close();
+
+ int numBlocks = 0;
+ DatanodeStorageReport[] reports
+ = nnRpc.getDatanodeStorageReport(HdfsConstants.DatanodeReportType.ALL);
+ for (DatanodeStorageReport r : reports) {
+ numBlocks += r.getDatanodeInfo().getNumBlocks();
+ }
+ assertEquals(1, numBlocks);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
