This is an automated email from the ASF dual-hosted git repository.
zhangshuyan pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new cc4c4be1b7bd HDFS-17331:Fix Blocks are always -1 and DataNode version
are always UNKNOWN in federationhealth.html (#6429). Contributed by lei w.
cc4c4be1b7bd is described below
commit cc4c4be1b7bda8f5869241a50197699da0f99f4d
Author: Lei313 <47049042+thinker...@users.noreply.github.com>
AuthorDate: Thu Jan 18 21:10:54 2024 +0800
HDFS-17331:Fix Blocks are always -1 and DataNode version are always UNKNOWN
in federationhealth.html (#6429). Contributed by lei w.
Signed-off-by: Shuyan Zhang <zhangshu...@apache.org>
---
.../org/apache/hadoop/hdfs/protocol/DatanodeInfo.java | 15 +++++++++++++--
.../org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java | 6 +++++-
.../hadoop-hdfs-client/src/main/proto/hdfs.proto | 1 +
.../server/federation/metrics/NamenodeBeanMetrics.java | 2 +-
.../server/federation/router/RouterClientProtocol.java | 9 +++++++--
5 files changed, 27 insertions(+), 6 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
index fbe6bcc4629d..d3e6e7115041 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
@@ -143,7 +143,7 @@ public class DatanodeInfo extends DatanodeID implements
Node {
final int xceiverCount, final String networkLocation,
final AdminStates adminState, final String upgradeDomain,
final long lastBlockReportTime, final long lastBlockReportMonotonic,
- final int blockCount) {
+ final int blockCount, final String softwareVersion) {
super(ipAddr, hostName, datanodeUuid, xferPort, infoPort, infoSecurePort,
ipcPort);
this.capacity = capacity;
@@ -162,6 +162,7 @@ public class DatanodeInfo extends DatanodeID implements
Node {
this.lastBlockReportTime = lastBlockReportTime;
this.lastBlockReportMonotonic = lastBlockReportMonotonic;
this.numBlocks = blockCount;
+ this.softwareVersion = softwareVersion;
}
/** Network location name. */
@@ -699,6 +700,7 @@ public class DatanodeInfo extends DatanodeID implements
Node {
private long lastBlockReportTime = 0L;
private long lastBlockReportMonotonic = 0L;
private int numBlocks = 0;
+ private String softwareVersion;
// Please use setNumBlocks explicitly to set numBlocks as this method
doesn't have
// sufficient info about numBlocks
@@ -718,6 +720,9 @@ public class DatanodeInfo extends DatanodeID implements
Node {
this.upgradeDomain = from.getUpgradeDomain();
this.lastBlockReportTime = from.getLastBlockReportTime();
this.lastBlockReportMonotonic = from.getLastBlockReportMonotonic();
+ if (from.getSoftwareVersion() != null) {
+ this.softwareVersion = from.getSoftwareVersion();
+ }
setNodeID(from);
return this;
}
@@ -844,18 +849,24 @@ public class DatanodeInfo extends DatanodeID implements
Node {
this.lastBlockReportMonotonic = time;
return this;
}
+
public DatanodeInfoBuilder setNumBlocks(int blockCount) {
this.numBlocks = blockCount;
return this;
}
+ public DatanodeInfoBuilder setSoftwareVersion(String dnVersion) {
+ this.softwareVersion = dnVersion;
+ return this;
+ }
+
public DatanodeInfo build() {
return new DatanodeInfo(ipAddr, hostName, datanodeUuid, xferPort,
infoPort, infoSecurePort, ipcPort, capacity, dfsUsed, nonDfsUsed,
remaining, blockPoolUsed, cacheCapacity, cacheUsed, lastUpdate,
lastUpdateMonotonic, xceiverCount, location, adminState,
upgradeDomain, lastBlockReportTime, lastBlockReportMonotonic,
- numBlocks);
+ numBlocks, softwareVersion);
}
}
}
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index 26ee5de2886e..b6d3bc7227d7 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -385,6 +385,9 @@ public class PBHelperClient {
if (info.getUpgradeDomain() != null) {
builder.setUpgradeDomain(info.getUpgradeDomain());
}
+ if (info.getSoftwareVersion() != null) {
+ builder.setSoftwareVersion(info.getSoftwareVersion());
+ }
builder
.setId(convert((DatanodeID) info))
.setCapacity(info.getCapacity())
@@ -786,7 +789,8 @@ public class PBHelperClient {
di.getLastBlockReportTime() : 0)
.setLastBlockReportMonotonic(di.hasLastBlockReportMonotonic() ?
di.getLastBlockReportMonotonic() : 0)
- .setNumBlocks(di.getNumBlocks());
+ .setNumBlocks(di.getNumBlocks())
+ .setSoftwareVersion(di.getSoftwareVersion());
if (di.hasNonDfsUsed()) {
dinfo.setNonDfsUsed(di.getNonDfsUsed());
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
index e1e7f7d780da..3d88702956c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
@@ -133,6 +133,7 @@ message DatanodeInfoProto {
optional uint64 lastBlockReportTime = 15 [default = 0];
optional uint64 lastBlockReportMonotonic = 16 [default = 0];
optional uint32 numBlocks = 17 [default = 0];
+ optional string softwareVersion = 18;
}
/**
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
index d16543e2d867..f90939acab55 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
@@ -481,7 +481,7 @@ public class NamenodeBeanMetrics
innerinfo.put("adminState", node.getAdminState().toString());
innerinfo.put("nonDfsUsedSpace", node.getNonDfsUsed());
innerinfo.put("capacity", node.getCapacity());
- innerinfo.put("numBlocks", -1); // node.numBlocks()
+ innerinfo.put("numBlocks", node.getNumBlocks());
innerinfo.put("version", (node.getSoftwareVersion() == null ?
"UNKNOWN" : node.getSoftwareVersion()));
innerinfo.put("used", node.getDfsUsed());
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java
index a85c0320470d..7350583264b2 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java
@@ -1101,10 +1101,15 @@ public class RouterClientProtocol implements
ClientProtocol {
DatanodeInfo dnInfo = dn.getDatanodeInfo();
String nodeId = dnInfo.getXferAddr();
DatanodeStorageReport oldDn = datanodesMap.get(nodeId);
- if (oldDn == null ||
- dnInfo.getLastUpdate() > oldDn.getDatanodeInfo().getLastUpdate()) {
+ if (oldDn == null) {
+ datanodesMap.put(nodeId, dn);
+ } else if (dnInfo.getLastUpdate() >
oldDn.getDatanodeInfo().getLastUpdate()) {
+ dnInfo.setNumBlocks(dnInfo.getNumBlocks() +
+ oldDn.getDatanodeInfo().getNumBlocks());
datanodesMap.put(nodeId, dn);
} else {
+ oldDn.getDatanodeInfo().setNumBlocks(
+ oldDn.getDatanodeInfo().getNumBlocks() + dnInfo.getNumBlocks());
LOG.debug("{} is in multiple subclusters", nodeId);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
