This is an automated email from the ASF dual-hosted git repository.
siddhant pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 2250a481d6 HDDS-7799. Add container count to datanode usage info
(#4209)
2250a481d6 is described below
commit 2250a481d60babf6de96860e03f00f2a56fd98ee
Author: djordje-mijatovic <[email protected]>
AuthorDate: Wed Feb 8 12:38:25 2023 +0100
HDDS-7799. Add container count to datanode usage info (#4209)
---
.../interface-client/src/main/proto/hdds.proto | 1 +
.../hadoop/hdds/scm/node/DatanodeUsageInfo.java | 12 ++++++++
.../hadoop/hdds/scm/node/SCMNodeManager.java | 13 +++++++--
.../hdds/scm/server/SCMClientProtocolServer.java | 15 ++--------
.../hdds/scm/cli/datanode/UsageInfoSubcommand.java | 12 +++++++-
.../scm/cli/datanode/TestUsageInfoSubcommand.java | 4 +++
.../hadoop/ozone/TestContainerOperations.java | 34 ++++++++++++++++++++++
7 files changed, 74 insertions(+), 17 deletions(-)
diff --git a/hadoop-hdds/interface-client/src/main/proto/hdds.proto
b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
index 555431b199..2ff39cab58 100644
--- a/hadoop-hdds/interface-client/src/main/proto/hdds.proto
+++ b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
@@ -186,6 +186,7 @@ message DatanodeUsageInfoProto {
optional int64 used = 2;
optional int64 remaining = 3;
optional DatanodeDetailsProto node = 4;
+ optional int64 containerCount = 5;
}
/**
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeUsageInfo.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeUsageInfo.java
index 8de0a412c2..14353cfa7e 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeUsageInfo.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeUsageInfo.java
@@ -31,6 +31,7 @@ public class DatanodeUsageInfo {
private DatanodeDetails datanodeDetails;
private SCMNodeStat scmNodeStat;
+ private int containerCount;
/**
* Constructs a DatanodeUsageInfo with DatanodeDetails and SCMNodeStat.
@@ -43,6 +44,7 @@ public class DatanodeUsageInfo {
SCMNodeStat scmNodeStat) {
this.datanodeDetails = datanodeDetails;
this.scmNodeStat = scmNodeStat;
+ this.containerCount = -1;
}
/**
@@ -135,6 +137,14 @@ public class DatanodeUsageInfo {
return scmNodeStat;
}
+ public int getContainerCount() {
+ return containerCount;
+ }
+
+ public void setContainerCount(int containerCount) {
+ this.containerCount = containerCount;
+ }
+
/**
* Gets Comparator that compares two DatanodeUsageInfo on the basis of
* their utilization values. Utilization is (capacity - remaining) divided
@@ -196,6 +206,8 @@ public class DatanodeUsageInfo {
builder.setUsed(scmNodeStat.getScmUsed().get());
builder.setRemaining(scmNodeStat.getRemaining().get());
}
+
+ builder.setContainerCount(containerCount);
return builder;
}
}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index a697dd2846..7a62088ba7 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -826,8 +826,8 @@ public class SCMNodeManager implements NodeManager {
// create a DatanodeUsageInfo from each DatanodeDetails and add it to the
// list
for (DatanodeDetails node : healthyNodes) {
- SCMNodeStat stat = getNodeStatInternal(node);
- datanodeUsageInfoList.add(new DatanodeUsageInfo(node, stat));
+ DatanodeUsageInfo datanodeUsageInfo = getUsageInfo(node);
+ datanodeUsageInfoList.add(datanodeUsageInfo);
}
// sort the list according to appropriate comparator
@@ -850,7 +850,14 @@ public class SCMNodeManager implements NodeManager {
@Override
public DatanodeUsageInfo getUsageInfo(DatanodeDetails dn) {
SCMNodeStat stat = getNodeStatInternal(dn);
- return new DatanodeUsageInfo(dn, stat);
+ DatanodeUsageInfo usageInfo = new DatanodeUsageInfo(dn, stat);
+ try {
+ int containerCount = getContainers(dn).size();
+ usageInfo.setContainerCount(containerCount);
+ } catch (NodeNotFoundException ex) {
+ LOG.error("Unknown datanode {}.", dn, ex);
+ }
+ return usageInfo;
}
/**
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index 0a7eeb81e5..5b98aad4b1 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -46,7 +46,6 @@ import
org.apache.hadoop.hdds.scm.container.balancer.ContainerBalancerConfigurat
import
org.apache.hadoop.hdds.scm.container.balancer.IllegalContainerBalancerStateException;
import
org.apache.hadoop.hdds.scm.container.balancer.InvalidContainerBalancerConfigurationException;
import
org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
-import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes;
@@ -1048,18 +1047,8 @@ public class SCMClientProtocolServer implements
*/
private HddsProtos.DatanodeUsageInfoProto getUsageInfoFromDatanodeDetails(
DatanodeDetails node, int clientVersion) {
- SCMNodeStat stat = scm.getScmNodeManager().getNodeStat(node).get();
-
- long capacity = stat.getCapacity().get();
- long used = stat.getScmUsed().get();
- long remaining = stat.getRemaining().get();
-
- return HddsProtos.DatanodeUsageInfoProto.newBuilder()
- .setCapacity(capacity)
- .setUsed(used)
- .setRemaining(remaining)
- .setNode(node.toProto(clientVersion))
- .build();
+ DatanodeUsageInfo usageInfo = scm.getScmNodeManager().getUsageInfo(node);
+ return usageInfo.toProto(clientVersion);
}
/**
diff --git
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/UsageInfoSubcommand.java
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/UsageInfoSubcommand.java
index d404a6ce1e..5bd6b42b44 100644
---
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/UsageInfoSubcommand.java
+++
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/UsageInfoSubcommand.java
@@ -151,8 +151,10 @@ public class UsageInfoSubcommand extends ScmSubcommand {
// print total remaining space and its percentage in a readable format
System.out.printf("%-13s: %s (%s) %n", "Remaining", info.getRemaining()
+ " B", StringUtils.byteDesc(info.getRemaining()));
- System.out.printf("%-13s: %s %n%n", "Remaining %",
+ System.out.printf("%-13s: %s %n", "Remaining %",
PERCENT_FORMAT.format(info.getRemainingRatio()));
+ System.out.printf("%-13s: %d %n%n", "Container(s)",
+ info.getContainerCount());
}
/**
@@ -177,6 +179,7 @@ public class UsageInfoSubcommand extends ScmSubcommand {
private long capacity = 0;
private long used = 0;
private long remaining = 0;
+ private long containerCount = 0;
DatanodeUsage(HddsProtos.DatanodeUsageInfoProto proto) {
if (proto.hasNode()) {
@@ -191,6 +194,9 @@ public class UsageInfoSubcommand extends ScmSubcommand {
if (proto.hasRemaining()) {
remaining = proto.getRemaining();
}
+ if (proto.hasContainerCount()) {
+ containerCount = proto.getContainerCount();
+ }
}
public DatanodeDetails getDatanodeDetails() {
@@ -213,6 +219,10 @@ public class UsageInfoSubcommand extends ScmSubcommand {
return remaining;
}
+ public long getContainerCount() {
+ return containerCount;
+ }
+
@JsonSerialize(using = DecimalJsonSerializer.class)
public double getTotalUsedPercent() {
return getTotalUsedRatio() * 100;
diff --git
a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestUsageInfoSubcommand.java
b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestUsageInfoSubcommand.java
index f82acc8c24..0cc8ed9be6 100644
---
a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestUsageInfoSubcommand.java
+++
b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestUsageInfoSubcommand.java
@@ -92,6 +92,9 @@ public class TestUsageInfoSubcommand {
json.get(0).get("ozoneUsedPercent").doubleValue(), 0.001);
Assertions.assertEquals(80.00,
json.get(0).get("remainingPercent").doubleValue(), 0.001);
+
+ Assertions.assertEquals(5,
+ json.get(0).get("containerCount").longValue());
}
private List<HddsProtos.DatanodeUsageInfoProto> getUsageProto() {
@@ -101,6 +104,7 @@ public class TestUsageInfoSubcommand {
.setCapacity(100)
.setRemaining(80)
.setUsed(10)
+ .setContainerCount(5)
.build());
return result;
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerOperations.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerOperations.java
index 708bbc003d..dba8136f96 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerOperations.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerOperations.java
@@ -132,4 +132,38 @@ public class TestContainerOperations {
.anyMatch(port -> REPLICATION.name().equals(port.getName())));
}
}
+
+ @Test
+ public void testDatanodeUsageInfoContainerCount() throws IOException {
+ List<DatanodeDetails> dnList = cluster.getStorageContainerManager()
+ .getScmNodeManager()
+ .getAllNodes();
+
+ for (DatanodeDetails dn : dnList) {
+ List<HddsProtos.DatanodeUsageInfoProto> usageInfoList =
+ storageClient.getDatanodeUsageInfo(
+ dn.getIpAddress(), dn.getUuidString());
+
+ assertEquals(1, usageInfoList.size());
+ assertEquals(0, usageInfoList.get(0).getContainerCount());
+ }
+
+ storageClient.createContainer(HddsProtos
+ .ReplicationType.STAND_ALONE, HddsProtos.ReplicationFactor
+ .ONE, OzoneConsts.OZONE);
+
+ int[] totalContainerCount = new int[2];
+ for (DatanodeDetails dn : dnList) {
+ List<HddsProtos.DatanodeUsageInfoProto> usageInfoList =
+ storageClient.getDatanodeUsageInfo(
+ dn.getIpAddress(), dn.getUuidString());
+
+ assertEquals(1, usageInfoList.size());
+ assertTrue(usageInfoList.get(0).getContainerCount() >= 0 &&
+ usageInfoList.get(0).getContainerCount() <= 1);
+ totalContainerCount[(int)usageInfoList.get(0).getContainerCount()]++;
+ }
+ assertEquals(2, totalContainerCount[0]);
+ assertEquals(1, totalContainerCount[1]);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
