This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 9afd1a141c HDDS-9988. Show used storage percent in SCM UI (#5882)
9afd1a141c is described below
commit 9afd1a141ce1a03bbb6bb84b080b8981d9c20e15
Author: jianghuazhu <[email protected]>
AuthorDate: Mon Jan 22 18:02:05 2024 +0800
HDDS-9988. Show used storage percent in SCM UI (#5882)
---
.../hadoop/hdds/scm/node/SCMNodeManager.java | 102 +++++++++++++++++++--
.../main/resources/webapps/scm/scm-overview.html | 6 ++
.../src/main/resources/webapps/scm/scm.js | 2 +
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 46 ++++++++++
4 files changed, 150 insertions(+), 6 deletions(-)
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index e9b7d220e1..cc5fb9aa77 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -69,7 +69,9 @@ import org.slf4j.LoggerFactory;
import javax.management.ObjectName;
import java.io.IOException;
+import java.math.RoundingMode;
import java.net.InetAddress;
+import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -138,9 +140,11 @@ public class SCMNodeManager implements NodeManager {
* consistent view of the node state.
*/
private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
- private final String opeState = "OPSTATE";
- private final String comState = "COMSTATE";
- private final String lastHeartbeat = "LASTHEARTBEAT";
+ private static final String OPESTATE = "OPSTATE";
+ private static final String COMSTATE = "COMSTATE";
+ private static final String LASTHEARTBEAT = "LASTHEARTBEAT";
+ private static final String USEDSPACEPERCENT = "USEDSPACEPERCENT";
+ private static final String TOTALCAPACITY = "CAPACITY";
/**
* Constructs SCM machine Manager.
*/
@@ -1103,9 +1107,9 @@ public class SCMNodeManager implements NodeManager {
heartbeatTimeDiff =
getLastHeartbeatTimeDiff(dni.getLastHeartbeatTime());
}
Map<String, String> map = new HashMap<>();
- map.put(opeState, opstate);
- map.put(comState, healthState);
- map.put(lastHeartbeat, heartbeatTimeDiff);
+ map.put(OPESTATE, opstate);
+ map.put(COMSTATE, healthState);
+ map.put(LASTHEARTBEAT, heartbeatTimeDiff);
if (httpPort != null) {
map.put(httpPort.getName().toString(), httpPort.getValue().toString());
}
@@ -1113,11 +1117,97 @@ public class SCMNodeManager implements NodeManager {
map.put(httpsPort.getName().toString(),
httpsPort.getValue().toString());
}
+ String capacity = calculateStorageCapacity(dni.getStorageReports());
+ map.put(TOTALCAPACITY, capacity);
+ String[] storagePercentage = calculateStoragePercentage(
+ dni.getStorageReports());
+ String scmUsedPerc = storagePercentage[0];
+ String nonScmUsedPerc = storagePercentage[1];
+ map.put(USEDSPACEPERCENT,
+ "Ozone: " + scmUsedPerc + "%, other: " + nonScmUsedPerc + "%");
nodes.put(hostName, map);
}
return nodes;
}
+ /**
+ * Calculate the storage capacity of the DataNode node.
+ * @param storageReports Calculate the storage capacity corresponding
+ * to the storage collection.
+ * @return
+ */
+ public static String calculateStorageCapacity(
+ List<StorageReportProto> storageReports) {
+ long capacityByte = 0;
+ if (storageReports != null && !storageReports.isEmpty()) {
+ for (StorageReportProto storageReport : storageReports) {
+ capacityByte += storageReport.getCapacity();
+ }
+ }
+
+ double ua = capacityByte;
+ StringBuilder unit = new StringBuilder("B");
+ if (ua > 1024) {
+ ua = ua / 1024;
+ unit.replace(0, 1, "KB");
+ }
+ if (ua > 1024) {
+ ua = ua / 1024;
+ unit.replace(0, 2, "MB");
+ }
+ if (ua > 1024) {
+ ua = ua / 1024;
+ unit.replace(0, 2, "GB");
+ }
+ if (ua > 1024) {
+ ua = ua / 1024;
+ unit.replace(0, 2, "TB");
+ }
+
+ DecimalFormat decimalFormat = new DecimalFormat("#0.0");
+ decimalFormat.setRoundingMode(RoundingMode.HALF_UP);
+ String capacity = decimalFormat.format(ua);
+ return capacity + unit.toString();
+ }
+
+ /**
+ * Calculate the storage usage percentage of a DataNode node.
+ * @param storageReports Calculate the storage percentage corresponding
+ * to the storage collection.
+ * @return
+ */
+ public static String[] calculateStoragePercentage(
+ List<StorageReportProto> storageReports) {
+ String[] storagePercentage = new String[2];
+ String usedPercentage = "N/A";
+ String nonUsedPercentage = "N/A";
+ if (storageReports != null && !storageReports.isEmpty()) {
+ long capacity = 0;
+ long scmUsed = 0;
+ long remaining = 0;
+ for (StorageReportProto storageReport : storageReports) {
+ capacity += storageReport.getCapacity();
+ scmUsed += storageReport.getScmUsed();
+ remaining += storageReport.getRemaining();
+ }
+ long scmNonUsed = capacity - scmUsed - remaining;
+
+ DecimalFormat decimalFormat = new DecimalFormat("#0.00");
+ decimalFormat.setRoundingMode(RoundingMode.HALF_UP);
+
+ double usedPerc = ((double) scmUsed / capacity) * 100;
+ usedPerc = usedPerc > 100.0 ? 100.0 : usedPerc;
+ double nonUsedPerc = ((double) scmNonUsed / capacity) * 100;
+ nonUsedPerc = nonUsedPerc > 100.0 ? 100.0 : nonUsedPerc;
+ usedPercentage = decimalFormat.format(usedPerc);
+ nonUsedPercentage = decimalFormat.format(nonUsedPerc);
+ }
+
+ storagePercentage[0] = usedPercentage;
+ storagePercentage[1] = nonUsedPercentage;
+ return storagePercentage;
+ }
+
/**
* Based on the current time and the last heartbeat, calculate the time
difference
* and get a string of the relative value. E.g. "2s ago", "1m 2s ago", etc.
diff --git
a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html
b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html
index 214a2ad786..fdd8de15b6 100644
--- a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html
+++ b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html
@@ -48,6 +48,10 @@
<tr>
<th ng-click = "columnSort('hostname')"
class="nodeStatusInfo"><span ng-class = "{'sorting' : (columnName !=
'hostname'), 'sortasc' : (columnName == 'hostname' && !reverse),
'sortdesc':(columnName == 'hostname'
&& !reverse)}">HostName</span></th>
+ <th ng-click = "columnSort('usedspacepercent')"
class="nodeStatusInfo" ><span ng-class="{'sorting' : (columnName !=
'usedspacepercent'), 'sortasc' : (columnName == 'usedspacepercent' && !reverse),
+ 'sortdesc':(columnName ==
'usedspacepercent' && !reverse)}">Used Space Percent</span></th>
+ <th ng-click = "columnSort('capacity')" class="nodeStatusInfo"
><span ng-class="{'sorting' : (columnName != 'capacity'), 'sortasc' :
(columnName == 'capacity' && !reverse),
+ 'sortdesc':(columnName == 'capacity'
&& !reverse)}">Capacity</span></th>
<th ng-click = "columnSort('opstate')" class="nodeStatusInfo"
><span ng-class="{'sorting' : (columnName != 'opstate'), 'sortasc' :
(columnName == 'opstate' && !reverse),
'sortdesc':(columnName == 'opstate' &&
!reverse)}">Operational State</span></th>
<th ng-click = "columnSort('comstate')" class="nodeStatusInfo">
<span ng-class="{'sorting' : (columnName != 'comstate'), 'sortasc' :
(columnName == 'comstate' && !reverse),
@@ -66,6 +70,8 @@
{{typestat.hostname}}
</span>
</td>
+ <td>{{typestat.usedspacepercent}}</td>
+ <td>{{typestat.capacity}}</td>
<td>{{typestat.opstate}}</td>
<td>{{typestat.comstate}}</td>
<td>{{typestat.lastheartbeat}}</td>
diff --git a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js
b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js
index 3802a8f744..c6c79b33d8 100644
--- a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js
+++ b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js
@@ -68,6 +68,8 @@
return {
hostname: key,
opstate: value && value.find((element) =>
element.key === "OPSTATE").value,
+ usedspacepercent: value &&
value.find((element) => element.key === "USEDSPACEPERCENT").value,
+ capacity: value && value.find((element) =>
element.key === "CAPACITY").value,
comstate: value && value.find((element) =>
element.key === "COMSTATE").value,
lastheartbeat: value &&
value.find((element) => element.key === "LASTHEARTBEAT").value,
port: portSpec.port,
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
index 85a70b6467..930774a54b 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
@@ -86,6 +86,7 @@ import org.junit.jupiter.api.Test;
import java.util.Map;
import java.util.function.Predicate;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import static java.util.Collections.emptyList;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
@@ -123,6 +124,8 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.eq;
import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.ArgumentCaptor;
import org.slf4j.Logger;
@@ -1572,6 +1575,49 @@ public class TestSCMNodeManager {
}
}
+ private List<StorageReportProto> generateStorageReportProto(
+ int volumeCount, UUID dnId, long capacity, long used, long remaining) {
+ List<StorageReportProto> reports = new ArrayList<>(volumeCount);
+ boolean failed = true;
+ for (int x = 0; x < volumeCount; x++) {
+ String storagePath = testDir.getAbsolutePath() + "/" + dnId;
+ reports.add(HddsTestUtils
+ .createStorageReport(dnId, storagePath, capacity,
+ used, remaining, null, failed));
+ failed = !failed;
+ }
+ return reports;
+ }
+
+ private static Stream<Arguments> calculateStoragePercentageScenarios() {
+ return Stream.of(
+ Arguments.of(600, 65, 500, 1, "600.0B", "10.83", "5.83"),
+ Arguments.of(10000, 1000, 8800, 12, "117.2KB", "10.00", "2.00"),
+ Arguments.of(100000000, 1000, 899999, 12, "1.1GB", "0.00", "99.10"),
+ Arguments.of(10000, 1000, 0, 0, "0.0B", "N/A", "N/A"),
+ Arguments.of(0, 0, 0, 0, "0.0B", "N/A", "N/A"),
+ Arguments.of(1010, 547, 400, 5, "4.9KB", "54.16", "6.24")
+ );
+ }
+
+ @ParameterizedTest
+ @MethodSource("calculateStoragePercentageScenarios")
+ public void testCalculateStoragePercentage(long perCapacity,
+ long used, long remaining, int volumeCount, String totalCapacity,
+ String scmUsedPerc, String nonScmUsedPerc) {
+ DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails();
+ UUID dnId = dn.getUuid();
+ List<StorageReportProto> reports = volumeCount > 0 ?
+ generateStorageReportProto(volumeCount, dnId, perCapacity,
+ used, remaining) : null;
+ String capacityResult = SCMNodeManager.calculateStorageCapacity(reports);
+ assertEquals(totalCapacity, capacityResult);
+ String[] storagePercentage = SCMNodeManager.calculateStoragePercentage(
+ reports);
+ assertEquals(scmUsedPerc, storagePercentage[0]);
+ assertEquals(nonScmUsedPerc, storagePercentage[1]);
+ }
+
/**
* Test multiple nodes sending initial heartbeat with their node report
* with multiple volumes.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
