This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-5713
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-5713 by this push:
new 87ad327074 HDDS-12435. [DiskBalancer] Add success move count and fail
move count in status report (#8009)
87ad327074 is described below
commit 87ad327074d9a7e0b0dadc55e7a423cf710474d4
Author: Gargi Jaiswal <[email protected]>
AuthorDate: Tue Mar 11 13:44:37 2025 +0530
HDDS-12435. [DiskBalancer] Add success move count and fail move count in
status report (#8009)
---
.../container/diskbalancer/DiskBalancerInfo.java | 16 +++++++++
.../diskbalancer/DiskBalancerService.java | 10 +++---
.../interface-client/src/main/proto/hdds.proto | 2 ++
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 2 ++
.../hadoop/hdds/scm/node/DiskBalancerManager.java | 38 ++++++++++------------
.../hadoop/hdds/scm/node/DiskBalancerStatus.java | 21 +++++++++---
.../cli/datanode/DiskBalancerStatusSubcommand.java | 14 +++++---
7 files changed, 69 insertions(+), 34 deletions(-)
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerInfo.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerInfo.java
index 79cf605dc6..87269f2a68 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerInfo.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerInfo.java
@@ -32,6 +32,8 @@ public class DiskBalancerInfo {
private long bandwidthInMB;
private int parallelThread;
private DiskBalancerVersion version;
+ private long successCount;
+ private long failureCount;
public DiskBalancerInfo(boolean shouldRun, double threshold,
long bandwidthInMB, int parallelThread) {
@@ -48,6 +50,18 @@ public DiskBalancerInfo(boolean shouldRun, double threshold,
this.version = version;
}
+ public DiskBalancerInfo(boolean shouldRun, double threshold,
+ long bandwidthInMB, int parallelThread, DiskBalancerVersion version,
+ long successCount, long failureCount) {
+ this.shouldRun = shouldRun;
+ this.threshold = threshold;
+ this.bandwidthInMB = bandwidthInMB;
+ this.parallelThread = parallelThread;
+ this.version = version;
+ this.successCount = successCount;
+ this.failureCount = failureCount;
+ }
+
public DiskBalancerInfo(boolean shouldRun,
DiskBalancerConfiguration diskBalancerConf) {
this.shouldRun = shouldRun;
@@ -78,6 +92,8 @@ public
StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto toDiskBala
StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto.newBuilder();
builder.setIsRunning(shouldRun);
builder.setDiskBalancerConf(confProto);
+ builder.setSuccessMoveCount(successCount);
+ builder.setFailureMoveCount(failureCount);
return builder.build();
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
index b51490f4c9..9897a0acdb 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
@@ -294,11 +294,6 @@ public void setVersion(DiskBalancerVersion version) {
this.version = version;
}
- public DiskBalancerInfo getDiskBalancerInfo() {
- return new DiskBalancerInfo(shouldRun, threshold, bandwidthInMB,
- parallelThread, version);
- }
-
public DiskBalancerReportProto getDiskBalancerReportProto() {
DiskBalancerReportProto.Builder builder =
DiskBalancerReportProto.newBuilder();
@@ -508,6 +503,11 @@ private void postCall() {
}
}
+ public DiskBalancerInfo getDiskBalancerInfo() {
+ return new DiskBalancerInfo(shouldRun, threshold, bandwidthInMB,
+ parallelThread, version, metrics.getSuccessCount(),
metrics.getFailureCount());
+ }
+
private Path getDiskBalancerTmpDir(HddsVolume hddsVolume) {
return Paths.get(hddsVolume.getVolumeRootDir())
.resolve(DISK_BALANCER_TMP_DIR).resolve(DISK_BALANCER_DIR);
diff --git a/hadoop-hdds/interface-client/src/main/proto/hdds.proto
b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
index e105826fbb..27ca714e8a 100644
--- a/hadoop-hdds/interface-client/src/main/proto/hdds.proto
+++ b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
@@ -561,4 +561,6 @@ message DatanodeDiskBalancerInfoProto {
required double currentVolumeDensitySum = 2;
optional DiskBalancerRunningStatus runningStatus = 3;
optional DiskBalancerConfigurationProto diskBalancerConf = 4;
+ optional uint64 successMoveCount = 5;
+ optional uint64 failureMoveCount = 6;
}
diff --git
a/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto
b/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto
index 4602274301..2f0bb37b59 100644
---
a/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto
+++
b/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto
@@ -499,6 +499,8 @@ message DiskBalancerReportProto {
required bool isRunning = 1;
optional uint64 balancedBytes = 2;
optional DiskBalancerConfigurationProto diskBalancerConf = 3;
+ optional uint64 successMoveCount = 4;
+ optional uint64 failureMoveCount = 5;
}
/**
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java
index e6771b5f9b..6ca5e610d8 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java
@@ -32,6 +32,7 @@
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import
org.apache.hadoop.hdds.protocol.proto.HddsProtos.DiskBalancerRunningStatus;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto;
import org.apache.hadoop.hdds.scm.DatanodeAdminError;
@@ -248,7 +249,7 @@ private boolean shouldReturnDatanode(
DatanodeDetails datanodeDetails) {
boolean shouldReturn = true;
// If status specified, do not return if status not match.
- if (status != null && getRunningStatus(datanodeDetails) != status) {
+ if (status != null && getStatus(datanodeDetails).getRunningStatus() !=
status) {
shouldReturn = false;
}
return shouldReturn;
@@ -258,14 +259,16 @@ private HddsProtos.DatanodeDiskBalancerInfoProto
getInfoProto(
DatanodeInfo dn, int clientVersion) {
double volumeDensitySum =
getVolumeDataDensitySumForDatanodeDetails(dn);
- HddsProtos.DiskBalancerRunningStatus runningStatus =
- getRunningStatus(dn);
+ DiskBalancerStatus status = getStatus(dn);
+
HddsProtos.DatanodeDiskBalancerInfoProto.Builder builder =
HddsProtos.DatanodeDiskBalancerInfoProto.newBuilder()
.setNode(dn.toProto(clientVersion))
.setCurrentVolumeDensitySum(volumeDensitySum)
- .setRunningStatus(getRunningStatus(dn));
- if (runningStatus != HddsProtos.DiskBalancerRunningStatus.UNKNOWN) {
+ .setRunningStatus(status.getRunningStatus())
+ .setSuccessMoveCount(status.getSuccessMoveCount())
+ .setFailureMoveCount(status.getFailureMoveCount());
+ if (status.getRunningStatus() != DiskBalancerRunningStatus.UNKNOWN) {
builder.setDiskBalancerConf(statusMap.get(dn)
.getDiskBalancerConfiguration().toProtobufBuilder());
}
@@ -301,23 +304,15 @@ private double getVolumeDataDensitySumForDatanodeDetails(
return volumeDensitySum;
}
- private HddsProtos.DiskBalancerRunningStatus getRunningStatus(
- DatanodeDetails datanodeDetails) {
- if (!statusMap.containsKey(datanodeDetails)) {
- return HddsProtos.DiskBalancerRunningStatus.UNKNOWN;
- } else {
- if (statusMap.get(datanodeDetails).isRunning()) {
- return HddsProtos.DiskBalancerRunningStatus.RUNNING;
- } else {
- return HddsProtos.DiskBalancerRunningStatus.STOPPED;
- }
- }
+ private DiskBalancerStatus getStatus(DatanodeDetails datanodeDetails) {
+ return statusMap.computeIfAbsent(datanodeDetails,
+ dn -> new DiskBalancerStatus(DiskBalancerRunningStatus.UNKNOWN, new
DiskBalancerConfiguration(), 0, 0));
}
@VisibleForTesting
public void addRunningDatanode(DatanodeDetails datanodeDetails) {
- statusMap.put(datanodeDetails, new DiskBalancerStatus(true,
- new DiskBalancerConfiguration()));
+ statusMap.put(datanodeDetails, new
DiskBalancerStatus(DiskBalancerRunningStatus.RUNNING,
+ new DiskBalancerConfiguration(), 0, 0));
}
public void processDiskBalancerReport(DiskBalancerReportProto reportProto,
@@ -328,8 +323,11 @@ public void
processDiskBalancerReport(DiskBalancerReportProto reportProto,
DiskBalancerConfiguration.fromProtobuf(
reportProto.getDiskBalancerConf(), conf) :
new DiskBalancerConfiguration();
- statusMap.put(dn, new DiskBalancerStatus(isRunning,
- diskBalancerConfiguration));
+ long successMoveCount = reportProto.getSuccessMoveCount();
+ long failureMoveCount = reportProto.getFailureMoveCount();
+ statusMap.put(dn, new DiskBalancerStatus(
+ isRunning ? DiskBalancerRunningStatus.RUNNING :
DiskBalancerRunningStatus.STOPPED,
+ diskBalancerConfiguration, successMoveCount, failureMoveCount));
if (reportProto.hasBalancedBytes()) {
balancedBytesMap.put(dn, reportProto.getBalancedBytes());
}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerStatus.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerStatus.java
index bf0f367d9b..dc8f4836ab 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerStatus.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerStatus.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.hdds.scm.node;
+import
org.apache.hadoop.hdds.protocol.proto.HddsProtos.DiskBalancerRunningStatus;
import org.apache.hadoop.hdds.scm.storage.DiskBalancerConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -28,20 +29,32 @@ public class DiskBalancerStatus {
public static final Logger LOG =
LoggerFactory.getLogger(DiskBalancerStatus.class);
- private boolean isRunning;
+ private DiskBalancerRunningStatus isRunning;
private DiskBalancerConfiguration diskBalancerConfiguration;
+ private long successMoveCount;
+ private long failureMoveCount;
-
- public DiskBalancerStatus(boolean isRunning, DiskBalancerConfiguration conf)
{
+ public DiskBalancerStatus(DiskBalancerRunningStatus isRunning,
DiskBalancerConfiguration conf,
+ long successMoveCount, long failureMoveCount) {
this.isRunning = isRunning;
this.diskBalancerConfiguration = conf;
+ this.successMoveCount = successMoveCount;
+ this.failureMoveCount = failureMoveCount;
}
- public boolean isRunning() {
+ public DiskBalancerRunningStatus getRunningStatus() {
return isRunning;
}
public DiskBalancerConfiguration getDiskBalancerConfiguration() {
return diskBalancerConfiguration;
}
+
+ public long getSuccessMoveCount() {
+ return successMoveCount;
+ }
+
+ public long getFailureMoveCount() {
+ return failureMoveCount;
+ }
}
diff --git
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStatusSubcommand.java
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStatusSubcommand.java
index c290308ecf..75d3759ffd 100644
---
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStatusSubcommand.java
+++
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStatusSubcommand.java
@@ -60,27 +60,31 @@ public void execute(ScmClient scmClient) throws IOException
{
private String generateStatus(
List<HddsProtos.DatanodeDiskBalancerInfoProto> protos) {
StringBuilder formatBuilder = new StringBuilder("Status result:%n" +
- "%-50s %s %s %s %s %s%n");
+ "%-40s %-20s %-10s %-10s %-15s %-15s %-15s %-15s%n");
List<String> contentList = new ArrayList<>();
contentList.add("Datanode");
contentList.add("VolumeDensity");
contentList.add("Status");
- contentList.add("Threshold");
+ contentList.add("Threshold(%)");
contentList.add("BandwidthInMB");
- contentList.add("ParallelThread");
+ contentList.add("Threads");
+ contentList.add("SuccessMove");
+ contentList.add("FailureMove");
for (HddsProtos.DatanodeDiskBalancerInfoProto proto: protos) {
- formatBuilder.append("%-50s %s %s %s %s %s%n");
+ formatBuilder.append("%-40s %-20s %-10s %-10s %-15s %-15s %-15s
%-15s%n");
contentList.add(proto.getNode().getHostName());
contentList.add(String.valueOf(proto.getCurrentVolumeDensitySum()));
contentList.add(proto.getRunningStatus().name());
contentList.add(
- String.valueOf(proto.getDiskBalancerConf().getThreshold()));
+ String.format("%.4f", proto.getDiskBalancerConf().getThreshold()));
contentList.add(
String.valueOf(proto.getDiskBalancerConf().getDiskBandwidthInMB()));
contentList.add(
String.valueOf(proto.getDiskBalancerConf().getParallelThread()));
+ contentList.add(String.valueOf(proto.getSuccessMoveCount()));
+ contentList.add(String.valueOf(proto.getFailureMoveCount()));
}
return String.format(formatBuilder.toString(),
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
