This is an automated email from the ASF dual-hosted git repository.
Gargi-jais11 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 547ac88e42c HDDS-15306. Expose Disk Balancer CLI in top-level datanode
help and improve usability (#10312).
547ac88e42c is described below
commit 547ac88e42c19d0efae943e9d6b3f192712e4785
Author: Gargi Jaiswal <[email protected]>
AuthorDate: Tue Jun 2 19:02:13 2026 +0530
HDDS-15306. Expose Disk Balancer CLI in top-level datanode help and improve
usability (#10312).
---
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 2 +-
.../common/src/main/resources/ozone-default.xml | 6 +-
.../diskbalancer/DiskBalancerService.java | 1 +
.../ozone/container/ozoneimpl/OzoneContainer.java | 4 +-
.../TestDiskBalancerProtocolServer.java | 4 ++
hadoop-hdds/docs/content/design/diskbalancer.md | 2 +-
hadoop-hdds/docs/content/feature/DiskBalancer.md | 10 ++--
.../docs/content/feature/DiskBalancer.zh.md | 10 ++--
.../interface-client/src/main/proto/hdds.proto | 1 +
.../hdds/scm/cli/datanode/DatanodeParameters.java | 20 +++++--
.../scm/cli/datanode/DiskBalancerCommands.java | 7 ++-
.../cli/datanode/DiskBalancerReportSubcommand.java | 69 ++++++++++++++--------
.../cli/datanode/TestDiskBalancerSubCommands.java | 34 ++++-------
.../hadoop/ozone/scm/node/TestDiskBalancer.java | 2 -
...skBalancerDuringDecommissionAndMaintenance.java | 2 -
15 files changed, 97 insertions(+), 77 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index 8d9cd1c6caf..b321d5cc260 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -410,7 +410,7 @@ public final class HddsConfigKeys {
public static final String HDDS_DATANODE_DISK_BALANCER_ENABLED_KEY =
"hdds.datanode.disk.balancer.enabled";
- public static final boolean HDDS_DATANODE_DISK_BALANCER_ENABLED_DEFAULT =
false;
+ public static final boolean HDDS_DATANODE_DISK_BALANCER_ENABLED_DEFAULT =
true;
public static final String HDDS_DATANODE_DNS_INTERFACE_KEY =
"hdds.datanode.dns.interface";
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml
b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 4d065a0c327..f9dce0b1e97 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -218,11 +218,9 @@
</property>
<property>
<name>hdds.datanode.disk.balancer.enabled</name>
- <value>false</value>
+ <value>true</value>
<tag>OZONE, DATANODE, DISKBALANCER</tag>
- <description>If this property is set to true, then the Disk Balancer
- feature is enabled on Datanodes, and users can use
- this service. By default, this is disabled.
+ <description>By default Disk Balancer feature is enabled on Datanodes.
</description>
</property>
<property>
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
index 3e80b3cb79c..5de1ee198a3 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
@@ -750,6 +750,7 @@ public static List<VolumeReportProto>
buildVolumeReportProto(List<VolumeFixedUsa
VolumeReportProto.Builder builder = VolumeReportProto.newBuilder()
.setStorageId(volume.getStorageID())
.setTotalCapacity(v.getUsage().getCapacity())
+ .setOzoneAvailable(v.getUsage().getAvailable())
.setUsedSpace(v.getUsage().getUsedSpace())
.setCommittedBytes(volume.getCommittedBytes())
.setEffectiveUsedSpace(v.getEffectiveUsed())
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index 27702064cf9..cd9d9640c86 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -50,6 +50,7 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
+import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name;
@@ -279,7 +280,8 @@ public OzoneContainer(HddsDatanodeService
hddsDatanodeService,
config);
} else {
diskBalancerService = null;
- LOG.info("Disk Balancer is disabled.");
+ LOG.info("Disk Balancer is not enabled. Please enable the " +
+ HddsConfigKeys.HDDS_DATANODE_DISK_BALANCER_ENABLED_KEY + "
configuration key.");
}
Duration recoveringContainerScrubbingSvcInterval =
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerProtocolServer.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerProtocolServer.java
index f07543300d5..03494eb78f9 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerProtocolServer.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerProtocolServer.java
@@ -108,6 +108,7 @@ void setup() throws IOException {
.setUtilization(TEST_UTILIZATION_1)
.setCommittedBytes(TEST_COMMITTED_BYTES_1)
.setTotalCapacity(TEST_TOTAL_CAPACITY)
+ .setOzoneAvailable(TEST_TOTAL_CAPACITY - TEST_USED_SPACE_1)
.setUsedSpace(TEST_USED_SPACE_1)
.setEffectiveUsedSpace(TEST_EFFECTIVE_USED_SPACE_1)
.build(),
@@ -117,6 +118,7 @@ void setup() throws IOException {
.setUtilization(TEST_UTILIZATION_2)
.setCommittedBytes(TEST_COMMITTED_BYTES_2)
.setTotalCapacity(TEST_TOTAL_CAPACITY)
+ .setOzoneAvailable(TEST_TOTAL_CAPACITY - TEST_USED_SPACE_2)
.setUsedSpace(TEST_USED_SPACE_2)
.setEffectiveUsedSpace(TEST_EFFECTIVE_USED_SPACE_2)
.build()));
@@ -157,6 +159,7 @@ void testGetDiskBalancerInfoReport() throws IOException {
assertEquals(TEST_UTILIZATION_1, volReport0.getUtilization());
assertEquals(TEST_COMMITTED_BYTES_1, volReport0.getCommittedBytes());
assertEquals(TEST_TOTAL_CAPACITY, volReport0.getTotalCapacity());
+ assertEquals(TEST_TOTAL_CAPACITY - TEST_USED_SPACE_1,
volReport0.getOzoneAvailable());
assertEquals(TEST_USED_SPACE_1, volReport0.getUsedSpace());
assertEquals(TEST_EFFECTIVE_USED_SPACE_1,
volReport0.getEffectiveUsedSpace());
assertEquals(TEST_STORAGE_ID_2, volReport1.getStorageId());
@@ -164,6 +167,7 @@ void testGetDiskBalancerInfoReport() throws IOException {
assertEquals(TEST_UTILIZATION_2, volReport1.getUtilization());
assertEquals(TEST_COMMITTED_BYTES_2, volReport1.getCommittedBytes());
assertEquals(TEST_TOTAL_CAPACITY, volReport1.getTotalCapacity());
+ assertEquals(TEST_TOTAL_CAPACITY - TEST_USED_SPACE_2,
volReport1.getOzoneAvailable());
assertEquals(TEST_USED_SPACE_2, volReport1.getUsedSpace());
assertEquals(TEST_EFFECTIVE_USED_SPACE_2,
volReport1.getEffectiveUsedSpace());
}
diff --git a/hadoop-hdds/docs/content/design/diskbalancer.md
b/hadoop-hdds/docs/content/design/diskbalancer.md
index 6b1edefc62d..98d206bfb7f 100644
--- a/hadoop-hdds/docs/content/design/diskbalancer.md
+++ b/hadoop-hdds/docs/content/design/diskbalancer.md
@@ -165,7 +165,7 @@ This ensures DiskBalancer respects datanode lifecycle
management and does not in
## Feature Flag
-The DiskBalancer feature is gated behind a feature flag
(`hdds.datanode.disk.balancer.enabled`) to allow controlled rollout. By
default, the feature is disabled. When disabled, the DiskBalancer service is
not initialized on datanodes, and the CLI commands are hidden from the main
help output to prevent accidental usage.
+The DiskBalancer feature is gated behind a feature flag
(`hdds.datanode.disk.balancer.enabled`) to allow controlled rollout. By
default, the feature is enabled. When disabled, the DiskBalancer service is not
initialized on datanodes, and the CLI commands refuse to run until the flag is
set back to true.
## DiskBalancer Metrics
diff --git a/hadoop-hdds/docs/content/feature/DiskBalancer.md
b/hadoop-hdds/docs/content/feature/DiskBalancer.md
index a022c696843..7fcb28dc6df 100644
--- a/hadoop-hdds/docs/content/feature/DiskBalancer.md
+++ b/hadoop-hdds/docs/content/feature/DiskBalancer.md
@@ -43,9 +43,9 @@ A disk is considered a candidate for balancing if its
## Feature Flag
-The Disk Balancer feature is introduced with a feature flag. By default, this
feature is disabled.
+The Disk Balancer feature is introduced with a feature flag. By default, this
feature is enabled.
-The feature can be **enabled** by setting the following property to `true` in
the `ozone-site.xml` configuration file:
+The feature can be **disabled** by setting the following property to `false`
in the `ozone-site.xml` configuration file:
`hdds.datanode.disk.balancer.enabled = false`
### Authentication and Authorization
@@ -119,8 +119,8 @@ restart the datanode service for the changes to take effect.
## Command Line Usage
The DiskBalancer is managed through the `ozone admin datanode diskbalancer`
command.
-**Note:** This command is hidden from the main help message (`ozone admin
datanode --help`). This is because the feature
-is currently considered experimental and is disabled by default. The command
is, however, fully functional for those who wish to enable and use the feature.
+**Note:** DiskBalancer is enabled by default on datanodes. Use
`hdds.datanode.disk.balancer.enabled=false` in
+`ozone-site.xml` to disable the service on datanodes and prevent CLI commands
from running.
### Command Syntax
@@ -241,7 +241,7 @@ The DiskBalancer's behavior can be controlled using the
following configuration
| Property | Default
Value
| Description
|
|----------------------------------------------------------------|----------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `hdds.datanode.disk.balancer.enabled` | `false`
| If
false, the DiskBalancer service on the Datanode is disabled. Configure it to
true for diskBalancer to be enabled.
|
+| `hdds.datanode.disk.balancer.enabled` | `true`
| If
false, the DiskBalancer service on the Datanode is disabled. By default,
DiskBalancer is enabled on datanodes.
|
| `hdds.datanode.disk.balancer.volume.density.threshold.percent` | `10.0`
| A
percentage (0-100). A datanode is considered balanced if for each volume, its
utilization differs from the average datanode utilization by no more than this
threshold.
|
| `hdds.datanode.disk.balancer.max.disk.throughputInMBPerSec` | `10`
|
The maximum bandwidth (in MB/s) that the balancer can use for moving data, to
avoid impacting client I/O.
|
| `hdds.datanode.disk.balancer.parallel.thread` | `5`
|
The number of worker threads to use for moving containers in parallel.
|
diff --git a/hadoop-hdds/docs/content/feature/DiskBalancer.zh.md
b/hadoop-hdds/docs/content/feature/DiskBalancer.zh.md
index fb8c704e203..8ec9c548a43 100644
--- a/hadoop-hdds/docs/content/feature/DiskBalancer.zh.md
+++ b/hadoop-hdds/docs/content/feature/DiskBalancer.zh.md
@@ -39,9 +39,9 @@ summary: 数据节点的磁盘平衡器.
## 功能标志
-磁盘平衡器功能已通过功能标志引入。默认情况下，此功能处于禁用状态。
+磁盘平衡器功能已通过功能标志引入。默认情况下，此功能处于启用状态。
-可以通过在“ozone-site.xml”配置文件中将以下属性设置为“true”来**启用**该功能：
+可以通过在“ozone-site.xml”配置文件中将以下属性设置为“false”来**禁用**该功能：
`hdds.datanode.disk.balancer.enabled = false`
### 身份验证和授权
@@ -115,8 +115,8 @@ DiskBalancer 命令通过 RPC 直接与数据节点通信，因此需要进行
## 命令行用法
DiskBalancer 通过 `ozone admin datanode diskbalancer` 命令进行管理。
-**注意：**此命令在主帮助信息（`ozone admin datanode
--help`）中隐藏。这是因为该功能目前处于实验阶段，默认禁用。隐藏该命令可防止意外使用，
-并为普通用户提供清晰的帮助输出。但是，对于希望启用和使用该功能的用户，该命令仍然完全可用。
+**注意：**DiskBalancer 在数据节点上默认启用。在 `ozone-site.xml` 中使用
`hdds.datanode.disk.balancer.enabled=false`
+可禁用数据节点上的服务并阻止 CLI 命令运行。
### 命令语法
**启动 DiskBalancer：**
@@ -233,7 +233,7 @@ The DiskBalancer's behavior can be controlled using the
following configuration
| Property | Default Value
| Description
|
|-------------------------------------------------------------|----------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `hdds.datanode.disk.balancer.enabled` | `false`
| 如果为 false，则 Datanode 上的 DiskBalancer 服务将被禁用。将其配置为
true 可启用 DiskBalancer。 |
|
|
[...]
+| `hdds.datanode.disk.balancer.enabled` | `true`
| 如果为 false，则 Datanode 上的 DiskBalancer
服务将被禁用。默认情况下，DiskBalancer 在 Datanode 上启用。
| |
|
[...]
| `hdds.datanode.disk.balancer.volume.density.threshold.percent` | `10.0`
|
百分比（0-100）。如果对于每个卷，其利用率与平均数据节点利用率之差不超过此阈值，则认为数据节点处于平衡状态。 |
| `hdds.datanode.disk.balancer.max.disk.throughputInMBPerSec` | `10`
| 平衡器可用于移动数据的最大带宽（以 MB/s 为单位），以避免影响客户端 I/O。
|
| `hdds.datanode.disk.balancer.parallel.thread` | `5`
| 用于并行移动容器的工作线程数。
|
diff --git a/hadoop-hdds/interface-client/src/main/proto/hdds.proto
b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
index 59c46499a18..09199967519 100644
--- a/hadoop-hdds/interface-client/src/main/proto/hdds.proto
+++ b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
@@ -591,6 +591,7 @@ message VolumeReportProto {
optional uint64 committedBytes = 5;
optional uint64 effectiveUsedSpace = 6;
optional double utilization = 7;
+ optional uint64 ozoneAvailable = 8;
}
message DatanodeDiskBalancerInfoProto {
diff --git
a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeParameters.java
b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeParameters.java
index b1fda282642..3406e340ae2 100644
---
a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeParameters.java
+++
b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeParameters.java
@@ -29,10 +29,22 @@ public class DatanodeParameters extends ItemsFromStdin {
@CommandLine.Spec
private CommandLine.Model.CommandSpec spec;
- @CommandLine.Parameters(description = "Datanode addresses: one or more,
separated by spaces." +
- " To read from stdin, specify '-' and supply one item per line." +
- "Port is optional and defaults to 19864 (CLIENT_RPC port). " +
- "Examples: 'DN-1', 'DN-1:19864', '192.168.1.10'. ",
+ @CommandLine.Parameters(
+ description = {
+ "Datanode addresses: one or more on the command line, OR read from
stdin with '-'.",
+ "Stdin usage:",
+ " ozone admin datanode diskbalancer <subcommand> -",
+ " Then type one datanode per line and end input:",
+ " - Linux/macOS: Ctrl-D",
+ " - Windows: Ctrl-Z, then Enter",
+ "Examples:",
+ " # Piped (recommended for scripts)",
+ " echo -e \"DN-1\\nDN-2\" | ozone admin datanode diskbalancer
status -",
+ " # From file having list of dns to balance",
+ " ozone admin datanode diskbalancer report - < datanode-lists.txt",
+ "Port is optional and defaults to 19864 (CLIENT_RPC port).",
+ "Address examples: 'DN-1', 'DN-1:19864', '192.168.1.10'."
+ },
arity = "0..*",
paramLabel = "<datanode address>")
diff --git
a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommands.java
b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommands.java
index c912ad73550..775f86c6848 100644
---
a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommands.java
+++
b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommands.java
@@ -17,6 +17,8 @@
package org.apache.hadoop.hdds.scm.cli.datanode;
+import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_DISK_BALANCER_ENABLED_KEY;
+
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import picocli.CommandLine.Command;
@@ -155,11 +157,10 @@
@Command(
name = "diskbalancer",
- description = "DiskBalancer specific operations. It is disabled by
default." +
- " To enable it, set 'hdds.datanode.disk.balancer.enabled' as true",
+ description = "DiskBalancer specific operations to ensure even disk
utilization." +
+ " It is enabled by default. Set " +
HDDS_DATANODE_DISK_BALANCER_ENABLED_KEY + " to false to disable.",
mixinStandardHelpOptions = true,
versionProvider = HddsVersionProvider.class,
- hidden = true,
subcommands = {
DiskBalancerStartSubcommand.class,
DiskBalancerStopSubcommand.class,
diff --git
a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerReportSubcommand.java
b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerReportSubcommand.java
index af730199ed8..69ad560821d 100644
---
a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerReportSubcommand.java
+++
b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerReportSubcommand.java
@@ -23,6 +23,7 @@
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
@@ -46,6 +47,8 @@ public class DiskBalancerReportSubcommand extends
AbstractDiskBalancerSubCommand
private final Map<String, DatanodeDiskBalancerInfoProto> reports =
new ConcurrentHashMap<>();
+ private static final String PERCENT_FORMAT = "%.2f%%";
+
@Override
protected Object executeCommand(String hostName) throws IOException {
DiskBalancerProtocol diskBalancerProxy = DiskBalancerSubCommandUtil
@@ -101,7 +104,8 @@ private String
generateReport(List<DatanodeDiskBalancerInfoProto> protos) {
StringBuilder header = new StringBuilder();
header.append("Datanode: ").append(dn).append(System.lineSeparator())
- .append("Aggregate VolumeDataDensity:
").append(p.getCurrentVolumeDensitySum())
+ .append("Aggregate VolumeDataDensity: ")
+ .append(formatPercent(p.getCurrentVolumeDensitySum()))
.append(System.lineSeparator());
if (p.hasIdealUsage() && p.hasDiskBalancerConf()
@@ -110,10 +114,11 @@ private String
generateReport(List<DatanodeDiskBalancerInfoProto> protos) {
double threshold = p.getDiskBalancerConf().getThreshold();
double lt = Math.max(0.0, idealUsage - threshold / 100.0);
double ut = Math.min(1.0, idealUsage + threshold / 100.0);
- header.append("IdealUsage: ").append(String.format("%.8f", idealUsage))
- .append(" | Threshold: ").append(threshold).append('%')
- .append(" | ThresholdRange: (").append(String.format("%.8f", lt))
- .append(", ").append(String.format("%.8f", ut)).append(')')
+ header.append("IdealUsage: ").append(formatPercent(idealUsage))
+ .append(" | Threshold: ")
+ .append(String.format(Locale.ROOT, PERCENT_FORMAT, threshold))
+ .append(" | ThresholdRange: (").append(formatPercent(lt))
+ .append(", ").append(formatPercent(ut)).append(')')
.append(System.lineSeparator())
.append(System.lineSeparator())
.append("Volume Details:").append(System.lineSeparator());
@@ -122,27 +127,29 @@ private String
generateReport(List<DatanodeDiskBalancerInfoProto> protos) {
contentList.add(header.toString());
if (p.getVolumeInfoCount() > 0 && p.hasIdealUsage()) {
- formatBuilder.append("%-45s %-40s %15s %15s %30s %20s %15s %15s%n");
+ formatBuilder.append("%-45s %-40s %15s %15s %15s %30s %20s %15s
%15s%n");
contentList.add("StorageID");
contentList.add("StoragePath");
- contentList.add("TotalCapacity");
- contentList.add("UsedSpace");
- contentList.add("Container Pre-AllocatedSpace");
+ contentList.add("OzoneCapacity");
+ contentList.add("OzoneAvailable");
+ contentList.add("OzoneUsed");
+ contentList.add("ContainerPreAllocatedSpace");
contentList.add("EffectiveUsedSpace");
contentList.add("Utilization");
contentList.add("VolumeDensity");
double ideal = p.getIdealUsage();
for (VolumeReportProto v : p.getVolumeInfoList()) {
- formatBuilder.append("%-45s %-40s %15s %15s %30s %20s %15s %15s%n");
+ formatBuilder.append("%-45s %-40s %15s %15s %15s %30s %20s %15s
%15s%n");
contentList.add(v.hasStorageId() ? v.getStorageId() : "-");
contentList.add(v.hasStoragePath() ? v.getStoragePath() : "-");
contentList.add(v.hasTotalCapacity() ?
StringUtils.byteDesc(v.getTotalCapacity()) : "-");
+ contentList.add(v.hasOzoneAvailable() ?
StringUtils.byteDesc(v.getOzoneAvailable()) : "-");
contentList.add(v.hasUsedSpace() ?
StringUtils.byteDesc(v.getUsedSpace()) : "-");
contentList.add(StringUtils.byteDesc(v.getCommittedBytes()));
contentList.add(v.hasEffectiveUsedSpace() ?
StringUtils.byteDesc(v.getEffectiveUsedSpace()) : "-");
- contentList.add(String.format("%.8f", v.getUtilization()));
- contentList.add(String.format("%.8f", Math.abs(v.getUtilization() -
ideal)));
+ contentList.add(formatPercent(v.getUtilization()));
+ contentList.add(formatPercent(Math.abs(v.getUtilization() - ideal)));
}
formatBuilder.append("%n");
}
@@ -155,17 +162,21 @@ private String
generateReport(List<DatanodeDiskBalancerInfoProto> protos) {
formatBuilder.append("%nNote:%n")
.append(" - Aggregate VolumeDataDensity: Sum of per-volume density
(deviation from ideal);")
.append(" higher means more imbalance.%n")
- .append(" - IdealUsage: Target utilization ratio (0-1) when volumes
are evenly balanced.%n")
+ .append(" - IdealUsage: Target utilization (0-100%%) when volumes are
evenly balanced.%n")
.append(" - ThresholdRange: Acceptable deviation (percent); volumes
within")
.append(" IdealUsage +/- Threshold are considered balanced.%n")
.append(" - VolumeDensity: Deviation of a particular volume's
utilization from IdealUsage.%n")
- .append(" - Utilization: Ratio of actual used space to capacity (0-1)
for a particular volume.%n")
- .append(" - TotalCapacity: Total volume capacity.%n")
- .append(" - UsedSpace: Ozone used space.%n")
- .append(" - Container Pre-AllocatedSpace: Space reserved for
containers not yet written to disk.%n")
+ .append(" - Utilization: how much a particular volume is utilized ")
+ .append("effectiveUsedSpace / ozoneCapacity) in %%.%n")
+ .append(" - OzoneCapacity: Ozone data volume capacity.%n")
+ .append(" - OzoneAvailable: Ozone data volume available space.%n")
+ .append(" - OzoneUsed: Ozone data volume used space.%n")
+ .append(" - ContainerPreAllocatedSpace: Space reserved for containers
not yet written to disk.%n")
.append(" - EffectiveUsedSpace: This is the actual used space of
volume which is visible")
.append(" to the diskBalancer : (ozoneCapacity minus ozoneAvailable) +
containerPreAllocatedSpace + ")
- .append("move delta for source volume.%n");
+ .append("move delta.%n")
+ .append(" - move delta: source volume space to be reclaimed after
move completion;" +
+ " this value is reflected only when diskBalancer is running else
it is 0.%n");
return String.format(formatBuilder.toString(), contentList.toArray(new
String[0]));
}
@@ -175,6 +186,10 @@ protected String getActionName() {
return "report";
}
+ private static String formatPercent(double ratio) {
+ return String.format(Locale.US, PERCENT_FORMAT, ratio * 100.0);
+ }
+
/**
* Create a JSON result map for a report.
*
@@ -186,7 +201,7 @@ private Map<String, Object>
toJson(DatanodeDiskBalancerInfoProto report) {
result.put("datanode",
DiskBalancerSubCommandUtil.getDatanodeHostAndIp(report.getNode()));
result.put("action", "report");
result.put("status", "success");
- result.put("volumeDensity", report.getCurrentVolumeDensitySum());
+ result.put("volumeDensity",
formatPercent(report.getCurrentVolumeDensitySum()));
if (report.hasIdealUsage() && report.hasDiskBalancerConf()
&& report.getDiskBalancerConf().hasThreshold()) {
@@ -194,9 +209,10 @@ private Map<String, Object>
toJson(DatanodeDiskBalancerInfoProto report) {
double threshold = report.getDiskBalancerConf().getThreshold();
double lt = Math.max(0.0, idealUsage - threshold / 100.0);
double ut = Math.min(1.0, idealUsage + threshold / 100.0);
- result.put("idealUsage", String.format("%.8f", idealUsage));
- result.put("threshold %", report.getDiskBalancerConf().getThreshold());
- result.put("thresholdRange", String.format("(%.08f, %.08f)", lt, ut));
+ result.put("idealUsage", formatPercent(idealUsage));
+ result.put("threshold %", String.format(Locale.ROOT, PERCENT_FORMAT,
threshold));
+ result.put("thresholdRange", String.format("(%s, %s)",
+ formatPercent(lt), formatPercent(ut)));
}
if (report.getVolumeInfoCount() > 0) {
@@ -206,13 +222,14 @@ private Map<String, Object>
toJson(DatanodeDiskBalancerInfoProto report) {
Map<String, Object> vm = new LinkedHashMap<>();
vm.put("storageId", v.getStorageId());
vm.put("storagePath", v.hasStoragePath() ? v.getStoragePath() : "-");
- vm.put("totalCapacity", v.hasTotalCapacity() ?
StringUtils.byteDesc(v.getTotalCapacity()) : "-");
- vm.put("usedSpace", v.hasUsedSpace() ?
StringUtils.byteDesc(v.getUsedSpace()) : "-");
+ vm.put("ozoneCapacity", v.hasTotalCapacity() ?
StringUtils.byteDesc(v.getTotalCapacity()) : "-");
+ vm.put("ozoneAvailable", v.hasOzoneAvailable() ?
StringUtils.byteDesc(v.getOzoneAvailable()) : "-");
+ vm.put("ozoneUsed", v.hasUsedSpace() ?
StringUtils.byteDesc(v.getUsedSpace()) : "-");
vm.put("containerPreAllocatedSpace",
StringUtils.byteDesc(v.getCommittedBytes()));
vm.put("effectiveUsedSpace", v.hasEffectiveUsedSpace() ?
StringUtils.byteDesc(v.getEffectiveUsedSpace()) : "-");
- vm.put("utilization", v.getUtilization());
- vm.put("volumeDensity", Math.abs(v.getUtilization() - ideal));
+ vm.put("utilization", formatPercent(v.getUtilization()));
+ vm.put("volumeDensity", formatPercent(Math.abs(v.getUtilization() -
ideal)));
vols.add(vm);
}
diff --git
a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestDiskBalancerSubCommands.java
b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestDiskBalancerSubCommands.java
index e5a418fb0ae..bd7a079eda7 100644
---
a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestDiskBalancerSubCommands.java
+++
b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestDiskBalancerSubCommands.java
@@ -22,7 +22,6 @@
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
@@ -44,8 +43,6 @@
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Stream;
-import org.apache.hadoop.hdds.HddsConfigKeys;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.DiskBalancerProtocol;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -96,24 +93,18 @@ public void setup() throws UnsupportedEncodingException {
* Helper class to hold all mocks needed for DiskBalancer tests.
*/
private static class DiskBalancerMocks implements AutoCloseable {
- private final MockedConstruction<OzoneConfiguration> mockedConf;
private final MockedConstruction<ContainerOperationClient> mockedClient;
private final MockedStatic<DiskBalancerSubCommandUtil> mockedUtil;
DiskBalancerMocks(
- MockedConstruction<OzoneConfiguration> mockedConf,
MockedConstruction<ContainerOperationClient> mockedClient,
MockedStatic<DiskBalancerSubCommandUtil> mockedUtil) {
- this.mockedConf = mockedConf;
this.mockedClient = mockedClient;
this.mockedUtil = mockedUtil;
}
@Override
public void close() {
- if (mockedConf != null) {
- mockedConf.close();
- }
if (mockedClient != null) {
mockedClient.close();
}
@@ -128,14 +119,6 @@ public void close() {
* Returns a DiskBalancerMocks object containing all three mocks.
*/
private DiskBalancerMocks setupAllMocks() {
- MockedConstruction<OzoneConfiguration> mockedConf =
- mockConstruction(OzoneConfiguration.class, (mock, context) -> {
- when(mock.getBoolean(
- eq(HddsConfigKeys.HDDS_DATANODE_DISK_BALANCER_ENABLED_KEY),
- eq(HddsConfigKeys.HDDS_DATANODE_DISK_BALANCER_ENABLED_DEFAULT)))
- .thenReturn(true);
- });
-
MockedConstruction<ContainerOperationClient> mockedClient =
mockConstruction(ContainerOperationClient.class);
@@ -184,7 +167,7 @@ private DiskBalancerMocks setupAllMocks() {
return addressPort;
});
- return new DiskBalancerMocks(mockedConf, mockedClient, mockedUtil);
+ return new DiskBalancerMocks(mockedClient, mockedUtil);
}
@AfterEach
@@ -598,11 +581,11 @@ public void testStatusDiskBalancerWithStdin() throws
Exception {
static Stream<Arguments> thresholdRangeReportCases() {
return Stream.of(
Arguments.of(0.08426521, 10.0, false,
- "ThresholdRange: (0.00000000, 0.18426521)", "ThresholdRange: (-"),
+ "ThresholdRange: (0.00%, 18.43%)", "ThresholdRange: (-"),
Arguments.of(0.95, 10.0, false,
- "ThresholdRange: (0.85000000, 1.00000000)", "1.05000000"),
+ "ThresholdRange: (85.00%, 100.00%)", "105.00%"),
Arguments.of(0.95, 10.0, true,
- "\"thresholdRange\" : \"(0.85000000, 1.00000000)\"",
"1.05000000"));
+ "\"thresholdRange\" : \"(85.00%, 100.00%)\"", "105.00%"));
}
@ParameterizedTest(name = "idealUsage={0}, threshold={1}%, json={2}")
@@ -682,8 +665,9 @@ public void testReportDiskBalancerWithJson() throws
Exception {
assertTrue(output.contains("\"volumes\""));
assertTrue(output.contains("\"storageId\""));
assertTrue(output.contains("\"storagePath\""));
- assertTrue(output.contains("\"totalCapacity\""));
- assertTrue(output.contains("\"usedSpace\""));
+ assertTrue(output.contains("\"ozoneCapacity\""));
+ assertTrue(output.contains("\"ozoneAvailable\""));
+ assertTrue(output.contains("\"ozoneUsed\""));
assertTrue(output.contains("\"effectiveUsedSpace\""));
assertTrue(output.contains("\"utilization\""));
assertTrue(output.contains("\"volumeDensity\""));
@@ -835,6 +819,8 @@ private DatanodeDiskBalancerInfoProto
generateRandomReportProto(String hostname)
double util2 = idealUsage - random.nextDouble() * 0.1;
long used1 = (long) (capacity1 * util1);
long used2 = (long) (capacity2 * util2);
+ long available1 = capacity1 - used1;
+ long available2 = capacity2 - used2;
long effective1 = used1 + committed1;
long effective2 = used2 + committed2;
String path1 = "/data/hdds-" + hostname + "-1";
@@ -845,6 +831,7 @@ private DatanodeDiskBalancerInfoProto
generateRandomReportProto(String hostname)
.setUtilization(util1)
.setCommittedBytes(committed1)
.setTotalCapacity(capacity1)
+ .setOzoneAvailable(available1)
.setUsedSpace(used1)
.setEffectiveUsedSpace(effective1)
.build();
@@ -854,6 +841,7 @@ private DatanodeDiskBalancerInfoProto
generateRandomReportProto(String hostname)
.setUtilization(util2)
.setCommittedBytes(committed2)
.setTotalCapacity(capacity2)
+ .setOzoneAvailable(available2)
.setUsedSpace(used2)
.setEffectiveUsedSpace(effective2)
.build();
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancer.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancer.java
index 6df761e45f5..cfdd4c7d37d 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancer.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancer.java
@@ -32,7 +32,6 @@
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
-import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.DatanodeDetails.Port;
@@ -71,7 +70,6 @@ public class TestDiskBalancer {
@BeforeAll
public static void setup() throws Exception {
ozoneConf = new OzoneConfiguration();
-
ozoneConf.setBoolean(HddsConfigKeys.HDDS_DATANODE_DISK_BALANCER_ENABLED_KEY,
true);
ozoneConf.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
SCMContainerPlacementCapacity.class, PlacementPolicy.class);
ozoneConf.setTimeDuration("hdds.datanode.disk.balancer.service.interval",
3, TimeUnit.SECONDS);
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancerDuringDecommissionAndMaintenance.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancerDuringDecommissionAndMaintenance.java
index 04c6273b8ae..0bad66f8756 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancerDuringDecommissionAndMaintenance.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancerDuringDecommissionAndMaintenance.java
@@ -34,7 +34,6 @@
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
-import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.DatanodeDetails.Port;
@@ -77,7 +76,6 @@ public class TestDiskBalancerDuringDecommissionAndMaintenance
{
@BeforeAll
public static void setup() throws Exception {
conf = new OzoneConfiguration();
- conf.setBoolean(HddsConfigKeys.HDDS_DATANODE_DISK_BALANCER_ENABLED_KEY,
true);
conf.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
SCMContainerPlacementCapacity.class, PlacementPolicy.class);
conf.setTimeDuration("hdds.datanode.disk.balancer.service.interval", 2,
TimeUnit.SECONDS);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
