This is an automated email from the ASF dual-hosted git repository.
ashishkumar50 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new cce1ca28944 HDDS-16160. Reduce container balancer idle wait time.
(#11000)
cce1ca28944 is described below
commit cce1ca28944886218132f01c0e271c1fd0700078
Author: Ashish Kumar <[email protected]>
AuthorDate: Wed Oct 14 17:08:11 2026 +0530
HDDS-16160. Reduce container balancer idle wait time. (#11000)
---
.../balancer/ContainerBalancerConfiguration.java | 8 +++--
.../docs/content/feature/ContainerBalancer.md | 2 +-
.../scm/container/balancer/ContainerBalancer.java | 41 +++++++++++++++++-----
.../container/balancer/TestContainerBalancer.java | 27 ++++++++++++++
.../TestContainerBalancerDatanodeNodeLimit.java | 13 +++++--
.../scm/cli/ContainerBalancerConfigOptions.java | 2 +-
.../ozone/TestContainerBalancerOperations.java | 2 +-
7 files changed, 79 insertions(+), 16 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java
index af4a2304a36..3c909003965 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java
@@ -112,9 +112,11 @@ public final class ContainerBalancerConfiguration {
private long moveReplicationTimeout = Duration.ofMinutes(50).toMillis();
@Config(key = "hdds.container.balancer.balancing.iteration.interval", type =
ConfigType.TIME,
- defaultValue = "70m", tags = {ConfigTag.BALANCER}, description =
- "The interval period between each iteration of Container Balancer.")
- private long balancingInterval = Duration.ofMinutes(70).toMillis();
+ defaultValue = "3m", tags = {ConfigTag.BALANCER}, description =
+ "The interval to wait between Container Balancer iterations. " +
+ "Container Balancer runs iterations continuously and waits only briefly
" +
+ "so SCM can receive refreshed datanode usage information before the next
iteration starts..")
+ private long balancingInterval = Duration.ofMinutes(3).toMillis();
@Config(key = "hdds.container.balancer.include.datanodes", type =
ConfigType.STRING, defaultValue =
"", tags = {ConfigTag.BALANCER}, description = "A list of Datanode " +
diff --git a/hadoop-hdds/docs/content/feature/ContainerBalancer.md
b/hadoop-hdds/docs/content/feature/ContainerBalancer.md
index 94689d53223..b6270c985b8 100644
--- a/hadoop-hdds/docs/content/feature/ContainerBalancer.md
+++ b/hadoop-hdds/docs/content/feature/ContainerBalancer.md
@@ -105,7 +105,7 @@ The Container Balancer can also be configured through the
`ozone-site.xml` file.
| `hdds.container.balancer.exclude.containers` | A comma-separated
list of container IDs to exclude from balancing.
| "" |
| `hdds.container.balancer.move.timeout` | The amount of time
to allow a single container to move from source to target.
| 65m |
| `hdds.container.balancer.move.replication.timeout` | The amount of time
to allow a single container's replication from source to target as part of a
container move. | 50m |
-| `hdds.container.balancer.balancing.iteration.interval` | The interval period
between each iteration of Container Balancer.
| 70m |
+| `hdds.container.balancer.balancing.iteration.interval` | The interval to
wait between iterations. The balancer runs continuously, this wait lets SCM
receive refreshed Datanode usage information before the next iteration starts.
| 3m |
| `hdds.container.balancer.include.datanodes` | A comma-separated
list of Datanode hostnames or IP addresses. Only the Datanodes specified in
this list are balanced. | "" |
| `hdds.container.balancer.exclude.datanodes` | A comma-separated
list of Datanode hostnames or IP addresses. The Datanodes specified in this
list are excluded from balancing. | "" |
| `hdds.container.balancer.move.networkTopology.enable` | Whether to take
network topology into account when selecting a target for a source.
| false |
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java
index 7515d93a747..d00f92d2065 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java
@@ -17,6 +17,8 @@
package org.apache.hadoop.hdds.scm.container.balancer;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_NODE_REPORT_INTERVAL;
+import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_NODE_REPORT_INTERVAL_DEFAULT;
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE;
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY;
@@ -32,7 +34,6 @@
import java.util.concurrent.locks.ReentrantLock;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.conf.StorageUnit;
-import org.apache.hadoop.hdds.fs.DUFactory;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import
org.apache.hadoop.hdds.protocol.proto.HddsProtos.ContainerBalancerConfigurationProto;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@@ -308,6 +309,10 @@ public void startBalancer(ContainerBalancerConfiguration
configuration)
saveConfiguration(configuration, true, 0);
this.config = configuration;
+ LOG.info("Container Balancer starting with balancing iteration interval
{} seconds " +
+ "(hdds.container.balancer.balancing.iteration.interval).",
+ configuration.getBalancingInterval().getSeconds());
+
// Set startedAt only after validation has passed, so a rejected start
// does not overwrite the recorded start time of the previous run.
startedAt = OffsetDateTime.now();
@@ -505,13 +510,20 @@ private void
validateConfiguration(ContainerBalancerConfiguration conf)
+ ".iteration");
}
- // balancing interval should be greater than DUFactory refresh period
- DUFactory.Conf duConf = ozoneConfiguration.getObject(DUFactory.Conf.class);
- long refreshPeriod = duConf.getRefreshPeriod().toMillis();
- if (conf.getBalancingInterval().toMillis() <= refreshPeriod) {
- LOG.warn("hdds.container.balancer.balancing.iteration.interval {} " +
- "should be greater than hdds.datanode.du.refresh.period {}",
- conf.getBalancingInterval().toMillis(), refreshPeriod);
+ // Balancing interval should be greater than the node report interval so
+ // that SCM receives refreshed datanode usage info between iterations.
+ // Datanode usage is tracked incrementally in memory and reported every
+ // hdds.node.report.interval, so the node report interval (not the hourly
+ // du refresh period) is the relevant freshness bound between iterations.
+ long nodeReportInterval = ozoneConfiguration.getTimeDuration(
+ HDDS_NODE_REPORT_INTERVAL, HDDS_NODE_REPORT_INTERVAL_DEFAULT,
+ TimeUnit.MILLISECONDS);
+ if (isBalancingIntervalBelowNodeReportInterval(
+ conf.getBalancingInterval().toMillis(), nodeReportInterval)) {
+ LOG.warn("hdds.container.balancer.balancing.iteration.interval {}ms " +
+ "should be greater than hdds.node.report.interval {}ms so that "
+
+ "datanode usage info is refreshed between iterations.",
+ conf.getBalancingInterval().toMillis(), nodeReportInterval);
}
// "move.replication.timeout" should be lesser than "move.timeout"
@@ -550,6 +562,19 @@ private void
validateConfiguration(ContainerBalancerConfiguration conf)
validateEligibleDatanodePool(conf);
}
+ /**
+ * A balancing interval that is not greater than the node report interval
means
+ * SCM may not receive refreshed datanode usage info before the next
iteration
+ * starts. Datanode usage is tracked incrementally in memory and reported
every
+ * node report interval, so that interval (not the hourly du refresh period)
is
+ * the relevant freshness bound between iterations.
+ */
+ @VisibleForTesting
+ static boolean isBalancingIntervalBelowNodeReportInterval(
+ long balancingIntervalMs, long nodeReportIntervalMs) {
+ return balancingIntervalMs < nodeReportIntervalMs;
+ }
+
/**
* Rejects include lists that are fully covered by the corresponding exclude
* lists, which would leave no datanodes or containers to balance.
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancer.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancer.java
index 440cf590f02..fe6c71c3f4c 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancer.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancer.java
@@ -460,6 +460,33 @@ public void testRejectInvalidStartupConfiguration() throws
Exception {
assertSame(ContainerBalancerTask.Status.STOPPED,
containerBalancer.getBalancerStatus());
}
+ /**
+ * Container Balancer runs iterations continuously with only a short wait in
+ * between, so the default balancing interval is 3 minutes.
+ */
+ @Test
+ public void testDefaultBalancingIntervalIsThreeMinutes() {
+ ContainerBalancerConfiguration defaultConfig =
+ new
OzoneConfiguration().getObject(ContainerBalancerConfiguration.class);
+ assertEquals(3, defaultConfig.getBalancingInterval().toMinutes());
+ }
+
+ /**
+ * The freshness bound for the balancing interval is the node report interval
+ * and not the hourly du refresh period. The
+ * default 3m interval is above a typical 1m node report interval.
+ */
+ @Test
+ public void testBalancingIntervalNodeReportFreshnessBoundary() {
+ long nodeReportIntervalMs = TimeUnit.MINUTES.toMillis(1);
+ // Below the node report interval -> too short.
+ assertTrue(ContainerBalancer.isBalancingIntervalBelowNodeReportInterval(
+ TimeUnit.SECONDS.toMillis(30), nodeReportIntervalMs));
+ // The default 3m interval is above the node report interval -> acceptable.
+ assertFalse(ContainerBalancer.isBalancingIntervalBelowNodeReportInterval(
+ TimeUnit.MINUTES.toMillis(3), nodeReportIntervalMs));
+ }
+
/**
* Verifies that a start request rejected during validation does not update
startedAt of last run.
* Prevents startedAt from advancing past stoppedAt, which previously caused
a negative duration and CLI status crash.
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerDatanodeNodeLimit.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerDatanodeNodeLimit.java
index a094a84240f..79e192e3d7b 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerDatanodeNodeLimit.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerDatanodeNodeLimit.java
@@ -620,8 +620,17 @@ public void checkIterationResultTimeout(@Nonnull MockedSCM
mockedSCM)
ContainerBalancerTask task = mockedSCM.startBalancerTask(config);
assertEquals(ContainerBalancerTask.IterationResult.ITERATION_COMPLETED,
task.getIterationResult());
- assertEquals(1,
task.getMetrics().getNumContainerMovesCompletedInLatestIteration());
-
assertThat(task.getMetrics().getNumContainerMovesTimeoutInLatestIteration()).isGreaterThanOrEqualTo(1);
+
+ // The random cluster layout decides how many moves get scheduled. The
first mocked move completes
+ // instantly; subsequent moves take 150ms which exceeds moveTimeout
(50ms). If the cluster layout
+ // allows more than one move, exactly the first completes and the rest
time out. If only one move
+ // is possible in this layout, it completes and no timeouts occur.
+ ContainerBalancerMetrics metrics = task.getMetrics();
+
assertThat(metrics.getNumContainerMovesCompletedInLatestIteration()).isGreaterThanOrEqualTo(1);
+ if (metrics.getNumContainerMovesScheduledInLatestIteration() > 1) {
+ assertEquals(1,
metrics.getNumContainerMovesCompletedInLatestIteration());
+
assertThat(metrics.getNumContainerMovesTimeoutInLatestIteration()).isGreaterThanOrEqualTo(1);
+ }
}
@ParameterizedTest(name = "MockedSCM #{index}: {0}")
diff --git
a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerConfigOptions.java
b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerConfigOptions.java
index e82a954f2fd..fde63d28987 100644
---
a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerConfigOptions.java
+++
b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerConfigOptions.java
@@ -69,7 +69,7 @@ public class ContainerBalancerConfigOptions {
@Option(names = {"--balancing-iteration-interval-minutes"},
description = "The interval period in minutes between each iteration of
Container Balancer. " +
- "The value should be positive, with a default of 70 (specify '70'
for 70 minutes).")
+ "The value should be positive, with a default of 3 (specify '3' for
3 minutes).")
private Optional<Integer> balancingInterval;
@Option(names = {"--move-timeout-minutes"},
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java
index c4f76c2cfd6..72e53562c54 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java
@@ -193,7 +193,7 @@ public void testIfCBCLIOverridesConfigs() throws Exception {
//If config value is not added in ozone-site.xml and CLI option is not
passed
//then it takes the default configuration
- assertEquals(70, config.getBalancingInterval().toMinutes());
+ assertEquals(3, config.getBalancingInterval().toMinutes());
//If config value is added in ozone-site.xml and CLI option is not passed
//then it takes the value from ozone-site.xml
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
