This is an automated email from the ASF dual-hosted git repository.
ljain pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new ac99b47 HDDS-5927. Improve defaults in ContainerBalancerConfiguration
(#2892)
ac99b47 is described below
commit ac99b47bba4b381f60fd0d414a02e83f0cab1d0b
Author: Siddhant Sangwan <[email protected]>
AuthorDate: Wed Jan 12 12:11:33 2022 +0530
HDDS-5927. Improve defaults in ContainerBalancerConfiguration (#2892)
---
.../apache/hadoop/hdds/scm/client/ScmClient.java | 4 +-
.../protocol/StorageContainerLocationProtocol.java | 4 +-
...inerLocationProtocolClientSideTranslatorPB.java | 34 +++---
.../src/main/proto/ScmAdminProtocol.proto | 7 +-
.../scm/container/balancer/ContainerBalancer.java | 39 ++++---
.../balancer/ContainerBalancerConfiguration.java | 127 ++++++++++++---------
...inerLocationProtocolServerSideTranslatorPB.java | 29 +++--
.../hdds/scm/server/SCMClientProtocolServer.java | 38 +++---
.../container/balancer/TestContainerBalancer.java | 76 ++++++------
.../hdds/scm/cli/ContainerBalancerCommands.java | 19 ++-
.../scm/cli/ContainerBalancerStartSubcommand.java | 50 ++++----
.../hdds/scm/cli/ContainerOperationClient.java | 6 +-
.../datanode/TestContainerBalancerSubCommand.java | 8 +-
.../ozone/TestContainerBalancerOperations.java | 19 +--
14 files changed, 249 insertions(+), 211 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
index 351870a..342a3e7 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
@@ -312,8 +312,8 @@ public interface ScmClient extends Closeable {
* Start ContainerBalancer.
*/
boolean startContainerBalancer(Optional<Double> threshold,
- Optional<Integer> idleiterations,
- Optional<Double> maxDatanodesRatioToInvolvePerIteration,
+ Optional<Integer> iterations,
+ Optional<Integer> maxDatanodesPercentageToInvolvePerIteration,
Optional<Long> maxSizeToMovePerIterationInGB,
Optional<Long> maxSizeEnteringTargetInGB,
Optional<Long> maxSizeLeavingSourceInGB) throws IOException;
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
index a6b9805..264a599 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
@@ -312,8 +312,8 @@ public interface StorageContainerLocationProtocol extends
Closeable {
* Start ContainerBalancer.
*/
boolean startContainerBalancer(Optional<Double> threshold,
- Optional<Integer> idleiterations,
- Optional<Double> maxDatanodesRatioToInvolvePerIteration,
+ Optional<Integer> iterations,
+ Optional<Integer> maxDatanodesPercentageToInvolvePerIteration,
Optional<Long> maxSizeToMovePerIterationInGB,
Optional<Long> maxSizeEnteringTargetInGB,
Optional<Long> maxSizeLeavingSourceInGB) throws IOException;
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
index 6e5bcf1..b745822 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
@@ -739,8 +739,8 @@ public final class
StorageContainerLocationProtocolClientSideTranslatorPB
@Override
public boolean startContainerBalancer(
- Optional<Double> threshold, Optional<Integer> idleiterations,
- Optional<Double> maxDatanodesRatioToInvolvePerIteration,
+ Optional<Double> threshold, Optional<Integer> iterations,
+ Optional<Integer> maxDatanodesPercentageToInvolvePerIteration,
Optional<Long> maxSizeToMovePerIterationInGB,
Optional<Long> maxSizeEnteringTargetInGB,
Optional<Long> maxSizeLeavingSourceInGB) throws IOException{
@@ -751,8 +751,8 @@ public final class
StorageContainerLocationProtocolClientSideTranslatorPB
//make balancer configuration optional
if (threshold.isPresent()) {
double tsd = threshold.get();
- Preconditions.checkState(tsd >= 0.0D && tsd < 1.0D,
- "threshold should to be specified in range [0.0, 1.0).");
+ Preconditions.checkState(tsd >= 0.0D && tsd < 100D,
+ "threshold should be specified in range [0.0, 100.0).");
builder.setThreshold(tsd);
}
if (maxSizeToMovePerIterationInGB.isPresent()) {
@@ -761,22 +761,22 @@ public final class
StorageContainerLocationProtocolClientSideTranslatorPB
"maxSizeToMovePerIterationInGB must be positive.");
builder.setMaxSizeToMovePerIterationInGB(mstm);
}
- if (maxDatanodesRatioToInvolvePerIteration.isPresent()) {
- double mdti = maxDatanodesRatioToInvolvePerIteration.get();
+ if (maxDatanodesPercentageToInvolvePerIteration.isPresent()) {
+ int mdti = maxDatanodesPercentageToInvolvePerIteration.get();
Preconditions.checkState(mdti >= 0,
- "maxDatanodesRatioToInvolvePerIteration must be " +
+ "maxDatanodesPercentageToInvolvePerIteration must be " +
"greater than equal to zero.");
- Preconditions.checkState(mdti <= 1,
- "maxDatanodesRatioToInvolvePerIteration must be " +
- "lesser than equal to one.");
- builder.setMaxDatanodesRatioToInvolvePerIteration(mdti);
+ Preconditions.checkState(mdti <= 100,
+ "maxDatanodesPercentageToInvolvePerIteration must be " +
+ "lesser than equal to hundred.");
+ builder.setMaxDatanodesPercentageToInvolvePerIteration(mdti);
}
- if (idleiterations.isPresent()) {
- int idi = idleiterations.get();
- Preconditions.checkState(idi > 0 || idi == -1,
- "idleiterations must be positive or" +
- " -1(infinitly run container balancer).");
- builder.setIdleiterations(idi);
+ if (iterations.isPresent()) {
+ int i = iterations.get();
+ Preconditions.checkState(i > 0 || i == -1,
+ "number of iterations must be positive or" +
+ " -1 (for running container balancer infinitely).");
+ builder.setIterations(i);
}
if (maxSizeEnteringTargetInGB.isPresent()) {
diff --git a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
index 3211d51..f8c6aa7 100644
--- a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
+++ b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
@@ -489,11 +489,14 @@ message GetContainerTokenResponseProto {
message StartContainerBalancerRequestProto {
optional string traceID = 1;
optional double threshold = 2;
- optional int32 idleiterations = 3;
- optional double maxDatanodesRatioToInvolvePerIteration = 4;
+ optional int32 idleiterations = 3 [deprecated = true];
+ optional double maxDatanodesRatioToInvolvePerIteration = 4 [deprecated =
+ true];
optional int64 maxSizeToMovePerIterationInGB = 5;
optional int64 maxSizeEnteringTargetInGB = 6;
optional int64 maxSizeLeavingSourceInGB = 7;
+ optional int32 maxDatanodesPercentageToInvolvePerIteration = 8;
+ optional int32 iterations = 9;
}
message StartContainerBalancerResponseProto {
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java
index cf3781f..ea32cfa 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java
@@ -74,7 +74,7 @@ public class ContainerBalancer {
private long maxSizeToMovePerIteration;
private int countDatanodesInvolvedPerIteration;
private long sizeMovedPerIteration;
- private int idleIteration;
+ private int iterations;
private List<DatanodeUsageInfo> unBalancedNodes;
private List<DatanodeUsageInfo> overUtilizedNodes;
private List<DatanodeUsageInfo> underUtilizedNodes;
@@ -159,6 +159,7 @@ public class ContainerBalancer {
if (!validateConfiguration(config)) {
return false;
}
+ ozoneConfiguration.setFromObject(balancerConfiguration);
balancerRunning = true;
LOG.info("Starting Container Balancer...{}", this);
@@ -178,23 +179,12 @@ public class ContainerBalancer {
* Balances the cluster.
*/
private void balance() {
- this.idleIteration = config.getIdleIteration();
- if(this.idleIteration == -1) {
+ this.iterations = config.getIterations();
+ if(this.iterations == -1) {
//run balancer infinitely
- this.idleIteration = Integer.MAX_VALUE;
+ this.iterations = Integer.MAX_VALUE;
}
- this.threshold = config.getThreshold();
- this.maxDatanodesRatioToInvolvePerIteration =
- config.getMaxDatanodesRatioToInvolvePerIteration();
- this.maxSizeToMovePerIteration = config.getMaxSizeToMovePerIteration();
- if (config.getNetworkTopologyEnable()) {
- findTargetStrategy = new FindTargetGreedyByNetworkTopology(
- containerManager, placementPolicy, nodeManager, networkTopology);
- } else {
- findTargetStrategy = new FindTargetGreedyByUsageInfo(containerManager,
- placementPolicy, nodeManager);
- }
- for (int i = 0; i < idleIteration && balancerRunning; i++) {
+ for (int i = 0; i < iterations && balancerRunning; i++) {
// stop balancing if iteration is not initialized
if (!initializeIteration()) {
stop();
@@ -217,7 +207,7 @@ public class ContainerBalancer {
// wait for configured time before starting next iteration, unless
// this was the final iteration
- if (i != idleIteration - 1) {
+ if (i != iterations - 1) {
synchronized (this) {
try {
wait(config.getBalancingInterval().toMillis());
@@ -259,6 +249,17 @@ public class ContainerBalancer {
}
return false;
}
+ this.threshold = config.getThresholdAsRatio();
+ this.maxDatanodesRatioToInvolvePerIteration =
+ config.getMaxDatanodesRatioToInvolvePerIteration();
+ this.maxSizeToMovePerIteration = config.getMaxSizeToMovePerIteration();
+ if (config.getNetworkTopologyEnable()) {
+ findTargetStrategy = new FindTargetGreedyByNetworkTopology(
+ containerManager, placementPolicy, nodeManager, networkTopology);
+ } else {
+ findTargetStrategy = new FindTargetGreedyByUsageInfo(containerManager,
+ placementPolicy, nodeManager);
+ }
this.excludeNodes = config.getExcludeNodes();
this.includeNodes = config.getIncludeNodes();
// include/exclude nodes from balancing according to configs
@@ -522,7 +523,7 @@ public class ContainerBalancer {
}
/**
- * Checks if limits maxDatanodesRatioToInvolvePerIteration and
+ * Checks if limits maxDatanodesPercentageToInvolvePerIteration and
* maxSizeToMovePerIteration have not been hit.
*
* @return {@link IterationResult#MAX_DATANODES_TO_INVOLVE_REACHED} if
reached
@@ -872,7 +873,7 @@ public class ContainerBalancer {
return countDatanodesInvolvedPerIteration;
}
- long getSizeMovedPerIteration() {
+ public long getSizeMovedPerIteration() {
return sizeMovedPerIteration;
}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java
index a51e0bd..8582347 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java
@@ -43,27 +43,25 @@ public final class ContainerBalancerConfiguration {
LoggerFactory.getLogger(ContainerBalancerConfiguration.class);
@Config(key = "utilization.threshold", type = ConfigType.AUTO, defaultValue =
- "0.1", tags = {ConfigTag.BALANCER},
- description = "Threshold is a fraction in the range of 0 to 1. A " +
+ "10", tags = {ConfigTag.BALANCER},
+ description = "Threshold is a percentage in the range of 0 to 100. A " +
"cluster is considered balanced if for each datanode, the " +
"utilization of the datanode (used space to capacity ratio) differs"
+
" from the utilization of the cluster (used space to capacity ratio"
+
- " of the entire cluster) no more than the threshold value.")
- private String threshold = "0.1";
+ " of the entire cluster) no more than the threshold.")
+ private String threshold = "10";
- @Config(key = "datanodes.involved.max.ratio.per.iteration", type =
- ConfigType.AUTO,
- defaultValue = "0.2", tags = {ConfigTag.BALANCER}, description = "The " +
- "ratio of maximum number of datanodes that should be involved in " +
- "balancing in one iteration to the total number of healthy, in service "
+
- "nodes known to container balancer.")
- private String maxDatanodesRatioToInvolvePerIteration = "0.2";
+ @Config(key = "datanodes.involved.max.percentage.per.iteration", type =
+ ConfigType.INT, defaultValue = "20", tags = {ConfigTag.BALANCER},
+ description = "Maximum percentage of healthy, in service datanodes " +
+ "that can be involved in balancing in one iteration.")
+ private int maxDatanodesPercentageToInvolvePerIteration = 20;
@Config(key = "size.moved.max.per.iteration", type = ConfigType.SIZE,
- defaultValue = "30GB", tags = {ConfigTag.BALANCER},
+ defaultValue = "500GB", tags = {ConfigTag.BALANCER},
description = "The maximum size of data in bytes that will be moved " +
"by Container Balancer in one iteration.")
- private long maxSizeToMovePerIteration = 30 * OzoneConsts.GB;
+ private long maxSizeToMovePerIteration = 500 * OzoneConsts.GB;
@Config(key = "size.entering.target.max", type = ConfigType.SIZE,
defaultValue = "26GB", tags = {ConfigTag.BALANCER}, description = "The "
+
@@ -81,10 +79,11 @@ public final class ContainerBalancerConfiguration {
" (or default) ozone.scm.container.size.")
private long maxSizeLeavingSource;
- @Config(key = "idle.iterations", type = ConfigType.INT,
+ @Config(key = "iterations", type = ConfigType.INT,
defaultValue = "10", tags = {ConfigTag.BALANCER},
- description = "The idle iteration count of Container Balancer.")
- private int idleIterations = 10;
+ description = "The number of iterations that Container Balancer will " +
+ "run for.")
+ private int iterations = 10;
@Config(key = "exclude.containers", type = ConfigType.STRING, defaultValue =
"", tags = {ConfigTag.BALANCER}, description = "List of container IDs " +
@@ -93,7 +92,7 @@ public final class ContainerBalancerConfiguration {
@Config(key = "move.timeout", type = ConfigType.TIME, defaultValue = "30m",
tags = {ConfigTag.BALANCER}, description =
- "The amount of time in minutes to allow a single container to move " +
+ "The amount of time to allow a single container to move " +
"from source to target.")
private long moveTimeout = Duration.ofMinutes(30).toMillis();
@@ -101,7 +100,7 @@ public final class ContainerBalancerConfiguration {
defaultValue = "70m", tags = {
ConfigTag.BALANCER}, description = "The interval period between each " +
"iteration of Container Balancer.")
- private long balancingInterval;
+ private long balancingInterval = Duration.ofMinutes(70).toMillis();
@Config(key = "include.datanodes", type = ConfigType.STRING, defaultValue =
"", tags = {ConfigTag.BALANCER}, description = "A list of Datanode " +
@@ -127,46 +126,61 @@ public final class ContainerBalancerConfiguration {
/**
* Gets the threshold value for Container Balancer.
*
- * @return a fraction in the range 0 to 1
+ * @return percentage value in the range 0 to 100
*/
public double getThreshold() {
return Double.parseDouble(threshold);
}
+ public double getThresholdAsRatio() {
+ return Double.parseDouble(threshold) / 100;
+ }
+
/**
* Sets the threshold value for Container Balancer.
*
- * @param threshold a fraction in the range 0 to 1
+ * @param threshold a percentage value in the range 0 to 100
*/
public void setThreshold(double threshold) {
- if (threshold < 0 || threshold > 1) {
+ if (threshold < 0d || threshold >= 100d) {
throw new IllegalArgumentException(
- "Threshold must be a fraction in the range 0 to 1.");
+ "Threshold must be a percentage(double) in the range 0 to 100.");
}
this.threshold = String.valueOf(threshold);
}
/**
- * Gets the idle iteration value for Container Balancer.
+ * Gets the iteration count for Container Balancer. A value of -1 means
+ * infinite number of iterations.
*
- * @return a idle iteration count larger than 0
+ * @return a value greater than 0, or -1
*/
- public int getIdleIteration() {
- return idleIterations;
+ public int getIterations() {
+ return iterations;
}
/**
- * Sets the idle iteration value for Container Balancer.
+ * Sets the number of iterations for Container Balancer.
*
- * @param count a idle iteration count larger than 0
+ * @param count a value greater than 0, or -1 for running balancer infinitely
*/
- public void setIdleIteration(int count) {
+ public void setIterations(int count) {
if (count < -1 || 0 == count) {
throw new IllegalArgumentException(
- "Idle iteration count must be larger than 0 or " +
- "-1(for infinitely running).");
+ "Iteration count must be greater than 0, or " +
+ "-1(for running balancer infinitely).");
}
- this.idleIterations = count;
+ this.iterations = count;
+ }
+
+ /**
+ * Gets the maximum percentage of healthy, in-service datanodes that will be
+ * involved in balancing in one iteration.
+ *
+ * @return percentage as an integer from 0 up to and including 100
+ */
+ public int getMaxDatanodesPercentageToInvolvePerIteration() {
+ return maxDatanodesPercentageToInvolvePerIteration;
}
/**
@@ -188,37 +202,36 @@ public final class ContainerBalancerConfiguration {
}
/**
- * Gets the ratio of maximum number of datanodes that will be involved in
- * balancing by Container Balancer in one iteration to the total number of
- * healthy, in-service nodes known to balancer.
+ * Gets the ratio of maximum datanodes involved in balancing to the total
+ * number of healthy, in-service datanodes known to SCM.
*
- * @return maximum datanodes to involve divided by total healthy,
- * in-service nodes
+ * @return ratio as a double from 0 up to and including 1
*/
public double getMaxDatanodesRatioToInvolvePerIteration() {
- return Double.parseDouble(maxDatanodesRatioToInvolvePerIteration);
+ return maxDatanodesPercentageToInvolvePerIteration / 100d;
}
/**
- * Sets the ratio of maximum number of datanodes that will be involved in
- * balancing by Container Balancer in one iteration to the total number of
- * healthy, in-service nodes known to balancer.
+ * Sets the maximum percentage of healthy, in-service datanodes that will be
+ * involved in balancing in one iteration.
*
- * @param maxDatanodesRatioToInvolvePerIteration number of datanodes to
- * involve divided by total
- * number of healthy, in
- * service nodes
+ * @param maxDatanodesPercentageToInvolvePerIteration number of datanodes
+ * to involve divided by
+ * total number of
+ * healthy, in-service
+ * datanodes multiplied
+ * by 100
*/
- public void setMaxDatanodesRatioToInvolvePerIteration(
- double maxDatanodesRatioToInvolvePerIteration) {
- if (maxDatanodesRatioToInvolvePerIteration < 0 ||
- maxDatanodesRatioToInvolvePerIteration > 1) {
- throw new IllegalArgumentException("Max datanodes to involve ratio must"
+
- " be a double greater than equal to zero and lesser than equal to " +
- "one.");
+ public void setMaxDatanodesPercentageToInvolvePerIteration(
+ int maxDatanodesPercentageToInvolvePerIteration) {
+ if (maxDatanodesPercentageToInvolvePerIteration < 0 ||
+ maxDatanodesPercentageToInvolvePerIteration > 100) {
+ throw new IllegalArgumentException(String.format("Argument %d is " +
+ "illegal. Percentage must be from 0 up to and including 100.",
+ maxDatanodesPercentageToInvolvePerIteration));
}
- this.maxDatanodesRatioToInvolvePerIteration =
- String.valueOf(maxDatanodesRatioToInvolvePerIteration);
+ this.maxDatanodesPercentageToInvolvePerIteration =
+ maxDatanodesPercentageToInvolvePerIteration;
}
/**
@@ -347,12 +360,12 @@ public final class ContainerBalancerConfiguration {
return String.format("Container Balancer Configuration values:%n" +
"%-50s %s%n" +
"%-50s %s%n" +
- "%-50s %s%n" +
+ "%-50s %d%n" +
"%-50s %dGB%n"+
"%-50s %dGB%n"+
"%-50s %dGB%n", "Key", "Value", "Threshold",
- threshold, "Max Datanodes to Involve per Iteration(ratio)",
- maxDatanodesRatioToInvolvePerIteration,
+ threshold, "Max Datanodes to Involve per Iteration(percent)",
+ maxDatanodesPercentageToInvolvePerIteration,
"Max Size to Move per Iteration",
maxSizeToMovePerIteration / OzoneConsts.GB,
"Max Size Entering Target per Iteration",
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
index 38be88a..b839e99 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
@@ -718,8 +718,9 @@ public final class
StorageContainerLocationProtocolServerSideTranslatorPB
StartContainerBalancerRequestProto request)
throws IOException {
Optional<Double> threshold = Optional.empty();
- Optional<Integer> idleiterations = Optional.empty();
- Optional<Double> maxDatanodesRatioToInvolvePerIteration = Optional.empty();
+ Optional<Integer> iterations = Optional.empty();
+ Optional<Integer> maxDatanodesPercentageToInvolvePerIteration =
+ Optional.empty();
Optional<Long> maxSizeToMovePerIterationInGB = Optional.empty();
Optional<Long> maxSizeEnteringTargetInGB = Optional.empty();
Optional<Long> maxSizeLeavingSourceInGB = Optional.empty();
@@ -727,13 +728,23 @@ public final class
StorageContainerLocationProtocolServerSideTranslatorPB
if(request.hasThreshold()) {
threshold = Optional.of(request.getThreshold());
}
- if(request.hasIdleiterations()) {
- idleiterations = Optional.of(request.getIdleiterations());
+
+ if (request.hasIterations()) {
+ iterations = Optional.of(request.getIterations());
+ } else if (request.hasIdleiterations()) {
+ iterations = Optional.of(request.getIdleiterations());
}
- if(request.hasMaxDatanodesRatioToInvolvePerIteration()) {
- maxDatanodesRatioToInvolvePerIteration =
- Optional.of(request.getMaxDatanodesRatioToInvolvePerIteration());
+
+ if (request.hasMaxDatanodesPercentageToInvolvePerIteration()) {
+ maxDatanodesPercentageToInvolvePerIteration =
+
Optional.of(request.getMaxDatanodesPercentageToInvolvePerIteration());
+ } else if (request.hasMaxDatanodesRatioToInvolvePerIteration()) {
+ maxDatanodesPercentageToInvolvePerIteration =
+ Optional.of(
+ (int) (request.getMaxDatanodesRatioToInvolvePerIteration() *
+ 100));
}
+
if(request.hasMaxSizeToMovePerIterationInGB()) {
maxSizeToMovePerIterationInGB =
Optional.of(request.getMaxSizeToMovePerIterationInGB());
@@ -747,11 +758,9 @@ public final class
StorageContainerLocationProtocolServerSideTranslatorPB
Optional.of(request.getMaxSizeLeavingSourceInGB());
}
-
-
return StartContainerBalancerResponseProto.newBuilder().
setStart(impl.startContainerBalancer(threshold,
- idleiterations, maxDatanodesRatioToInvolvePerIteration,
+ iterations, maxDatanodesPercentageToInvolvePerIteration,
maxSizeToMovePerIterationInGB,
maxSizeEnteringTargetInGB, maxSizeLeavingSourceInGB)).build();
}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index 5e3da50..fa1d316 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -744,8 +744,8 @@ public class SCMClientProtocolServer implements
@Override
public boolean startContainerBalancer(
- Optional<Double> threshold, Optional<Integer> idleiterations,
- Optional<Double> maxDatanodesRatioToInvolvePerIteration,
+ Optional<Double> threshold, Optional<Integer> iterations,
+ Optional<Integer> maxDatanodesPercentageToInvolvePerIteration,
Optional<Long> maxSizeToMovePerIterationInGB,
Optional<Long> maxSizeEnteringTarget,
Optional<Long> maxSizeLeavingSource) throws IOException {
@@ -754,8 +754,8 @@ public class SCMClientProtocolServer implements
scm.getConfiguration().getObject(ContainerBalancerConfiguration.class);
if (threshold.isPresent()) {
double tsd = threshold.get();
- Preconditions.checkState(tsd >= 0.0D && tsd < 1.0D,
- "threshold should to be specified in range [0.0, 1.0).");
+ Preconditions.checkState(tsd >= 0.0D && tsd < 100.0D,
+ "threshold should be specified in range [0.0, 100.0).");
cbc.setThreshold(tsd);
}
if (maxSizeToMovePerIterationInGB.isPresent()) {
@@ -764,22 +764,22 @@ public class SCMClientProtocolServer implements
"maxSizeToMovePerIterationInGB must be positive.");
cbc.setMaxSizeToMovePerIteration(mstm * OzoneConsts.GB);
}
- if (maxDatanodesRatioToInvolvePerIteration.isPresent()) {
- double mdti = maxDatanodesRatioToInvolvePerIteration.get();
- Preconditions.checkState(mdti >= 0.0,
- "maxDatanodesRatioToInvolvePerIteration must be " +
+ if (maxDatanodesPercentageToInvolvePerIteration.isPresent()) {
+ int mdti = maxDatanodesPercentageToInvolvePerIteration.get();
+ Preconditions.checkState(mdti >= 0,
+ "maxDatanodesPercentageToInvolvePerIteration must be " +
"greater than equal to zero.");
- Preconditions.checkState(mdti <= 1,
- "maxDatanodesRatioToInvolvePerIteration must be " +
- "lesser than or equal to one.");
- cbc.setMaxDatanodesRatioToInvolvePerIteration(mdti);
+ Preconditions.checkState(mdti <= 100,
+ "maxDatanodesPercentageToInvolvePerIteration must be " +
+ "lesser than or equal to 100.");
+ cbc.setMaxDatanodesPercentageToInvolvePerIteration(mdti);
}
- if (idleiterations.isPresent()) {
- int idi = idleiterations.get();
- Preconditions.checkState(idi > 0 || idi == -1,
- "idleiterations must be positive or" +
- " -1(infinitly run container balancer).");
- cbc.setIdleIteration(idi);
+ if (iterations.isPresent()) {
+ int i = iterations.get();
+ Preconditions.checkState(i > 0 || i == -1,
+ "number of iterations must be positive or" +
+ " -1 (for running container balancer infinitely).");
+ cbc.setIterations(i);
}
if (maxSizeEnteringTarget.isPresent()) {
@@ -807,7 +807,7 @@ public class SCMClientProtocolServer implements
AUDIT.logWriteFailure(buildAuditMessageForSuccess(
SCMAction.START_CONTAINER_BALANCER, null));
}
- return isStartedSuccessfully;
+ return isStartedSuccessfully;
}
@Override
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancer.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancer.java
index 3ad6269..e6f32e0 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancer.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancer.java
@@ -105,9 +105,9 @@ public class TestContainerBalancer {
balancerConfiguration =
conf.getObject(ContainerBalancerConfiguration.class);
- balancerConfiguration.setThreshold(0.1);
- balancerConfiguration.setIdleIteration(1);
- balancerConfiguration.setMaxDatanodesRatioToInvolvePerIteration(1.0d);
+ balancerConfiguration.setThreshold(10);
+ balancerConfiguration.setIterations(1);
+ balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100);
balancerConfiguration.setMaxSizeToMovePerIteration(50 * OzoneConsts.GB);
balancerConfiguration.setMaxSizeEnteringTarget(50 * OzoneConsts.GB);
conf.setFromObject(balancerConfiguration);
@@ -176,7 +176,7 @@ public class TestContainerBalancer {
// check for random threshold values
for (int i = 0; i < 50; i++) {
- double randomThreshold = RANDOM.nextDouble();
+ double randomThreshold = RANDOM.nextDouble() * 100;
balancerConfiguration.setThreshold(randomThreshold);
containerBalancer.start(balancerConfiguration);
@@ -211,7 +211,7 @@ public class TestContainerBalancer {
*/
@Test
public void unBalancedNodesListShouldBeEmptyWhenClusterIsBalanced() {
- balancerConfiguration.setThreshold(0.99);
+ balancerConfiguration.setThreshold(99.99);
containerBalancer.start(balancerConfiguration);
// waiting for balance completed.
@@ -231,10 +231,12 @@ public class TestContainerBalancer {
*/
@Test
public void containerBalancerShouldObeyMaxDatanodesToInvolveLimit() {
- balancerConfiguration.setMaxDatanodesRatioToInvolvePerIteration(0.3d);
+ int percent = 20;
+ balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(
+ percent);
balancerConfiguration.setMaxSizeToMovePerIteration(100 * OzoneConsts.GB);
- balancerConfiguration.setThreshold(0.01);
- balancerConfiguration.setIdleIteration(1);
+ balancerConfiguration.setThreshold(1);
+ balancerConfiguration.setIterations(1);
containerBalancer.start(balancerConfiguration);
// waiting for balance completed.
@@ -246,7 +248,7 @@ public class TestContainerBalancer {
Assert.assertFalse(
containerBalancer.getCountDatanodesInvolvedPerIteration() >
- (int) (0.3 * numberOfNodes));
+ (percent * numberOfNodes / 100));
containerBalancer.stop();
}
@@ -256,7 +258,7 @@ public class TestContainerBalancer {
for (ContainerInfo containerInfo : cidToInfoMap.values()) {
containerInfo.setState(HddsProtos.LifeCycleState.OPEN);
}
- balancerConfiguration.setThreshold(0.1);
+ balancerConfiguration.setThreshold(10);
containerBalancer.start(balancerConfiguration);
// waiting for balance completed.
@@ -298,9 +300,9 @@ public class TestContainerBalancer {
@Test
public void containerBalancerShouldObeyMaxSizeToMoveLimit() {
- balancerConfiguration.setThreshold(0.01);
+ balancerConfiguration.setThreshold(1);
balancerConfiguration.setMaxSizeToMovePerIteration(10 * OzoneConsts.GB);
- balancerConfiguration.setIdleIteration(1);
+ balancerConfiguration.setIterations(1);
containerBalancer.start(balancerConfiguration);
// waiting for balance completed.
@@ -318,9 +320,9 @@ public class TestContainerBalancer {
@Test
public void targetDatanodeShouldNotAlreadyContainSelectedContainer() {
- balancerConfiguration.setThreshold(0.1);
+ balancerConfiguration.setThreshold(10);
balancerConfiguration.setMaxSizeToMovePerIteration(100 * OzoneConsts.GB);
- balancerConfiguration.setMaxDatanodesRatioToInvolvePerIteration(1.0d);
+ balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100);
containerBalancer.start(balancerConfiguration);
// waiting for balance completed.
@@ -345,9 +347,9 @@ public class TestContainerBalancer {
@Test
public void containerMoveSelectionShouldFollowPlacementPolicy() {
- balancerConfiguration.setThreshold(0.1);
+ balancerConfiguration.setThreshold(10);
balancerConfiguration.setMaxSizeToMovePerIteration(50 * OzoneConsts.GB);
- balancerConfiguration.setMaxDatanodesRatioToInvolvePerIteration(1.0d);
+ balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100);
containerBalancer.start(balancerConfiguration);
// waiting for balance completed.
@@ -387,8 +389,8 @@ public class TestContainerBalancer {
@Test
public void targetDatanodeShouldBeInServiceHealthy()
throws NodeNotFoundException {
- balancerConfiguration.setThreshold(0.1);
- balancerConfiguration.setMaxDatanodesRatioToInvolvePerIteration(1.0d);
+ balancerConfiguration.setThreshold(10);
+ balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100);
balancerConfiguration.setMaxSizeToMovePerIteration(50 * OzoneConsts.GB);
balancerConfiguration.setMaxSizeEnteringTarget(50 * OzoneConsts.GB);
containerBalancer.start(balancerConfiguration);
@@ -414,8 +416,8 @@ public class TestContainerBalancer {
@Test
public void selectedContainerShouldNotAlreadyHaveBeenSelected() {
- balancerConfiguration.setThreshold(0.1);
- balancerConfiguration.setMaxDatanodesRatioToInvolvePerIteration(1.0d);
+ balancerConfiguration.setThreshold(10);
+ balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100);
balancerConfiguration.setMaxSizeToMovePerIteration(50 * OzoneConsts.GB);
balancerConfiguration.setMaxSizeEnteringTarget(50 * OzoneConsts.GB);
@@ -440,8 +442,8 @@ public class TestContainerBalancer {
@Test
public void balancerShouldNotSelectConfiguredExcludeContainers() {
- balancerConfiguration.setThreshold(0.1);
- balancerConfiguration.setMaxDatanodesRatioToInvolvePerIteration(1.0d);
+ balancerConfiguration.setThreshold(10);
+ balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100);
balancerConfiguration.setMaxSizeToMovePerIteration(50 * OzoneConsts.GB);
balancerConfiguration.setMaxSizeEnteringTarget(50 * OzoneConsts.GB);
balancerConfiguration.setExcludeContainers("1, 4, 5");
@@ -467,8 +469,8 @@ public class TestContainerBalancer {
@Test
public void balancerShouldObeyMaxSizeEnteringTargetLimit() {
- balancerConfiguration.setThreshold(0.1);
- balancerConfiguration.setMaxDatanodesRatioToInvolvePerIteration(1.0d);
+ balancerConfiguration.setThreshold(10);
+ balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100);
balancerConfiguration.setMaxSizeToMovePerIteration(50 * OzoneConsts.GB);
// no containers should be selected when the limit is zero
@@ -498,11 +500,11 @@ public class TestContainerBalancer {
@Test
public void testMetrics() {
- balancerConfiguration.setThreshold(0.1);
- balancerConfiguration.setIdleIteration(1);
+ balancerConfiguration.setThreshold(10);
+ balancerConfiguration.setIterations(1);
balancerConfiguration.setMaxSizeEnteringTarget(10 * OzoneConsts.GB);
balancerConfiguration.setMaxSizeToMovePerIteration(100 * OzoneConsts.GB);
- balancerConfiguration.setMaxDatanodesRatioToInvolvePerIteration(1.0);
+ balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100);
containerBalancer.start(balancerConfiguration);
@@ -533,11 +535,11 @@ public class TestContainerBalancer {
*/
@Test
public void balancerShouldFollowExcludeAndIncludeDatanodesConfigurations() {
- balancerConfiguration.setThreshold(0.1);
- balancerConfiguration.setIdleIteration(1);
+ balancerConfiguration.setThreshold(10);
+ balancerConfiguration.setIterations(1);
balancerConfiguration.setMaxSizeEnteringTarget(10 * OzoneConsts.GB);
balancerConfiguration.setMaxSizeToMovePerIteration(100 * OzoneConsts.GB);
- balancerConfiguration.setMaxDatanodesRatioToInvolvePerIteration(1.0);
+ balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100);
// only these nodes should be included
// the ones also specified in excludeNodes should be excluded
@@ -588,28 +590,30 @@ public class TestContainerBalancer {
OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
ozoneConfiguration.set("ozone.scm.container.size", "5GB");
ozoneConfiguration.setDouble(
- "hdds.container.balancer.utilization.threshold", 0.01);
+ "hdds.container.balancer.utilization.threshold", 1);
ContainerBalancerConfiguration cbConf =
ozoneConfiguration.getObject(ContainerBalancerConfiguration.class);
- Assert.assertEquals(cbConf.getThreshold(), 0.01d, 0.001);
+ Assert.assertEquals(1, cbConf.getThreshold(), 0.001);
- Assert.assertEquals(cbConf.getMaxSizeLeavingSource(),
- 26 * 1024 * 1024 * 1024L);
+ Assert.assertEquals(26 * 1024 * 1024 * 1024L,
+ cbConf.getMaxSizeLeavingSource());
- Assert.assertEquals(cbConf.getMoveTimeout().toMillis(), 30 * 60 * 1000);
+ Assert.assertEquals(30 * 60 * 1000,
+ cbConf.getMoveTimeout().toMillis());
}
/**
* Determines unBalanced nodes, that is, over and under utilized nodes,
* according to the generated utilization values for nodes and the threshold.
*
- * @param threshold A fraction from range 0 to 1.
+ * @param threshold A percentage in the range 0 to 100
* @return List of DatanodeUsageInfo containing the expected(correct)
* unBalanced nodes.
*/
private List<DatanodeUsageInfo> determineExpectedUnBalancedNodes(
double threshold) {
+ threshold /= 100;
double lowerLimit = averageUtilization - threshold;
double upperLimit = averageUtilization + threshold;
diff --git
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCommands.java
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCommands.java
index 3041651..7f24d84 100644
---
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCommands.java
+++
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCommands.java
@@ -41,24 +41,23 @@ import java.util.concurrent.Callable;
* To start:
* ozone admin containerbalancer start
* [ -t/--threshold {@literal <threshold>}]
- * [ -i/--idleiterations {@literal <idleiterations>}]
- * [ -d/--maxDatanodesRatioToInvolvePerIteration
- * {@literal <maxDatanodesRatioToInvolvePerIteration>}]
+ * [ -i/--iterations {@literal <iterations>}]
+ * [ -d/--maxDatanodesPercentageToInvolvePerIteration
+ * {@literal <maxDatanodesPercentageToInvolvePerIteration>}]
* [ -s/--maxSizeToMovePerIterationInGB
* {@literal <maxSizeToMovePerIterationInGB>}]
* Examples:
* ozone admin containerbalancer start
* start balancer with default values in the configuration
- * ozone admin containerbalancer start -t 0.05
+ * ozone admin containerbalancer start -t 5
* start balancer with a threshold of 5%
* ozone admin containerbalancer start -i 20
- * start balancer with maximum 20 consecutive idle iterations
- * ozone admin containerbalancer start -i 0
+ * start balancer with maximum 20 consecutive iterations
+ * ozone admin containerbalancer start -i -1
* run balancer infinitely with default values in the configuration
- * ozone admin containerbalancer start -d 0.4
- * start balancer with the ratio of maximum datanodes to involve in
- * balancing in one iteration to the total number of healthy datanodes
as
- * 0.4
+ * ozone admin containerbalancer start -d 40
+ * start balancer with maximum 40% of healthy, in-service datanodes
+ * involved in balancing
* ozone admin containerbalancer start -s 10
* start balancer with maximum size of 10GB to move in one iteration
* To stop:
diff --git
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java
index a4ee0d2..73d8511 100644
---
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java
+++
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java
@@ -36,44 +36,50 @@ import java.util.Optional;
public class ContainerBalancerStartSubcommand extends ScmSubcommand {
@Option(names = {"-t", "--threshold"},
- description = "Threshold target whether the cluster is balanced")
+ description = "Percentage deviation from average utilization of " +
+ "the cluster after which a datanode will be rebalanced (for " +
+ "example, '10' for 10%).")
private Optional<Double> threshold;
- @Option(names = {"-i", "--idleiterations"},
- description = "Maximum consecutive idle iterations")
- private Optional<Integer> idleiterations;
+ @Option(names = {"-i", "--iterations"},
+ description = "Maximum consecutive iterations that" +
+ " balancer will run for.")
+ private Optional<Integer> iterations;
- @Option(names = {"-d", "--maxDatanodesRatioToInvolvePerIteration"},
- description = "The ratio of maximum number of datanodes that should be "
+
- "involved in balancing in one iteration to the total number of " +
- "healthy, in service nodes known to container balancer.")
- private Optional<Double> maxDatanodesRatioToInvolvePerIteration;
+ @Option(names = {"-d", "--maxDatanodesPercentageToInvolvePerIteration"},
+ description = "Max percentage of healthy, in service datanodes " +
+ "that can be involved in balancing in one iteration (for example, " +
+ "'20' for 20%).")
+ private Optional<Integer> maxDatanodesPercentageToInvolvePerIteration;
@Option(names = {"-s", "--maxSizeToMovePerIterationInGB"},
- description = "Maximum size to move per iteration of balancing in GB, " +
- "for 10GB it should be set as 10")
+ description = "Maximum size that can be moved per iteration of " +
+ "balancing (for example, '500' for 500GB).")
private Optional<Long> maxSizeToMovePerIterationInGB;
- @Option(names = {"-e", "--maxSizeEnteringTarget"},
- description = "the maximum size that can enter a target datanode while "
+
- "balancing in GB. This is the sum of data from multiple sources.")
+ @Option(names = {"-e", "--maxSizeEnteringTargetInGB"},
+ description = "Maximum size that can enter a target datanode while " +
+ "balancing. This is the sum of data from multiple sources (for " +
+ "example, '26' for 26GB).")
private Optional<Long> maxSizeEnteringTargetInGB;
- @Option(names = {"-l", "--maxSizeLeavingSource"},
- description = "maximum size that can leave a source datanode while " +
- "balancing in GB, it is the sum of data moving to multiple targets.")
+ @Option(names = {"-l", "--maxSizeLeavingSourceInGB"},
+ description = "Maximum size that can leave a source datanode while " +
+ "balancing. This is the sum of data moving to multiple targets " +
+ "(for example, '26' for 26GB).")
private Optional<Long> maxSizeLeavingSourceInGB;
@Override
public void execute(ScmClient scmClient) throws IOException {
- boolean result = scmClient.startContainerBalancer(threshold,
idleiterations,
- maxDatanodesRatioToInvolvePerIteration, maxSizeToMovePerIterationInGB,
- maxSizeEnteringTargetInGB, maxSizeLeavingSourceInGB);
+ boolean result = scmClient.startContainerBalancer(threshold, iterations,
+ maxDatanodesPercentageToInvolvePerIteration,
+ maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB,
+ maxSizeLeavingSourceInGB);
if (result) {
- System.out.println("Starting ContainerBalancer Successfully.");
+ System.out.println("Container Balancer started successfully.");
return;
}
- System.out.println("ContainerBalancer is already running, " +
+ System.out.println("Container Balancer is already running. " +
"Please stop it first.");
}
}
\ No newline at end of file
diff --git
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
index 30f0ec8..123a74d 100644
---
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
+++
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
@@ -552,14 +552,14 @@ public class ContainerOperationClient implements
ScmClient {
@Override
public boolean startContainerBalancer(
- Optional<Double> threshold, Optional<Integer> idleiterations,
- Optional<Double> maxDatanodesRatioToInvolvePerIteration,
+ Optional<Double> threshold, Optional<Integer> iterations,
+ Optional<Integer> maxDatanodesPercentageToInvolvePerIteration,
Optional<Long> maxSizeToMovePerIterationInGB,
Optional<Long> maxSizeEnteringTargetInGB,
Optional<Long> maxSizeLeavingSourceInGB)
throws IOException {
return storageContainerLocationClient.startContainerBalancer(threshold,
- idleiterations, maxDatanodesRatioToInvolvePerIteration,
+ iterations, maxDatanodesPercentageToInvolvePerIteration,
maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB,
maxSizeLeavingSourceInGB);
}
diff --git
a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java
b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java
index 8ed6acd..a7ef779 100644
---
a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java
+++
b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java
@@ -119,8 +119,8 @@ public class TestContainerBalancerSubCommand {
.thenAnswer(invocation -> true);
startCmd.execute(scmClient);
- Pattern p = Pattern.compile("^Starting\\sContainerBalancer" +
- "\\sSuccessfully.");
+ Pattern p = Pattern.compile("^Container\\sBalancer\\sstarted" +
+ "\\ssuccessfully.");
Matcher m = p.matcher(outContent.toString(DEFAULT_ENCODING));
assertTrue(m.find());
}
@@ -134,8 +134,8 @@ public class TestContainerBalancerSubCommand {
.thenAnswer(invocation -> false);
startCmd.execute(scmClient);
- Pattern p = Pattern.compile("^ContainerBalancer\\sis\\salready\\srunning,"
+
- "\\sPlease\\sstop\\sit\\sfirst.");
+ Pattern p = Pattern.compile("^Container\\sBalancer\\sis\\salready" +
+ "\\srunning.\\sPlease\\sstop\\sit\\sfirst.");
Matcher m = p.matcher(outContent.toString(DEFAULT_ENCODING));
assertTrue(m.find());
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java
index ac5b737..2b0dfc2 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java
@@ -81,15 +81,17 @@ public class TestContainerBalancerOperations {
boolean running = containerBalancerClient.getContainerBalancerStatus();
assertFalse(running);
Optional<Double> threshold = Optional.of(0.1);
- Optional<Integer> idleiterations = Optional.of(10000);
- Optional<Double> maxDatanodesRatioToInvolvePerIteration = Optional.of(1d);
+ Optional<Integer> iterations = Optional.of(10000);
+ Optional<Integer> maxDatanodesPercentageToInvolvePerIteration =
+ Optional.of(100);
Optional<Long> maxSizeToMovePerIterationInGB = Optional.of(1L);
Optional<Long> maxSizeEnteringTargetInGB = Optional.of(1L);
Optional<Long> maxSizeLeavingSourceInGB = Optional.of(1L);
- containerBalancerClient.startContainerBalancer(threshold, idleiterations,
- maxDatanodesRatioToInvolvePerIteration, maxSizeToMovePerIterationInGB,
- maxSizeEnteringTargetInGB, maxSizeLeavingSourceInGB);
+ containerBalancerClient.startContainerBalancer(threshold, iterations,
+ maxDatanodesPercentageToInvolvePerIteration,
+ maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB,
+ maxSizeLeavingSourceInGB);
running = containerBalancerClient.getContainerBalancerStatus();
assertTrue(running);
@@ -104,9 +106,10 @@ public class TestContainerBalancerOperations {
assertFalse(running);
// test normally start , and stop it before balance is completed
- containerBalancerClient.startContainerBalancer(threshold, idleiterations,
- maxDatanodesRatioToInvolvePerIteration, maxSizeToMovePerIterationInGB,
- maxSizeEnteringTargetInGB, maxSizeLeavingSourceInGB);
+ containerBalancerClient.startContainerBalancer(threshold, iterations,
+ maxDatanodesPercentageToInvolvePerIteration,
+ maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB,
+ maxSizeLeavingSourceInGB);
running = containerBalancerClient.getContainerBalancerStatus();
assertTrue(running);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
