This is an automated email from the ASF dual-hosted git repository.
sarvekshayr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new e31f36d2316 HDDS-15667. `containerbalancer status`: show stop reason
and iteration details after stop (#10681)
e31f36d2316 is described below
commit e31f36d2316a5be67e2e86a91bc6deecbacf0a0f
Author: sreejasahithi <[email protected]>
AuthorDate: Wed Jul 15 09:38:26 2026 +0530
HDDS-15667. `containerbalancer status`: show stop reason and iteration
details after stop (#10681)
---
.../src/main/proto/ScmAdminProtocol.proto | 3 +
.../interface-admin/src/main/resources/proto.lock | 18 ++
.../scm/container/balancer/ContainerBalancer.java | 19 +-
.../balancer/ContainerBalancerStatusInfo.java | 43 +++-
.../balancer/ContainerBalancerStopReason.java | 63 ++++++
.../container/balancer/ContainerBalancerTask.java | 101 ++++++++-
.../hdds/scm/server/SCMClientProtocolServer.java | 13 +-
.../container/balancer/TestContainerBalancer.java | 43 ++++
.../balancer/TestContainerBalancerStatusInfo.java | 66 ++++++
.../scm/cli/ContainerBalancerStatusSubcommand.java | 133 +++++++----
.../datanode/TestContainerBalancerSubCommand.java | 242 ++++++++++++++++++++-
.../src/main/smoketest/balancer/testBalancer.robot | 2 +-
12 files changed, 684 insertions(+), 62 deletions(-)
diff --git a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
index d33e949c01a..6c39fc22cc4 100644
--- a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
+++ b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
@@ -658,6 +658,9 @@ message ContainerBalancerStatusInfoProto {
optional uint64 startedAt = 1;
optional ContainerBalancerConfigurationProto configuration = 2;
repeated ContainerBalancerTaskIterationStatusInfoProto iterationsStatusInfo
= 3;
+ optional uint64 stoppedAt = 4;
+ optional string stopReason = 5;
+ optional string stopMessage = 6;
}
message ContainerBalancerTaskIterationStatusInfoProto {
diff --git a/hadoop-hdds/interface-admin/src/main/resources/proto.lock
b/hadoop-hdds/interface-admin/src/main/resources/proto.lock
index e4d79fc60e2..02184011b69 100644
--- a/hadoop-hdds/interface-admin/src/main/resources/proto.lock
+++ b/hadoop-hdds/interface-admin/src/main/resources/proto.lock
@@ -2250,6 +2250,24 @@
"name": "iterationsStatusInfo",
"type": "ContainerBalancerTaskIterationStatusInfoProto",
"is_repeated": true
+ },
+ {
+ "id": 4,
+ "name": "stoppedAt",
+ "type": "uint64",
+ "optional": true
+ },
+ {
+ "id": 5,
+ "name": "stopReason",
+ "type": "string",
+ "optional": true
+ },
+ {
+ "id": 6,
+ "name": "stopMessage",
+ "type": "string",
+ "optional": true
}
]
},
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java
index c516447f09f..3631df49f5a 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java
@@ -191,13 +191,28 @@ public ContainerBalancerTask.Status getBalancerStatus() {
public ContainerBalancerStatusInfo getBalancerStatusInfo() throws
IOException {
lock.lock();
try {
- if (isBalancerRunning()) {
+ if (task == null) {
+ return null;
+ }
+ ContainerBalancerTask.Status status = task.getBalancerStatus();
+ if (status == ContainerBalancerTask.Status.RUNNING
+ || status == ContainerBalancerTask.Status.STOPPING) {
return new ContainerBalancerStatusInfo(
this.startedAt,
config.toProtobufBuilder().setShouldRun(true).build(),
task.getCurrentIterationsStatistic()
);
}
+ if (status == ContainerBalancerTask.Status.STOPPED) {
+ return new ContainerBalancerStatusInfo(
+ this.startedAt,
+ config.toProtobufBuilder().setShouldRun(false).build(),
+ task.getCurrentIterationsStatistic(),
+ task.getStoppedAt(),
+ task.getStopReason(),
+ task.getStopMessage()
+ );
+ }
return null;
} finally {
lock.unlock();
@@ -362,6 +377,7 @@ public void stop() {
return;
}
LOG.info("Trying to stop ContainerBalancer in this SCM.");
+ task.recordStopReason(ContainerBalancerStopReason.SCM_STATE_CHANGE);
task.stop();
balancingThread = currentBalancingThread;
} finally {
@@ -402,6 +418,7 @@ public void stopBalancer()
saveConfiguration(config, false, 0);
if (isBalancerRunning()) {
LOG.info("Trying to stop ContainerBalancer service.");
+ task.recordStopReason(ContainerBalancerStopReason.USER_REQUESTED);
task.stop();
balancingThread = currentBalancingThread;
}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerStatusInfo.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerStatusInfo.java
index 0ac0a26682a..be1ea624560 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerStatusInfo.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerStatusInfo.java
@@ -30,14 +30,30 @@ public class ContainerBalancerStatusInfo {
private final OffsetDateTime startedAt;
private final HddsProtos.ContainerBalancerConfigurationProto configuration;
private final List<ContainerBalancerTaskIterationStatusInfo>
iterationsStatusInfo;
+ private final OffsetDateTime stoppedAt;
+ private final String stopReason;
+ private final String stopMessage;
public ContainerBalancerStatusInfo(
OffsetDateTime startedAt,
HddsProtos.ContainerBalancerConfigurationProto configuration,
List<ContainerBalancerTaskIterationStatusInfo> iterationsStatusInfo)
{
+ this(startedAt, configuration, iterationsStatusInfo, null, null, null);
+ }
+
+ public ContainerBalancerStatusInfo(
+ OffsetDateTime startedAt,
+ HddsProtos.ContainerBalancerConfigurationProto configuration,
+ List<ContainerBalancerTaskIterationStatusInfo> iterationsStatusInfo,
+ OffsetDateTime stoppedAt,
+ String stopReason,
+ String stopMessage) {
this.startedAt = startedAt;
this.configuration = configuration;
this.iterationsStatusInfo = iterationsStatusInfo;
+ this.stoppedAt = stoppedAt;
+ this.stopReason = stopReason;
+ this.stopMessage = stopMessage;
}
public OffsetDateTime getStartedAt() {
@@ -52,12 +68,25 @@ public List<ContainerBalancerTaskIterationStatusInfo>
getIterationsStatusInfo()
return iterationsStatusInfo;
}
+ public OffsetDateTime getStoppedAt() {
+ return stoppedAt;
+ }
+
+ public String getStopReason() {
+ return stopReason;
+ }
+
+ public String getStopMessage() {
+ return stopMessage;
+ }
+
/**
* Converts an instance into a protobuf-compatible object.
* @return proto representation
*/
public
StorageContainerLocationProtocolProtos.ContainerBalancerStatusInfoProto
toProto() {
- return
StorageContainerLocationProtocolProtos.ContainerBalancerStatusInfoProto
+
StorageContainerLocationProtocolProtos.ContainerBalancerStatusInfoProto.Builder
builder =
+ StorageContainerLocationProtocolProtos.ContainerBalancerStatusInfoProto
.newBuilder()
.setStartedAt(getStartedAt().toEpochSecond())
.setConfiguration(getConfiguration())
@@ -66,6 +95,16 @@ public
StorageContainerLocationProtocolProtos.ContainerBalancerStatusInfoProto t
.stream()
.map(ContainerBalancerTaskIterationStatusInfo::toProto)
.collect(Collectors.toList())
- ).build();
+ );
+ if (stoppedAt != null) {
+ builder.setStoppedAt(stoppedAt.toEpochSecond());
+ }
+ if (stopReason != null) {
+ builder.setStopReason(stopReason);
+ }
+ if (stopMessage != null) {
+ builder.setStopMessage(stopMessage);
+ }
+ return builder.build();
}
}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerStopReason.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerStopReason.java
new file mode 100644
index 00000000000..46b9908db8c
--- /dev/null
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerStopReason.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.container.balancer;
+
+/**
+ * Stop reason codes and messages for ContainerBalancer.
+ */
+public enum ContainerBalancerStopReason {
+ USER_REQUESTED("Stopped by user request."),
+ SCM_STATE_CHANGE("Stopped because SCM state changed."),
+ COMPLETED_ALL_ITERATIONS("Completed all configured number of iterations."),
+ CAN_NOT_BALANCE_ANY_MORE("No more eligible container moves were found."),
+ INITIALIZATION_FAILED("Failed to initialize a container balancer
iteration."),
+ ERROR("Stopped because of an unexpected error."),
+ UNKNOWN("Stopped for an unknown reason.");
+
+ public static final String INIT_SCM_NOT_READY = "SCM is in safe mode or is
not leader ready.";
+ public static final String INIT_EMPTY_DATANODE_LIST = "Received an empty
list of Datanodes from Node Manager.";
+ public static final String INIT_NO_UNBALANCED_DATANODES = "Did not find any
unbalanced Datanodes.";
+
+ private final String message;
+
+ ContainerBalancerStopReason(String message) {
+ this.message = message;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+
+ public String formatMessage(String details) {
+ if (details == null || details.isEmpty()) {
+ return message;
+ }
+ return message + " Details: " + details;
+ }
+
+ public static String exceptionDetails(Throwable throwable) {
+ if (throwable == null) {
+ return "";
+ }
+ String exceptionMessage = throwable.getMessage();
+ if (exceptionMessage != null && !exceptionMessage.isEmpty()) {
+ return throwable.getClass().getName() + ": " + exceptionMessage;
+ }
+ return throwable.toString();
+ }
+}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java
index e0281575f17..0a7c2c62156 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java
@@ -120,6 +120,10 @@ public class ContainerBalancerTask implements Runnable {
private Queue<ContainerBalancerTaskIterationStatusInfo> iterationsStatistic;
private OffsetDateTime currentIterationStarted;
private AtomicBoolean isCurrentIterationInProgress = new
AtomicBoolean(false);
+ private volatile String stopReason;
+ private volatile String stopMessage;
+ private volatile OffsetDateTime stoppedAt;
+ private volatile String lastInitializationFailureDetail;
/**
* Constructs ContainerBalancerTask with the specified arguments.
@@ -190,8 +194,17 @@ public void run() {
balance();
} catch (Exception e) {
LOG.error("Container Balancer is stopped abnormally, ", e);
+ recordStopReason(ContainerBalancerStopReason.ERROR,
+ ContainerBalancerStopReason.exceptionDetails(e));
} finally {
synchronized (this) {
+ finalizeInProgressIteration();
+ if (stoppedAt == null) {
+ stoppedAt = now();
+ }
+ if (stopReason == null) {
+ recordStopReason(ContainerBalancerStopReason.UNKNOWN);
+ }
taskStatus = Status.STOPPED;
}
}
@@ -264,8 +277,9 @@ private void balance() {
return;
}
// otherwise, try to stop balancer
- tryStopWithSaveConfiguration("Could not initialize " +
- "ContainerBalancer's iteration number " + i);
+ isCurrentIterationInProgress.compareAndSet(true, false);
+
tryStopWithSaveConfiguration(ContainerBalancerStopReason.INITIALIZATION_FAILED,
+ " iteration number " + (i + 1) + ", " +
lastInitializationFailureDetail);
return;
}
@@ -286,7 +300,7 @@ private void balance() {
// if no new move option is generated, it means the cluster cannot be
// balanced anymore; so just stop balancer
if (currentIterationResult == IterationResult.CAN_NOT_BALANCE_ANY_MORE) {
- tryStopWithSaveConfiguration(currentIterationResult.toString());
+
tryStopWithSaveConfiguration(ContainerBalancerStopReason.CAN_NOT_BALANCE_ANY_MORE);
return;
}
@@ -319,7 +333,7 @@ private void balance() {
}
}
- tryStopWithSaveConfiguration("Completed all iterations.");
+
tryStopWithSaveConfiguration(ContainerBalancerStopReason.COMPLETED_ALL_ITERATIONS);
}
private ContainerBalancerTaskIterationStatusInfo
getIterationStatistic(Integer iterationNumber,
@@ -417,21 +431,88 @@ private long getCurrentIterationDuration() {
/**
* Logs the reason for stop and save configuration and stop the task.
*
- * @param stopReason a string specifying the reason for stop
+ * @param reason stop reason
+ */
+ private void tryStopWithSaveConfiguration(ContainerBalancerStopReason
reason) {
+ tryStopWithSaveConfiguration(reason, null);
+ }
+
+ /**
+ * Logs the reason for stop and save configuration and stop the task.
+ *
+ * @param reason stable stop reason code
+ * @param details optional details appended to the human-readable message
*/
- private void tryStopWithSaveConfiguration(String stopReason) {
+ private void tryStopWithSaveConfiguration(ContainerBalancerStopReason
reason, String details) {
synchronized (this) {
try {
- LOG.info("Save Configuration for stopping. Reason: {}", stopReason);
saveConfiguration(config, false, 0);
+ recordStopReason(reason, details);
+ LOG.info("Save Configuration for stopping. Reason: {}, Message: {}",
+ reason.name(), stopMessage);
stop();
} catch (IOException | TimeoutException e) {
+ recordStopReason(reason, details);
LOG.warn("Save configuration failed. Reason for " +
- "stopping: {}", stopReason, e);
+ "stopping: {}, Message: {}", reason.name(), stopMessage, e);
}
}
}
+ /**
+ * Records the reason why the balancer task is stopping.
+ *
+ * @param reason stop reason
+ */
+ public void recordStopReason(ContainerBalancerStopReason reason) {
+ recordStopReason(reason, null);
+ }
+
+ /**
+ * Records the reason why the balancer task is stopping.
+ *
+ * @param reason stop reason
+ * @param details optional details appended to the message
+ */
+ public void recordStopReason(ContainerBalancerStopReason reason, String
details) {
+ synchronized (this) {
+ if (stopReason == null) {
+ stopReason = reason.name();
+ stopMessage = reason.formatMessage(details);
+ }
+ }
+ }
+
+ private void finalizeInProgressIteration() {
+ if (!isCurrentIterationInProgress.get()) {
+ return;
+ }
+ List<ContainerBalancerTaskIterationStatusInfo> resultList = new
ArrayList<>(iterationsStatistic);
+ int lastIterationNumber = resultList.stream()
+ .mapToInt(ContainerBalancerTaskIterationStatusInfo::getIterationNumber)
+ .max()
+ .orElse(0);
+ long iterationDuration = getCurrentIterationDuration();
+ iterationsStatistic.offer(
+ getIterationStatistic(
+ lastIterationNumber + 1,
+ IterationResult.ITERATION_INTERRUPTED,
+ iterationDuration));
+ isCurrentIterationInProgress.set(false);
+ }
+
+ public String getStopReason() {
+ return stopReason;
+ }
+
+ public String getStopMessage() {
+ return stopMessage;
+ }
+
+ public OffsetDateTime getStoppedAt() {
+ return stoppedAt;
+ }
+
private void saveConfiguration(ContainerBalancerConfiguration configuration,
boolean shouldRun, int index)
throws IOException, TimeoutException {
@@ -454,7 +535,9 @@ private void
saveConfiguration(ContainerBalancerConfiguration configuration,
* @return true if successfully initialized, otherwise false.
*/
private boolean initializeIteration() {
+ lastInitializationFailureDetail = null;
if (!isValidSCMState()) {
+ lastInitializationFailureDetail =
ContainerBalancerStopReason.INIT_SCM_NOT_READY;
return false;
}
// sorted list in order from most to least used
@@ -463,6 +546,7 @@ private boolean initializeIteration() {
if (datanodeUsageInfos.isEmpty()) {
LOG.warn("Received an empty list of datanodes from Node Manager when " +
"trying to identify which nodes to balance");
+ lastInitializationFailureDetail =
ContainerBalancerStopReason.INIT_EMPTY_DATANODE_LIST;
return false;
}
@@ -540,6 +624,7 @@ private boolean initializeIteration() {
if (overUtilizedNodes.isEmpty() && underUtilizedNodes.isEmpty()) {
LOG.info("Did not find any unbalanced Datanodes.");
+ lastInitializationFailureDetail =
ContainerBalancerStopReason.INIT_NO_UNBALANCED_DATANODES;
return false;
}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index 27f9d124401..7324ae32b6d 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -1379,14 +1379,13 @@ public ContainerBalancerStatusInfoResponseProto
getContainerBalancerStatusInfo()
.newBuilder()
.setIsRunning(false)
.build();
- } else {
-
- return ContainerBalancerStatusInfoResponseProto
- .newBuilder()
- .setIsRunning(true)
- .setContainerBalancerStatusInfo(balancerStatusInfo.toProto())
- .build();
}
+
+ return ContainerBalancerStatusInfoResponseProto
+ .newBuilder()
+ .setIsRunning(balancerStatusInfo.getConfiguration().getShouldRun())
+ .setContainerBalancerStatusInfo(balancerStatusInfo.toProto())
+ .build();
}
/**
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancer.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancer.java
index 78a6491a70e..4864f235d5b 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancer.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancer.java
@@ -23,6 +23,7 @@
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertThrowsExactly;
@@ -328,6 +329,48 @@ public void testStartBalancerWithInvalidNodes() throws
Exception {
assertSame(ContainerBalancerTask.Status.STOPPED,
containerBalancer.getBalancerStatus());
}
+ @Test
+ public void testGetBalancerStatusInfoAfterUserStop() throws Exception {
+ balancerConfiguration.setIterations(10);
+ balancerConfiguration.setTriggerDuEnable(true);
+ conf.setFromObject(balancerConfiguration);
+
+ startBalancer(balancerConfiguration);
+ assertSame(ContainerBalancerTask.Status.RUNNING,
containerBalancer.getBalancerStatus());
+
+ stopBalancer();
+ assertSame(ContainerBalancerTask.Status.STOPPED,
containerBalancer.getBalancerStatus());
+
+ ContainerBalancerStatusInfo statusInfo =
containerBalancer.getBalancerStatusInfo();
+ assertNotNull(statusInfo);
+ assertEquals(ContainerBalancerStopReason.USER_REQUESTED.name(),
+ statusInfo.getStopReason());
+ assertEquals(ContainerBalancerStopReason.USER_REQUESTED.getMessage(),
+ statusInfo.getStopMessage());
+ assertNotNull(statusInfo.getStoppedAt());
+ assertFalse(statusInfo.getConfiguration().getShouldRun());
+ assertFalse(containerBalancer.isBalancerRunning());
+ }
+
+ @Test
+ public void testGetBalancerStatusInfoAfterScmStop() throws Exception {
+ balancerConfiguration.setIterations(10);
+ balancerConfiguration.setTriggerDuEnable(true);
+ conf.setFromObject(balancerConfiguration);
+
+ startBalancer(balancerConfiguration);
+ containerBalancer.stop();
+ assertSame(ContainerBalancerTask.Status.STOPPED,
containerBalancer.getBalancerStatus());
+
+ ContainerBalancerStatusInfo statusInfo =
containerBalancer.getBalancerStatusInfo();
+ assertNotNull(statusInfo);
+ assertEquals(ContainerBalancerStopReason.SCM_STATE_CHANGE.name(),
+ statusInfo.getStopReason());
+ assertEquals(ContainerBalancerStopReason.SCM_STATE_CHANGE.getMessage(),
+ statusInfo.getStopMessage());
+ assertNotNull(statusInfo.getStoppedAt());
+ }
+
private void startBalancer(ContainerBalancerConfiguration config)
throws IllegalContainerBalancerStateException, IOException,
InvalidContainerBalancerConfigurationException, TimeoutException {
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerStatusInfo.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerStatusInfo.java
index 6d082fe7cf5..f0c4e32c829 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerStatusInfo.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerStatusInfo.java
@@ -22,6 +22,9 @@
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
import java.util.List;
import java.util.Map;
@@ -29,6 +32,7 @@
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeID;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.ozone.test.LambdaTestUtils;
@@ -258,4 +262,66 @@ void
testGetCurrentIterationsStatisticDoesNotThrowNullPointerExceptionWhenBalanc
thread.start();
Assertions.assertDoesNotThrow(task::getCurrentIterationsStatistic);
}
+
+ @Test
+ void testFinalizeInProgressIterationOnStop() throws Exception {
+ MockedSCM mockedScm = new MockedSCM(new MockCluster(20, OzoneConsts.GB));
+
+ ContainerBalancerConfiguration config =
+ new
OzoneConfiguration().getObject(ContainerBalancerConfiguration.class);
+ config.setIterations(3);
+ config.setBalancingInterval(0);
+ config.setMaxSizeToMovePerIteration(50 * OzoneConsts.GB);
+ config.setTriggerDuEnable(false);
+
+ ContainerBalancerTask task = mockedScm.startBalancerTaskAsync(config,
false);
+ LambdaTestUtils.await(5000, 10,
+ () -> !task.getCurrentIterationsStatistic().isEmpty()
+ && task.getCurrentIterationsStatistic().stream()
+ .anyMatch(it -> it.getContainerMovesScheduled() > 0));
+
+ task.stop();
+ LambdaTestUtils.await(5000, 10,
+ () -> task.getBalancerStatus() ==
ContainerBalancerTask.Status.STOPPED);
+ assertNotNull(task.getStoppedAt());
+ assertEquals(ContainerBalancerStopReason.UNKNOWN.name(),
task.getStopReason());
+ assertEquals(ContainerBalancerStopReason.UNKNOWN.getMessage(),
task.getStopMessage());
+
+ boolean hasInterruptedIteration =
task.getCurrentIterationsStatistic().stream()
+ .anyMatch(it ->
"ITERATION_INTERRUPTED".equals(it.getIterationResult()));
+ assertTrue(hasInterruptedIteration);
+ }
+
+ @Test
+ void testAbnormalStopRecordsErrorReasonAndFinalizesIteration() throws
Exception {
+ MockedSCM mockedScm = new MockedSCM(new MockCluster(20, OzoneConsts.GB));
+
+ ContainerBalancerConfiguration config =
+ new
OzoneConfiguration().getObject(ContainerBalancerConfiguration.class);
+ config.setIterations(3);
+ config.setBalancingInterval(0);
+ config.setMaxSizeToMovePerIteration(50 * OzoneConsts.GB);
+ config.setTriggerDuEnable(false);
+
+ mockedScm.init(config, new OzoneConfiguration());
+ NodeManager throwingNodeManager = mock(NodeManager.class);
+ when(throwingNodeManager.getMostOrLeastUsedDatanodes(anyBoolean()))
+ .thenThrow(new RuntimeException());
+ when(mockedScm.getStorageContainerManager().getScmNodeManager())
+ .thenReturn(throwingNodeManager);
+
+ ContainerBalancerTask task = mockedScm.startBalancerTaskAsync(
+ new ContainerBalancer(mockedScm.getStorageContainerManager()), config,
false);
+ LambdaTestUtils.await(5000, 10,
+ () -> task.getBalancerStatus() ==
ContainerBalancerTask.Status.STOPPED);
+ assertEquals(ContainerBalancerStopReason.ERROR.name(),
task.getStopReason());
+ assertEquals(ContainerBalancerStopReason.ERROR.formatMessage(
+ ContainerBalancerStopReason.exceptionDetails(new
RuntimeException())),
+ task.getStopMessage());
+ assertNotNull(task.getStoppedAt());
+
+ boolean hasInterruptedIteration =
task.getCurrentIterationsStatistic().stream()
+ .anyMatch(it ->
"ITERATION_INTERRUPTED".equals(it.getIterationResult()));
+ assertTrue(hasInterruptedIteration);
+ }
}
diff --git
a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStatusSubcommand.java
b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStatusSubcommand.java
index a6180d687c1..a55fc7906db 100644
---
a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStatusSubcommand.java
+++
b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStatusSubcommand.java
@@ -56,54 +56,107 @@ public class ContainerBalancerStatusSubcommand extends
ScmSubcommand {
@Override
public void execute(ScmClient scmClient) throws IOException {
+ if (verboseWithHistory && !isVerbose()) {
+ System.err.println("Warning: -H/--history has no effect without
-v/--verbose.");
+ }
ContainerBalancerStatusInfoResponseProto response =
scmClient.getContainerBalancerStatusInfo();
boolean isRunning = response.getIsRunning();
ContainerBalancerStatusInfoProto balancerStatusInfo =
response.getContainerBalancerStatusInfo();
if (isRunning) {
- Instant startedAtInstant =
Instant.ofEpochSecond(balancerStatusInfo.getStartedAt());
- LocalDateTime dateTime =
- LocalDateTime.ofInstant(startedAtInstant, ZoneId.systemDefault());
System.out.println("ContainerBalancer is Running.");
+ } else if (response.hasContainerBalancerStatusInfo()) {
+ System.out.println("ContainerBalancer is Not Running.");
+ printStopReasonAndMessage(balancerStatusInfo);
+ } else {
+ System.out.println("ContainerBalancer is Not Running.");
+ }
- if (isVerbose()) {
- System.out.printf("Started at: %s %s%n",
- dateTime.toLocalDate().format(DateTimeFormatter.ISO_LOCAL_DATE),
- dateTime.toLocalTime().format(DateTimeFormatter.ISO_LOCAL_TIME));
- Duration balancingDuration = Duration.between(startedAtInstant,
OffsetDateTime.now());
- System.out.printf("Balancing duration: %s%n%n",
getPrettyDuration(balancingDuration));
-
System.out.println(getConfigurationPrettyString(balancerStatusInfo.getConfiguration()));
- List<ContainerBalancerTaskIterationStatusInfoProto>
iterationsStatusInfoList
- = balancerStatusInfo.getIterationsStatusInfoList();
-
- System.out.println("Current iteration info:");
- ContainerBalancerTaskIterationStatusInfoProto
currentIterationStatistic = iterationsStatusInfoList.stream()
- .filter(it -> it.getIterationResult().isEmpty())
- .findFirst()
- .orElse(null);
- if (currentIterationStatistic == null) {
- System.out.println("-");
- System.out.println();
- } else {
- System.out.println(
- getPrettyIterationStatusInfo(currentIterationStatistic)
- );
- }
-
-
- if (verboseWithHistory) {
- System.out.println("Iteration history list:");
- System.out.println(
- iterationsStatusInfoList
- .stream()
- .filter(it -> !it.getIterationResult().isEmpty())
- .map(this::getPrettyIterationStatusInfo)
- .collect(Collectors.joining(System.lineSeparator()))
- );
- }
- }
+ if (isVerbose() && response.hasContainerBalancerStatusInfo()) {
+ printVerboseStatusInfo(balancerStatusInfo, isRunning);
+ }
+ }
+
+ private void printVerboseStatusInfo(ContainerBalancerStatusInfoProto
balancerStatusInfo, boolean isRunning) {
+ Instant startedAtInstant =
Instant.ofEpochSecond(balancerStatusInfo.getStartedAt());
+ LocalDateTime startedAtDateTime =
+ LocalDateTime.ofInstant(startedAtInstant, ZoneId.systemDefault());
+ System.out.printf("Started at: %s %s%n",
+
startedAtDateTime.toLocalDate().format(DateTimeFormatter.ISO_LOCAL_DATE),
+
startedAtDateTime.toLocalTime().format(DateTimeFormatter.ISO_LOCAL_TIME));
+
+ Instant endInstant = balancerStatusInfo.hasStoppedAt()
+ ? Instant.ofEpochSecond(balancerStatusInfo.getStoppedAt())
+ : OffsetDateTime.now().toInstant();
+ if (balancerStatusInfo.hasStoppedAt()) {
+ LocalDateTime stoppedAtDateTime =
+ LocalDateTime.ofInstant(endInstant, ZoneId.systemDefault());
+ System.out.printf("Stopped at: %s %s%n",
+
stoppedAtDateTime.toLocalDate().format(DateTimeFormatter.ISO_LOCAL_DATE),
+
stoppedAtDateTime.toLocalTime().format(DateTimeFormatter.ISO_LOCAL_TIME));
+ }
+ Duration balancingDuration = Duration.between(startedAtInstant,
endInstant);
+ System.out.printf("Balancing duration: %s%n%n",
getPrettyDuration(balancingDuration));
+
System.out.println(getConfigurationPrettyString(balancerStatusInfo.getConfiguration()));
+ List<ContainerBalancerTaskIterationStatusInfoProto>
iterationsStatusInfoList =
+ balancerStatusInfo.getIterationsStatusInfoList();
+ ContainerBalancerTaskIterationStatusInfoProto lastIterationStatistic =
null;
+ if (isRunning) {
+ System.out.println("Current iteration info:");
+ ContainerBalancerTaskIterationStatusInfoProto currentIterationStatistic
= iterationsStatusInfoList.stream()
+ .filter(it -> it.getIterationResult().isEmpty())
+ .findFirst()
+ .orElse(null);
+ if (currentIterationStatistic == null) {
+ System.out.println("-");
+ System.out.println();
+ } else {
+ System.out.println(
+ getPrettyIterationStatusInfo(currentIterationStatistic)
+ );
+ }
} else {
- System.out.println("ContainerBalancer is Not Running.");
+ System.out.println("Last iteration info:");
+ lastIterationStatistic = iterationsStatusInfoList.stream()
+ .filter(it -> !it.getIterationResult().isEmpty())
+ .reduce((first, second) -> second)
+ .orElse(null);
+ if (lastIterationStatistic == null) {
+ System.out.println("-");
+ System.out.println();
+ } else {
+ System.out.println(
+ getPrettyIterationStatusInfo(lastIterationStatistic)
+ );
+ }
+ }
+
+ if (verboseWithHistory) {
+ System.out.println("Completed iteration history:");
+ final int lastCompletedIterationNumber = lastIterationStatistic == null
+ ? -1
+ : lastIterationStatistic.getIterationNumber();
+ String history = iterationsStatusInfoList
+ .stream()
+ .filter(it -> !it.getIterationResult().isEmpty())
+ .filter(it -> isRunning || it.getIterationNumber() !=
lastCompletedIterationNumber)
+ .map(this::getPrettyIterationStatusInfo)
+ .collect(Collectors.joining(System.lineSeparator()));
+ if (history.isEmpty()) {
+ System.out.println("-");
+ } else {
+ System.out.println(history);
+ }
+ System.out.println();
+ }
+ }
+
+ private void printStopReasonAndMessage(ContainerBalancerStatusInfoProto
balancerStatusInfo) {
+ if (balancerStatusInfo.hasStopReason()) {
+ System.out.printf("Stop reason: %s%n",
balancerStatusInfo.getStopReason());
+ }
+ if (balancerStatusInfo.hasStopMessage()) {
+ System.out.printf("Message: %s%n", balancerStatusInfo.getStopMessage());
}
}
diff --git
a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java
b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java
index 86e9129dcfa..613ca37e586 100644
---
a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java
+++
b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java
@@ -85,6 +85,78 @@ class TestContainerBalancerSubCommand {
"Datanodes Specified to be Balanced None\n" +
"Datanodes Excluded from Balancing None";
+ private static final String ITERATION_1_COMPLETED_OUTPUT =
+ "Key Value\n" +
+ "Iteration number 1\n" +
+ "Iteration duration 6m 40s\n" +
+ "Iteration result
ITERATION_COMPLETED\n" +
+ "Size scheduled to move 54 GB\n" +
+ "Moved data size 54 GB\n" +
+ "Scheduled to move containers 11\n" +
+ "Already moved containers 11\n" +
+ "Failed to move containers 0\n" +
+ "Failed to move containers by timeout 0\n" +
+ "Entered data to nodes \n" +
+ "80f6bc27-e6f3-493e-b1f4-25f810ad960d <- 28 GB\n" +
+ "701ca98e-aa1a-4b36-b817-e28ed634bba6 <- 26 GB\n" +
+ "Exited data from nodes \n" +
+ "b8b9c511-c30f-4933-8938-2f272e307070 -> 25 GB\n" +
+ "7bd99815-47e7-4015-bc61-ca6ef6dfd130 -> 29 GB";
+
+ private static final String ITERATION_2_COMPLETED_OUTPUT =
+ "Key Value\n" +
+ "Iteration number 2\n" +
+ "Iteration duration 5m 0s\n" +
+ "Iteration result
ITERATION_COMPLETED\n" +
+ "Size scheduled to move 30 GB\n" +
+ "Moved data size 30 GB\n" +
+ "Scheduled to move containers 8\n" +
+ "Already moved containers 8\n" +
+ "Failed to move containers 0\n" +
+ "Failed to move containers by timeout 0\n" +
+ "Entered data to nodes \n" +
+ "80f6bc27-e6f3-493e-b1f4-25f810ad960d <- 20 GB\n" +
+ "701ca98e-aa1a-4b36-b817-e28ed634bba6 <- 10 GB\n" +
+ "Exited data from nodes \n" +
+ "b8b9c511-c30f-4933-8938-2f272e307070 -> 15 GB\n" +
+ "7bd99815-47e7-4015-bc61-ca6ef6dfd130 -> 15 GB";
+
+ private static final String ITERATION_3_INTERRUPTED_OUTPUT =
+ "Key Value\n" +
+ "Iteration number 3\n" +
+ "Iteration duration 6m 10s\n" +
+ "Iteration result
ITERATION_INTERRUPTED\n" +
+ "Size scheduled to move 48 GB\n" +
+ "Moved data size 48 GB\n" +
+ "Scheduled to move containers 5\n" +
+ "Already moved containers 5\n" +
+ "Failed to move containers 0\n" +
+ "Failed to move containers by timeout 0\n" +
+ "Entered data to nodes \n" +
+ "80f6bc27-e6f3-493e-b1f4-25f810ad960d <- 20 GB\n" +
+ "701ca98e-aa1a-4b36-b817-e28ed634bba6 <- 28 GB\n" +
+ "Exited data from nodes \n" +
+ "b8b9c511-c30f-4933-8938-2f272e307070 -> 30 GB\n" +
+ "7bd99815-47e7-4015-bc61-ca6ef6dfd130 -> 18 GB";
+
+ private static final String ITERATION_3_COMPLETED_OUTPUT =
+ "Key Value\n" +
+ "Iteration number 3\n" +
+ "Iteration duration 6m 10s\n" +
+ "Iteration result
ITERATION_COMPLETED\n" +
+ "Size scheduled to move 48 GB\n" +
+ "Moved data size 48 GB\n" +
+ "Scheduled to move containers 5\n" +
+ "Already moved containers 5\n" +
+ "Failed to move containers 0\n" +
+ "Failed to move containers by timeout 0\n" +
+ "Entered data to nodes \n" +
+ "80f6bc27-e6f3-493e-b1f4-25f810ad960d <- 20 GB\n" +
+ "701ca98e-aa1a-4b36-b817-e28ed634bba6 <- 28 GB\n" +
+ "Exited data from nodes \n" +
+ "b8b9c511-c30f-4933-8938-2f272e307070 -> 30 GB\n" +
+ "7bd99815-47e7-4015-bc61-ca6ef6dfd130 -> 18 GB";
+
private ContainerBalancerStopSubcommand stopCmd;
private ContainerBalancerStartSubcommand startCmd;
private ContainerBalancerStatusSubcommand statusCmd;
@@ -92,6 +164,17 @@ class TestContainerBalancerSubCommand {
private GenericTestUtils.PrintStreamCapturer err;
private AtomicBoolean verbose;
+ private static final Pattern STOP_REASON = Pattern.compile(
+ "^Stop reason: USER_REQUESTED$", Pattern.MULTILINE);
+ private static final Pattern STOP_MESSAGE = Pattern.compile(
+ "^Message: Stopped by user request\\.$", Pattern.MULTILINE);
+ private static final Pattern COMPLETED_ALL_ITERATIONS_STOP_REASON =
Pattern.compile(
+ "^Stop reason: COMPLETED_ALL_ITERATIONS$", Pattern.MULTILINE);
+ private static final Pattern COMPLETED_ALL_ITERATIONS_STOP_MESSAGE =
Pattern.compile(
+ "^Message: Completed all configured number of iterations\\.$",
Pattern.MULTILINE);
+ private static final Pattern STOPPED_AT = Pattern.compile(
+ "^Stopped at: (\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2})$",
Pattern.MULTILINE);
+
private static ContainerBalancerStatusInfoResponseProto
getContainerBalancerStatusInfoResponseProto(
ContainerBalancerConfiguration config) {
StorageContainerLocationProtocolProtos.ContainerBalancerTaskIterationStatusInfoProto
iteration1StatusInfo =
@@ -234,6 +317,47 @@ private static ContainerBalancerConfiguration
getContainerBalancerConfiguration(
return config;
}
+ /**
+ * Builds a stopped-balancer response.
+ *
+ * @param config configuration
+ * @param stopReason stop reason
+ * @param stopMessage stop message
+ * @param lastIterationResult result for iteration 3, e.g.
ITERATION_INTERRUPTED or ITERATION_COMPLETED
+ * @param balancingDurationSeconds wall-clock duration between startedAt and
stoppedAt
+ */
+ private static ContainerBalancerStatusInfoResponseProto
getStoppedStatusInfoResponseProto(
+ ContainerBalancerConfiguration config, String stopReason, String
stopMessage,
+ String lastIterationResult, long balancingDurationSeconds) {
+ ContainerBalancerStatusInfoProto runningInfo =
+
getContainerBalancerStatusInfoResponseProto(config).getContainerBalancerStatusInfo();
+
+
StorageContainerLocationProtocolProtos.ContainerBalancerTaskIterationStatusInfoProto
iteration3 =
+ runningInfo.getIterationsStatusInfo(2).toBuilder()
+ .setIterationResult(lastIterationResult)
+ .build();
+
+ long stoppedAt = OffsetDateTime.now().toEpochSecond();
+ long startedAt = stoppedAt - balancingDurationSeconds;
+
+ ContainerBalancerStatusInfoProto stoppedInfo = runningInfo.toBuilder()
+ .setStartedAt(startedAt)
+ .setStoppedAt(stoppedAt)
+ .setStopReason(stopReason)
+ .setStopMessage(stopMessage)
+ .setConfiguration(config.toProtobufBuilder().setShouldRun(false))
+ .clearIterationsStatusInfo()
+ .addIterationsStatusInfo(runningInfo.getIterationsStatusInfo(0))
+ .addIterationsStatusInfo(runningInfo.getIterationsStatusInfo(1))
+ .addIterationsStatusInfo(iteration3)
+ .build();
+
+ return ContainerBalancerStatusInfoResponseProto.newBuilder()
+ .setIsRunning(false)
+ .setContainerBalancerStatusInfo(stoppedInfo)
+ .build();
+ }
+
@BeforeEach
void setup() {
verbose = new AtomicBoolean();
@@ -290,7 +414,7 @@ void
testContainerBalancerStatusInfoSubcommandRunningWithoutFlags()
assertThat(out.get()).containsPattern(IS_RUNNING)
.doesNotContain(BALANCER_CONFIG_OUTPUT)
.doesNotContain(currentIterationOutput)
- .doesNotContain("Iteration history list:");
+ .doesNotContain("Completed iteration history:");
}
@Test
@@ -351,7 +475,7 @@ void
testContainerBalancerStatusInfoSubcommandVerboseHistory()
.containsPattern(STARTED_AT)
.containsPattern(DURATION)
.contains(BALANCER_CONFIG_OUTPUT)
- .contains("Iteration history list:")
+ .contains("Completed iteration history:")
.contains(firstHistoryIterationOutput)
.contains(secondHistoryIterationOutput);
}
@@ -396,7 +520,7 @@ void testContainerBalancerStatusInfoSubcommandVerbose()
.containsPattern(DURATION)
.contains(BALANCER_CONFIG_OUTPUT)
.contains(currentIterationOutput)
- .doesNotContain("Iteration history list:");
+ .doesNotContain("Completed iteration history:");
}
@Test
@@ -476,4 +600,116 @@ public void
testContainerBalancerStartSubcommandWhenBalancerIsRunning()
assertThat(err.get()).containsPattern(FAILED_TO_START);
}
+ @Test
+ void
testContainerBalancerStatusSubcommandStoppedWithoutFlagsShowsStopReason()
throws IOException {
+ ScmClient scmClient = mock(ScmClient.class);
+ ContainerBalancerConfiguration config =
getContainerBalancerConfiguration();
+ when(scmClient.getContainerBalancerStatusInfo())
+ .thenReturn(getStoppedStatusInfoResponseProto(
+ config, "USER_REQUESTED", "Stopped by user request.",
+ "ITERATION_INTERRUPTED", 1070L));
+ statusCmd.execute(scmClient);
+ assertThat(out.get())
+ .containsPattern(IS_NOT_RUNNING)
+ .containsPattern(STOP_REASON)
+ .containsPattern(STOP_MESSAGE)
+ .doesNotContain(BALANCER_CONFIG_OUTPUT)
+ .doesNotContain("Last iteration info:")
+ .doesNotContain("Stopped at:")
+ .doesNotContain("Completed iteration history:");
+ }
+
+ @Test
+ void testContainerBalancerStatusSubcommandStoppedVerbose() throws
IOException {
+ ScmClient scmClient = mock(ScmClient.class);
+ ContainerBalancerConfiguration config =
getContainerBalancerConfiguration();
+ when(scmClient.getContainerBalancerStatusInfo())
+ .thenReturn(getStoppedStatusInfoResponseProto(
+ config, "USER_REQUESTED", "Stopped by user request.",
+ "ITERATION_INTERRUPTED", 1070L));
+ verbose.set(true);
+ statusCmd.execute(scmClient);
+
+ assertThat(out.get())
+ .containsPattern(IS_NOT_RUNNING)
+ .containsPattern(STOP_REASON)
+ .containsPattern(STOP_MESSAGE)
+ .containsPattern(STARTED_AT)
+ .containsPattern(STOPPED_AT)
+ .contains(BALANCER_CONFIG_OUTPUT)
+ .contains("Last iteration info:")
+ .contains(ITERATION_3_INTERRUPTED_OUTPUT)
+ .doesNotContain("Current iteration info:")
+ .doesNotContain("Completed iteration history:");
+ }
+
+ @Test
+ void testContainerBalancerStatusSubcommandStoppedVerboseWithHistory() throws
IOException {
+ ScmClient scmClient = mock(ScmClient.class);
+ ContainerBalancerConfiguration config =
getContainerBalancerConfiguration();
+ when(scmClient.getContainerBalancerStatusInfo())
+ .thenReturn(getStoppedStatusInfoResponseProto(
+ config, "USER_REQUESTED", "Stopped by user request.",
+ "ITERATION_INTERRUPTED", 1070L));
+ CommandLine cmd = new CommandLine(statusCmd);
+ verbose.set(true);
+ cmd.parseArgs("--history");
+ statusCmd.execute(scmClient);
+
+ String output = out.get();
+ int lastIterationStart = output.indexOf("Last iteration info:");
+ int historyStart = output.indexOf("Completed iteration history:");
+ String lastIterationSection = output.substring(lastIterationStart,
historyStart);
+ String historySection = output.substring(historyStart);
+
+ assertThat(output)
+ .containsPattern(IS_NOT_RUNNING)
+ .containsPattern(STOP_REASON)
+ .containsPattern(STOP_MESSAGE)
+ .containsPattern(STARTED_AT)
+ .containsPattern(STOPPED_AT)
+ .contains(BALANCER_CONFIG_OUTPUT)
+ .doesNotContain("Current iteration info:");
+ assertThat(lastIterationSection).contains(ITERATION_3_INTERRUPTED_OUTPUT);
+ assertThat(historySection)
+ .contains(ITERATION_1_COMPLETED_OUTPUT)
+ .contains(ITERATION_2_COMPLETED_OUTPUT)
+ .doesNotContain(ITERATION_3_INTERRUPTED_OUTPUT);
+ }
+
+ @Test
+ void
testContainerBalancerStatusSubcommandStoppedAfterAllIterationsCompleteVerboseWithHistory()
+ throws IOException {
+ ScmClient scmClient = mock(ScmClient.class);
+ ContainerBalancerConfiguration config =
getContainerBalancerConfiguration();
+ when(scmClient.getContainerBalancerStatusInfo())
+ .thenReturn(getStoppedStatusInfoResponseProto(config,
"COMPLETED_ALL_ITERATIONS",
+ "Completed all configured number of iterations.",
"ITERATION_COMPLETED",
+ 1070L));
+
+ CommandLine cmd = new CommandLine(statusCmd);
+ verbose.set(true);
+ cmd.parseArgs("--history");
+ statusCmd.execute(scmClient);
+
+ String output = out.get();
+ int lastIterationStart = output.indexOf("Last iteration info:");
+ int historyStart = output.indexOf("Completed iteration history:");
+ String lastIterationSection = output.substring(lastIterationStart,
historyStart);
+ String historySection = output.substring(historyStart);
+
+ assertThat(output)
+ .containsPattern(IS_NOT_RUNNING)
+ .containsPattern(COMPLETED_ALL_ITERATIONS_STOP_REASON)
+ .containsPattern(COMPLETED_ALL_ITERATIONS_STOP_MESSAGE)
+ .containsPattern(STARTED_AT)
+ .containsPattern(STOPPED_AT)
+ .contains(BALANCER_CONFIG_OUTPUT)
+ .doesNotContain("Current iteration info:");
+ assertThat(lastIterationSection).contains(ITERATION_3_COMPLETED_OUTPUT);
+ assertThat(historySection)
+ .contains(ITERATION_1_COMPLETED_OUTPUT)
+ .contains(ITERATION_2_COMPLETED_OUTPUT)
+ .doesNotContain(ITERATION_3_COMPLETED_OUTPUT);
+ }
}
diff --git a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot
b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot
index 4dc9ce74106..3419ac2a9f4 100644
--- a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot
@@ -90,7 +90,7 @@ Verify Balancer Iteration
Verify Balancer Iteration History
[arguments] ${output}
- Should Contain ${output} Iteration history
list:
+ Should Contain ${output} Completed iteration
history:
Should Contain X Times ${output} Size scheduled to
move 1 collapse_spaces=True
Should Contain X Times ${output} Moved data size
1 collapse_spaces=True
Should Contain X Times ${output} Scheduled to move
containers 1 collapse_spaces=True
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
