This is an automated email from the ASF dual-hosted git repository.
nanda pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new a3c9c0e040 HDDS-12443. Intermittent failure in
TestContainerBalancerSubCommand (#7989)
a3c9c0e040 is described below
commit a3c9c0e04021faf494a0b714e7b747e192112b09
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Sun Mar 2 18:10:04 2025 +0100
HDDS-12443. Intermittent failure in TestContainerBalancerSubCommand (#7989)
---
hadoop-hdds/tools/pom.xml | 5 +
.../scm/cli/ContainerBalancerStatusSubcommand.java | 5 +-
.../datanode/TestContainerBalancerSubCommand.java | 210 +++++++--------------
3 files changed, 78 insertions(+), 142 deletions(-)
diff --git a/hadoop-hdds/tools/pom.xml b/hadoop-hdds/tools/pom.xml
index 071e972f66..7f37a36b65 100644
--- a/hadoop-hdds/tools/pom.xml
+++ b/hadoop-hdds/tools/pom.xml
@@ -140,6 +140,11 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.ozone</groupId>
+ <artifactId>hdds-test-utils</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
diff --git
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStatusSubcommand.java
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStatusSubcommand.java
index 8abc7a2dd5..7edecd7822 100644
---
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStatusSubcommand.java
+++
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStatusSubcommand.java
@@ -26,6 +26,7 @@
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
@@ -69,7 +70,9 @@ public void execute(ScmClient scmClient) throws IOException {
System.out.println("ContainerBalancer is Running.");
if (verbose) {
- System.out.printf("Started at: %s %s%n", dateTime.toLocalDate(),
dateTime.toLocalTime());
+ System.out.printf("Started at: %s %s%n",
+ dateTime.toLocalDate().format(DateTimeFormatter.ISO_LOCAL_DATE),
+ dateTime.toLocalTime().format(DateTimeFormatter.ISO_LOCAL_TIME));
Duration balancingDuration = Duration.between(startedAtInstant,
OffsetDateTime.now());
System.out.printf("Balancing duration: %s%n%n",
getPrettyDuration(balancingDuration));
System.out.println(getConfigurationPrettyString(balancerStatusInfo.getConfiguration()));
diff --git
a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java
b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java
index 0594ae38ab..30919d525f 100644
---
a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java
+++
b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java
@@ -18,19 +18,13 @@
package org.apache.hadoop.hdds.scm.cli.datanode;
import static org.apache.hadoop.ozone.OzoneConsts.GB;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import java.io.ByteArrayOutputStream;
import java.io.IOException;
-import java.io.PrintStream;
-import java.io.UnsupportedEncodingException;
-import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.util.Arrays;
-import java.util.regex.Matcher;
import java.util.regex.Pattern;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ContainerBalancerStatusInfoProto;
@@ -40,6 +34,8 @@
import org.apache.hadoop.hdds.scm.cli.ContainerBalancerStopSubcommand;
import org.apache.hadoop.hdds.scm.client.ScmClient;
import
org.apache.hadoop.hdds.scm.container.balancer.ContainerBalancerConfiguration;
+import org.apache.hadoop.hdds.utils.IOUtils;
+import org.apache.ozone.test.GenericTestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -51,14 +47,44 @@
*/
class TestContainerBalancerSubCommand {
- private static final String DEFAULT_ENCODING = StandardCharsets.UTF_8.name();
- private final ByteArrayOutputStream outContent = new ByteArrayOutputStream();
- private final ByteArrayOutputStream errContent = new ByteArrayOutputStream();
- private final PrintStream originalOut = System.out;
- private final PrintStream originalErr = System.err;
+ private static final Pattern DURATION = Pattern.compile(
+ "^Balancing duration: \\d{1}s$", Pattern.MULTILINE);
+ private static final Pattern FAILED_TO_START = Pattern.compile(
+ "^Failed\\sto\\sstart\\sContainer\\sBalancer.");
+ private static final Pattern IS_NOT_RUNNING = Pattern.compile(
+ "^ContainerBalancer\\sis\\sNot\\sRunning.");
+ private static final Pattern IS_RUNNING = Pattern.compile(
+ "^ContainerBalancer\\sis\\sRunning.$", Pattern.MULTILINE);
+ private static final Pattern STARTED_AT = Pattern.compile(
+ "^Started at: (\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2})$",
Pattern.MULTILINE);
+ private static final Pattern STARTED_SUCCESSFULLY = Pattern.compile(
+ "^Container\\sBalancer\\sstarted\\ssuccessfully.");
+ private static final Pattern WAITING_TO_STOP = Pattern.compile(
+
"^Sending\\sstop\\scommand.\\sWaiting\\sfor\\sContainer\\sBalancer\\sto\\sstop...\\n"
+
+ "Container\\sBalancer\\sstopped.");
+
+ private static final String BALANCER_CONFIG_OUTPUT = "Container Balancer
Configuration values:\n" +
+ "Key Value\n" +
+ "Threshold 10.0\n" +
+ "Max Datanodes to Involve per Iteration(percent) 20\n" +
+ "Max Size to Move per Iteration 0GB\n" +
+ "Max Size Entering Target per Iteration 26GB\n" +
+ "Max Size Leaving Source per Iteration 26GB\n" +
+ "Number of Iterations 3\n" +
+ "Time Limit for Single Container's Movement 65min\n" +
+ "Time Limit for Single Container's Replication 50min\n" +
+ "Interval between each Iteration 0min\n" +
+ "Whether to Enable Network Topology false\n" +
+ "Whether to Trigger Refresh Datanode Usage Info false\n" +
+ "Container IDs to Exclude from Balancing None\n" +
+ "Datanodes Specified to be Balanced None\n" +
+ "Datanodes Excluded from Balancing None";
+
private ContainerBalancerStopSubcommand stopCmd;
private ContainerBalancerStartSubcommand startCmd;
private ContainerBalancerStatusSubcommand statusCmd;
+ private GenericTestUtils.PrintStreamCapturer out;
+ private GenericTestUtils.PrintStreamCapturer err;
private static ContainerBalancerStatusInfoResponseProto
getContainerBalancerStatusInfoResponseProto(
ContainerBalancerConfiguration config) {
@@ -203,18 +229,17 @@ private static ContainerBalancerConfiguration
getContainerBalancerConfiguration(
}
@BeforeEach
- public void setup() throws UnsupportedEncodingException {
+ void setup() {
stopCmd = new ContainerBalancerStopSubcommand();
startCmd = new ContainerBalancerStartSubcommand();
statusCmd = new ContainerBalancerStatusSubcommand();
- System.setOut(new PrintStream(outContent, false, DEFAULT_ENCODING));
- System.setErr(new PrintStream(errContent, false, DEFAULT_ENCODING));
+ out = GenericTestUtils.captureOut();
+ err = GenericTestUtils.captureErr();
}
@AfterEach
- public void tearDown() {
- System.setOut(originalOut);
- System.setErr(originalErr);
+ void tearDown() {
+ IOUtils.closeQuietly(out, err);
}
@Test
@@ -230,30 +255,6 @@ void
testContainerBalancerStatusInfoSubcommandRunningWithoutFlags()
//test status is running
when(scmClient.getContainerBalancerStatusInfo()).thenReturn(statusInfoResponseProto);
statusCmd.execute(scmClient);
- Pattern p = Pattern.compile(
- "^ContainerBalancer\\sis\\sRunning.");
- String output = outContent.toString(DEFAULT_ENCODING);
- Matcher m = p.matcher(output);
- assertTrue(m.find());
-
- String balancerConfigOutput =
- "Container Balancer Configuration values:\n" +
- "Key Value\n" +
- "Threshold 10.0\n" +
- "Max Datanodes to Involve per Iteration(percent) 20\n" +
- "Max Size to Move per Iteration 0GB\n" +
- "Max Size Entering Target per Iteration 26GB\n" +
- "Max Size Leaving Source per Iteration 26GB\n" +
- "Number of Iterations 3\n" +
- "Time Limit for Single Container's Movement 65min\n" +
- "Time Limit for Single Container's Replication 50min\n" +
- "Interval between each Iteration 0min\n" +
- "Whether to Enable Network Topology false\n" +
- "Whether to Trigger Refresh Datanode Usage Info false\n" +
- "Container IDs to Exclude from Balancing None\n" +
- "Datanodes Specified to be Balanced None\n" +
- "Datanodes Excluded from Balancing None";
- assertFalse(output.contains(balancerConfigOutput));
String currentIterationOutput =
"Current iteration info:\n" +
@@ -273,9 +274,11 @@ void
testContainerBalancerStatusInfoSubcommandRunningWithoutFlags()
"Exited data from nodes \n" +
"b8b9c511-c30f-4933-8938-2f272e307070 -> 30 GB\n" +
"7bd99815-47e7-4015-bc61-ca6ef6dfd130 -> 18 GB";
- assertFalse(output.contains(currentIterationOutput));
- assertFalse(output.contains("Iteration history list:"));
+ assertThat(out.get()).containsPattern(IS_RUNNING)
+ .doesNotContain(BALANCER_CONFIG_OUTPUT)
+ .doesNotContain(currentIterationOutput)
+ .doesNotContain("Iteration history list:");
}
@Test
@@ -293,42 +296,7 @@ void
testContainerBalancerStatusInfoSubcommandVerboseHistory()
CommandLine c = new CommandLine(statusCmd);
c.parseArgs("--verbose", "--history");
statusCmd.execute(scmClient);
- String output = outContent.toString(DEFAULT_ENCODING);
- Pattern p = Pattern.compile(
- "^ContainerBalancer\\sis\\sRunning.$", Pattern.MULTILINE);
- Matcher m = p.matcher(output);
- assertTrue(m.find());
-
- p = Pattern.compile(
- "^Started at: (\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2})$",
Pattern.MULTILINE);
- m = p.matcher(output);
- assertTrue(m.find());
-
- p = Pattern.compile(
- "^Balancing duration: \\d{1}s$", Pattern.MULTILINE);
- m = p.matcher(output);
- assertTrue(m.find());
-
- String balancerConfigOutput =
- "Container Balancer Configuration values:\n" +
- "Key Value\n" +
- "Threshold 10.0\n" +
- "Max Datanodes to Involve per Iteration(percent) 20\n" +
- "Max Size to Move per Iteration 0GB\n" +
- "Max Size Entering Target per Iteration 26GB\n" +
- "Max Size Leaving Source per Iteration 26GB\n" +
- "Number of Iterations 3\n" +
- "Time Limit for Single Container's Movement 65min\n" +
- "Time Limit for Single Container's Replication 50min\n" +
- "Interval between each Iteration 0min\n" +
- "Whether to Enable Network Topology false\n" +
- "Whether to Trigger Refresh Datanode Usage Info false\n" +
- "Container IDs to Exclude from Balancing None\n" +
- "Datanodes Specified to be Balanced None\n" +
- "Datanodes Excluded from Balancing None";
- assertTrue(output.contains(balancerConfigOutput));
-
- assertTrue(output.contains("Iteration history list:"));
+
String firstHistoryIterationOutput =
"Key Value\n" +
"Iteration number 3\n" +
@@ -346,7 +314,6 @@ void
testContainerBalancerStatusInfoSubcommandVerboseHistory()
"Exited data from nodes \n" +
"b8b9c511-c30f-4933-8938-2f272e307070 -> 30 GB\n" +
"7bd99815-47e7-4015-bc61-ca6ef6dfd130 -> 18 GB";
- assertTrue(output.contains(firstHistoryIterationOutput));
String secondHistoryIterationOutput =
"Key Value\n" +
@@ -365,7 +332,15 @@ void
testContainerBalancerStatusInfoSubcommandVerboseHistory()
"Exited data from nodes \n" +
"b8b9c511-c30f-4933-8938-2f272e307070 -> 15 GB\n" +
"7bd99815-47e7-4015-bc61-ca6ef6dfd130 -> 15 GB";
- assertTrue(output.contains(secondHistoryIterationOutput));
+
+ assertThat(out.get())
+ .containsPattern(IS_RUNNING)
+ .containsPattern(STARTED_AT)
+ .containsPattern(DURATION)
+ .contains(BALANCER_CONFIG_OUTPUT)
+ .contains("Iteration history list:")
+ .contains(firstHistoryIterationOutput)
+ .contains(secondHistoryIterationOutput);
}
@Test
@@ -383,40 +358,6 @@ void testContainerBalancerStatusInfoSubcommandVerbose()
CommandLine c = new CommandLine(statusCmd);
c.parseArgs("--verbose");
statusCmd.execute(scmClient);
- String output = outContent.toString(DEFAULT_ENCODING);
- Pattern p = Pattern.compile(
- "^ContainerBalancer\\sis\\sRunning.$", Pattern.MULTILINE);
- Matcher m = p.matcher(output);
- assertTrue(m.find());
-
- p = Pattern.compile(
- "^Started at: (\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2})$",
Pattern.MULTILINE);
- m = p.matcher(output);
- assertTrue(m.find());
-
- p = Pattern.compile(
- "^Balancing duration: \\d{1}s$", Pattern.MULTILINE);
- m = p.matcher(output);
- assertTrue(m.find());
-
- String balancerConfigOutput =
- "Container Balancer Configuration values:\n" +
- "Key Value\n" +
- "Threshold 10.0\n" +
- "Max Datanodes to Involve per Iteration(percent) 20\n" +
- "Max Size to Move per Iteration 0GB\n" +
- "Max Size Entering Target per Iteration 26GB\n" +
- "Max Size Leaving Source per Iteration 26GB\n" +
- "Number of Iterations 3\n" +
- "Time Limit for Single Container's Movement 65min\n" +
- "Time Limit for Single Container's Replication 50min\n" +
- "Interval between each Iteration 0min\n" +
- "Whether to Enable Network Topology false\n" +
- "Whether to Trigger Refresh Datanode Usage Info false\n" +
- "Container IDs to Exclude from Balancing None\n" +
- "Datanodes Specified to be Balanced None\n" +
- "Datanodes Excluded from Balancing None";
- assertTrue(output.contains(balancerConfigOutput));
String currentIterationOutput =
"Current iteration info:\n" +
@@ -436,9 +377,14 @@ void testContainerBalancerStatusInfoSubcommandVerbose()
"Exited data from nodes \n" +
"b8b9c511-c30f-4933-8938-2f272e307070 -> 30 GB\n" +
"7bd99815-47e7-4015-bc61-ca6ef6dfd130 -> 18 GB";
- assertTrue(output.contains(currentIterationOutput));
- assertFalse(output.contains("Iteration history list:"));
+ assertThat(out.get())
+ .containsPattern(IS_RUNNING)
+ .containsPattern(STARTED_AT)
+ .containsPattern(DURATION)
+ .contains(BALANCER_CONFIG_OUTPUT)
+ .contains(currentIterationOutput)
+ .doesNotContain("Iteration history list:");
}
@Test
@@ -453,10 +399,7 @@ void
testContainerBalancerStatusInfoSubcommandRunningOnStoppedBalancer()
.build());
statusCmd.execute(scmClient);
- Pattern p = Pattern.compile(
- "^ContainerBalancer\\sis\\sNot\\sRunning.");
- Matcher m = p.matcher(outContent.toString(DEFAULT_ENCODING));
- assertTrue(m.find());
+ assertThat(out.get()).containsPattern(IS_NOT_RUNNING);
}
@Test
@@ -471,10 +414,7 @@ void testContainerBalancerStatusSubcommandNotRunning()
statusCmd.execute(scmClient);
- Pattern p = Pattern.compile(
- "^ContainerBalancer\\sis\\sNot\\sRunning.");
- Matcher m = p.matcher(outContent.toString(DEFAULT_ENCODING));
- assertTrue(m.find());
+ assertThat(out.get()).containsPattern(IS_NOT_RUNNING);
}
@Test
@@ -482,12 +422,7 @@ public void testContainerBalancerStopSubcommand() throws
IOException {
ScmClient scmClient = mock(ScmClient.class);
stopCmd.execute(scmClient);
- Pattern p = Pattern.compile("^Sending\\sstop\\scommand." +
- "\\sWaiting\\sfor\\sContainer\\sBalancer\\sto\\sstop...\\n" +
- "Container\\sBalancer\\sstopped.");
-
- Matcher m = p.matcher(outContent.toString(DEFAULT_ENCODING));
- assertTrue(m.find());
+ assertThat(out.get()).containsPattern(WAITING_TO_STOP);
}
@Test
@@ -503,10 +438,7 @@ public void
testContainerBalancerStartSubcommandWhenBalancerIsNotRunning()
.build());
startCmd.execute(scmClient);
- Pattern p = Pattern.compile("^Container\\sBalancer\\sstarted" +
- "\\ssuccessfully.");
- Matcher m = p.matcher(outContent.toString(DEFAULT_ENCODING));
- assertTrue(m.find());
+ assertThat(out.get()).containsPattern(STARTED_SUCCESSFULLY);
}
@Test
@@ -522,11 +454,7 @@ public void
testContainerBalancerStartSubcommandWhenBalancerIsRunning()
.build());
startCmd.execute(scmClient);
- Pattern p = Pattern.compile("^Failed\\sto\\sstart\\sContainer" +
- "\\sBalancer.");
-
- Matcher m = p.matcher(outContent.toString(DEFAULT_ENCODING));
- assertTrue(m.find());
+ assertThat(out.get()).containsPattern(FAILED_TO_START);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
