This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 372d3cd7b4 [Test][E2E] Isolate the RocketMqIT sink topics and wait for
row visibility before reading (#12701)
372d3cd7b4 is described below
commit 372d3cd7b4d7715eb6fc811c1b43e23cb6309c4d
Author: Sepuri Sai Krishna <[email protected]>
AuthorDate: Sun Oct 11 13:02:01 2026 +0000
[Test][E2E] Isolate the RocketMqIT sink topics and wait for row visibility
before reading (#12701)
---
.../e2e/connector/rocketmq/RocketMqIT.java | 70 ++++++++++++++++++----
.../resources/rocketmq-sink_fake_to_rocketmq.conf | 2 +-
...rocketmq-sink_fake_to_rocketmq_message_tag.conf | 2 +-
.../rocketmq-text-sink_fake_to_rocketmq.conf | 2 +-
4 files changed, 61 insertions(+), 15 deletions(-)
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rocketmq-e2e/src/test/java/org/apache/seatunnel/e2e/connector/rocketmq/RocketMqIT.java
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rocketmq-e2e/src/test/java/org/apache/seatunnel/e2e/connector/rocketmq/RocketMqIT.java
index 7fc989a005..f987f9bc2d 100644
---
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rocketmq-e2e/src/test/java/org/apache/seatunnel/e2e/connector/rocketmq/RocketMqIT.java
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rocketmq-e2e/src/test/java/org/apache/seatunnel/e2e/connector/rocketmq/RocketMqIT.java
@@ -92,6 +92,16 @@ import java.util.concurrent.atomic.AtomicLong;
@Slf4j
public class RocketMqIT extends TestSuiteBase implements TestResource {
+ /**
+ * Rows each of the three fake-source sink jobs writes, asserted after the
sink completes.
+ *
+ * <p>Must stay in sync with {@code row.num = 10} in
rocketmq-sink_fake_to_rocketmq.conf,
+ * rocketmq-text-sink_fake_to_rocketmq.conf and
rocketmq-sink_fake_to_rocketmq_message_tag.conf.
+ * The visibility wait and the count assertion both read it, so changing
one without the other
+ * would make the wait pass early or never finish.
+ */
+ private static final int EXPECTED_SINK_ROWS = 10;
+
private static final String IMAGE = "apache/rocketmq:4.9.4";
private static final String ROCKETMQ_GROUP = "SeaTunnel-rocketmq-group";
private static final String HOST = "rocketmq-e2e";
@@ -185,33 +195,42 @@ public class RocketMqIT extends TestSuiteBase implements
TestResource {
@TestTemplate
public void testSinkRocketMq(TestContainer container) throws IOException,
InterruptedException {
- waitForTopicRoute("test_topic");
+ final String topicName = "test_topic_" + uniqueTestSuffix();
+ waitForTopicRoute(topicName);
Container.ExecResult execResult =
- container.executeJob("/rocketmq-sink_fake_to_rocketmq.conf");
+ container.executeJob(
+ "/rocketmq-sink_fake_to_rocketmq.conf",
+ Arrays.asList("sinkTopic=" + topicName));
Assertions.assertEquals(0, execResult.getExitCode(),
execResult.getStderr());
- String topicName = "test_topic";
+ awaitSinkRowsVisible(topicName);
+
Map<String, RocketMqConsumerMessage> data =
getRocketMqConsumerData(topicName);
ObjectMapper objectMapper = new ObjectMapper();
String key = data.keySet().iterator().next();
ObjectNode objectNode = objectMapper.readValue(key, ObjectNode.class);
Assertions.assertTrue(objectNode.has("c_map"));
Assertions.assertTrue(objectNode.has("c_string"));
- Assertions.assertEquals(10, data.size());
+ Assertions.assertEquals(EXPECTED_SINK_ROWS, data.size());
}
@TestTemplate
public void testTextFormatSinkRocketMq(TestContainer container)
throws IOException, InterruptedException {
- waitForTopicRoute("test_text_topic");
+ final String topicName = "test_text_topic_" + uniqueTestSuffix();
+ waitForTopicRoute(topicName);
Container.ExecResult execResult =
-
container.executeJob("/rocketmq-text-sink_fake_to_rocketmq.conf");
+ container.executeJob(
+ "/rocketmq-text-sink_fake_to_rocketmq.conf",
+ Arrays.asList("sinkTopic=" + topicName));
Assertions.assertEquals(0, execResult.getExitCode(),
execResult.getStderr());
- String topicName = "test_text_topic";
+
+ awaitSinkRowsVisible(topicName);
+
Map<String, RocketMqConsumerMessage> data =
getRocketMqConsumerData(topicName);
- Assertions.assertEquals(10, data.size());
+ Assertions.assertEquals(EXPECTED_SINK_ROWS, data.size());
}
@TestTemplate
@@ -410,20 +429,31 @@ public class RocketMqIT extends TestSuiteBase implements
TestResource {
@TestTemplate
public void testSinkRocketMqMessageTag(TestContainer container)
throws IOException, InterruptedException {
+ final String topicName = "test_topic_message_tag_" +
uniqueTestSuffix();
+ // Warm the route before the job, not after. With the previous fixed
topic a later
+ // template invocation always found the topic already created; a
per-invocation topic
+ // does not exist yet, so the sink would otherwise depend on broker
auto-creation
+ // racing the name server, which is the "No topic route info in name
server" failure
+ // this file has hit before.
+ waitForTopicRoute(topicName);
+
Container.ExecResult execResult =
-
container.executeJob("/rocketmq-sink_fake_to_rocketmq_message_tag.conf");
+ container.executeJob(
+ "/rocketmq-sink_fake_to_rocketmq_message_tag.conf",
+ Arrays.asList("sinkTopic=" + topicName));
Assertions.assertEquals(0, execResult.getExitCode(),
execResult.getStderr());
- String topicName = "test_topic_message_tag";
String tag = "test_tag";
- waitForTopicRoute(topicName);
+
+ awaitSinkRowsVisible(topicName);
+
Map<String, RocketMqConsumerMessage> data =
getRocketMqConsumerData(topicName);
ObjectMapper objectMapper = new ObjectMapper();
String key = data.keySet().iterator().next();
ObjectNode objectNode = objectMapper.readValue(key, ObjectNode.class);
Assertions.assertTrue(objectNode.has("c_map"));
Assertions.assertTrue(objectNode.has("c_string"));
- Assertions.assertEquals(10, data.size());
+ Assertions.assertEquals(EXPECTED_SINK_ROWS, data.size());
Assertions.assertEquals(tag, data.get(key).getTag());
}
@@ -921,6 +951,22 @@ public class RocketMqIT extends TestSuiteBase implements
TestResource {
return new ArrayList<>(bodyByPosition.values());
}
+ /**
+ * Waits until every row a sink job wrote is visible to an admin offset
read, before the single
+ * consuming read that follows.
+ *
+ * <p>Each sink test uses a topic of its own, so every queue starts at
offset 0 and the summed
+ * max offset is exactly the number of rows stored. Without this wait the
read can poll before
+ * the last rows are readable and the count assertion fails low. Retrying
the read instead is
+ * not an option, because {@link #getRocketMqConsumerData(String)} commits
offsets as it reads,
+ * so a second call would resume past what the first consumed.
+ *
+ * @param topicName the per-invocation topic the job just wrote to
+ */
+ private void awaitSinkRowsVisible(String topicName) {
+ awaitTopicMaxOffset(topicName, EXPECTED_SINK_ROWS,
Duration.ofMinutes(1));
+ }
+
/**
* Waits for RocketMQ admin offset visibility and returns the successful
observed offset.
*
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rocketmq-e2e/src/test/resources/rocketmq-sink_fake_to_rocketmq.conf
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rocketmq-e2e/src/test/resources/rocketmq-sink_fake_to_rocketmq.conf
index 91addf77e3..0850e3bcc5 100644
---
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rocketmq-e2e/src/test/resources/rocketmq-sink_fake_to_rocketmq.conf
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rocketmq-e2e/src/test/resources/rocketmq-sink_fake_to_rocketmq.conf
@@ -62,7 +62,7 @@ transform {
sink {
Rocketmq {
name.srv.addr = "rocketmq-e2e:9876"
- topic = "test_topic"
+ topic = "${sinkTopic:test_topic}"
partition.key.fields = ["c_map", "c_string"]
producer.send.sync = true
}
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rocketmq-e2e/src/test/resources/rocketmq-sink_fake_to_rocketmq_message_tag.conf
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rocketmq-e2e/src/test/resources/rocketmq-sink_fake_to_rocketmq_message_tag.conf
index 0badca6bf7..9904d21a3e 100644
---
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rocketmq-e2e/src/test/resources/rocketmq-sink_fake_to_rocketmq_message_tag.conf
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rocketmq-e2e/src/test/resources/rocketmq-sink_fake_to_rocketmq_message_tag.conf
@@ -62,7 +62,7 @@ transform {
sink {
Rocketmq {
name.srv.addr = "rocketmq-e2e:9876"
- topic = "test_topic_message_tag"
+ topic = "${sinkTopic:test_topic_message_tag}"
partition.key.fields = ["c_map", "c_string"]
producer.send.sync = true
tag = "test_tag"
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rocketmq-e2e/src/test/resources/rocketmq-text-sink_fake_to_rocketmq.conf
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rocketmq-e2e/src/test/resources/rocketmq-text-sink_fake_to_rocketmq.conf
index ef8b325db8..bffdf9b158 100644
---
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rocketmq-e2e/src/test/resources/rocketmq-text-sink_fake_to_rocketmq.conf
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rocketmq-e2e/src/test/resources/rocketmq-text-sink_fake_to_rocketmq.conf
@@ -62,7 +62,7 @@ transform {
sink {
Rocketmq {
name.srv.addr = "rocketmq-e2e:9876"
- topic = "test_text_topic"
+ topic = "${sinkTopic:test_text_topic}"
format = text
partition.key.fields = ["c_map", "c_string"]
producer.send.sync = true
