This is an automated email from the ASF dual-hosted git repository.
NSAmelchev pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-extensions.git
The following commit(s) were added to refs/heads/master by this push:
new 3214a311 IGNITE-29149 Fix flaky testWithExpiryPolicy and metadata
consumer leak in Kafka to Ignite streamer (#365)
3214a311 is described below
commit 3214a311aa63ba37ad4d52b77e0ca3b375c7e8d7
Author: Nikita Amelchev <[email protected]>
AuthorDate: Wed Oct 28 12:43:07 2026 +0300
IGNITE-29149 Fix flaky testWithExpiryPolicy and metadata consumer leak in
Kafka to Ignite streamer (#365)
---
.../kafka/AbstractKafkaToIgniteCdcStreamer.java | 3 +++
.../cdc/kafka/KafkaToIgniteCdcStreamerApplier.java | 2 --
.../cdc/kafka/KafkaToIgniteMetadataUpdater.java | 5 +++++
.../cdc/kafka/CdcKafkaReplicationAbstractTest.java | 4 ++++
.../kafka/CdcKafkaReplicationAppsAbstractTest.java | 4 ++++
.../kafka/KafkaToIgniteMetadataUpdaterTest.java | 26 ++++++++++++++++++++++
.../replication/kafka-to-ignite-client.xml | 1 +
.../test/resources/replication/kafka-to-ignite.xml | 1 +
8 files changed, 44 insertions(+), 2 deletions(-)
diff --git
a/modules/cdc-ext/src/main/java/org/apache/ignite/cdc/kafka/AbstractKafkaToIgniteCdcStreamer.java
b/modules/cdc-ext/src/main/java/org/apache/ignite/cdc/kafka/AbstractKafkaToIgniteCdcStreamer.java
index 7b14ca65..fa7e64c6 100644
---
a/modules/cdc-ext/src/main/java/org/apache/ignite/cdc/kafka/AbstractKafkaToIgniteCdcStreamer.java
+++
b/modules/cdc-ext/src/main/java/org/apache/ignite/cdc/kafka/AbstractKafkaToIgniteCdcStreamer.java
@@ -198,6 +198,9 @@ abstract class AbstractKafkaToIgniteCdcStreamer implements
Runnable {
log.warning("Kafka to Ignite streamer interrupted", e);
}
+ finally {
+ U.close(metaUpdr, log);
+ }
}
/**
diff --git
a/modules/cdc-ext/src/main/java/org/apache/ignite/cdc/kafka/KafkaToIgniteCdcStreamerApplier.java
b/modules/cdc-ext/src/main/java/org/apache/ignite/cdc/kafka/KafkaToIgniteCdcStreamerApplier.java
index c28e18ba..95ff98ab 100644
---
a/modules/cdc-ext/src/main/java/org/apache/ignite/cdc/kafka/KafkaToIgniteCdcStreamerApplier.java
+++
b/modules/cdc-ext/src/main/java/org/apache/ignite/cdc/kafka/KafkaToIgniteCdcStreamerApplier.java
@@ -280,8 +280,6 @@ class KafkaToIgniteCdcStreamerApplier implements Runnable,
AutoCloseable {
@Override public void close() {
log.warning("Close applier!");
- metaUpdr.close();
-
cnsmrs.forEach(KafkaConsumer::wakeup);
}
diff --git
a/modules/cdc-ext/src/main/java/org/apache/ignite/cdc/kafka/KafkaToIgniteMetadataUpdater.java
b/modules/cdc-ext/src/main/java/org/apache/ignite/cdc/kafka/KafkaToIgniteMetadataUpdater.java
index b6a8bfb2..aa4f15c2 100644
---
a/modules/cdc-ext/src/main/java/org/apache/ignite/cdc/kafka/KafkaToIgniteMetadataUpdater.java
+++
b/modules/cdc-ext/src/main/java/org/apache/ignite/cdc/kafka/KafkaToIgniteMetadataUpdater.java
@@ -195,7 +195,12 @@ public class KafkaToIgniteMetadataUpdater implements
AutoCloseable, OffsetCommit
/** {@inheritDoc} */
@Override public void close() {
+ // Consumer is not thread-safe: wake up updateMetadata() and close
under its lock.
cnsmr.wakeup();
+
+ synchronized (this) {
+ cnsmr.close(Duration.ofMillis(kafkaReqTimeout));
+ }
}
/** {@inheritDoc} */
diff --git
a/modules/cdc-ext/src/test/java/org/apache/ignite/cdc/kafka/CdcKafkaReplicationAbstractTest.java
b/modules/cdc-ext/src/test/java/org/apache/ignite/cdc/kafka/CdcKafkaReplicationAbstractTest.java
index 7b848cff..5051d4e0 100644
---
a/modules/cdc-ext/src/test/java/org/apache/ignite/cdc/kafka/CdcKafkaReplicationAbstractTest.java
+++
b/modules/cdc-ext/src/test/java/org/apache/ignite/cdc/kafka/CdcKafkaReplicationAbstractTest.java
@@ -70,6 +70,9 @@ public abstract class CdcKafkaReplicationAbstractTest extends
AbstractReplicatio
/** */
public static final int DFLT_PARTS = 16;
+ /** */
+ public static final long KAFKA_CONSUMER_POLL_TIMEOUT = 500;
+
/** */
private static EmbeddedKafkaCluster KAFKA = null;
@@ -314,6 +317,7 @@ public abstract class CdcKafkaReplicationAbstractTest
extends AbstractReplicatio
cfg.setTopic(topic);
cfg.setMetadataTopic(metadataTopic);
cfg.setKafkaRequestTimeout(DFLT_KAFKA_REQ_TIMEOUT);
+ cfg.setKafkaConsumerPollTimeout(KAFKA_CONSUMER_POLL_TIMEOUT);
cfg.setMetricRegistryName(DFLT_METRICS_REG_NAME + "-" +
igniteCfg.getIgniteInstanceName());
diff --git
a/modules/cdc-ext/src/test/java/org/apache/ignite/cdc/kafka/CdcKafkaReplicationAppsAbstractTest.java
b/modules/cdc-ext/src/test/java/org/apache/ignite/cdc/kafka/CdcKafkaReplicationAppsAbstractTest.java
index 8279ebd9..0275928e 100644
---
a/modules/cdc-ext/src/test/java/org/apache/ignite/cdc/kafka/CdcKafkaReplicationAppsAbstractTest.java
+++
b/modules/cdc-ext/src/test/java/org/apache/ignite/cdc/kafka/CdcKafkaReplicationAppsAbstractTest.java
@@ -78,6 +78,9 @@ public abstract class CdcKafkaReplicationAppsAbstractTest
extends CdcKafkaReplic
/** */
public static final String KAFKA_REQ_TIMEOUT = "KAFKA_REQ_TIMEOUT";
+ /** */
+ public static final String KAFKA_POLL_TIMEOUT = "KAFKA_POLL_TIMEOUT";
+
/** */
public static final String PROPS_PATH = "PROPS_PATH";
@@ -173,6 +176,7 @@ public abstract class CdcKafkaReplicationAppsAbstractTest
extends CdcKafkaReplic
params.put(PARTS_TO, Integer.toString(partTo));
params.put(THREAD_CNT, Integer.toString((partTo - partFrom) / 3));
params.put(KAFKA_REQ_TIMEOUT, Long.toString(DFLT_KAFKA_REQ_TIMEOUT));
+ params.put(KAFKA_POLL_TIMEOUT,
Long.toString(KAFKA_CONSUMER_POLL_TIMEOUT));
params.put(METRIC_REG_NAME, DFLT_METRICS_REG_NAME + "-" +
igniteCfg.getIgniteInstanceName());
return runAsync(
diff --git
a/modules/cdc-ext/src/test/java/org/apache/ignite/cdc/kafka/KafkaToIgniteMetadataUpdaterTest.java
b/modules/cdc-ext/src/test/java/org/apache/ignite/cdc/kafka/KafkaToIgniteMetadataUpdaterTest.java
index 8b88df67..3e2893d5 100644
---
a/modules/cdc-ext/src/test/java/org/apache/ignite/cdc/kafka/KafkaToIgniteMetadataUpdaterTest.java
+++
b/modules/cdc-ext/src/test/java/org/apache/ignite/cdc/kafka/KafkaToIgniteMetadataUpdaterTest.java
@@ -32,6 +32,7 @@ import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.ListeningTestLogger;
import org.apache.ignite.testframework.LogListener;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.kafka.clients.admin.Admin;
import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster;
import org.junit.Test;
@@ -144,6 +145,31 @@ public class KafkaToIgniteMetadataUpdaterTest extends
GridCommonAbstractTest {
}
}
+ /** */
+ @Test
+ public void testCloseLeavesGroup() throws Exception {
+ KafkaToIgniteCdcStreamerConfiguration cfg = streamerConfiguration();
+
+ String grp = "metadata-group";
+
+ cfg.setMetadataConsumerGroup(grp);
+
+ try (Admin admin = Admin.create(kafkaProperties(kafka))) {
+ try (KafkaToIgniteMetadataUpdater updater = metadataUpdater(cfg)) {
+ updater.updateMetadata();
+
+ assertEquals(1, groupMembers(admin, grp));
+ }
+
+ assertEquals(0, groupMembers(admin, grp));
+ }
+ }
+
+ /** */
+ private static int groupMembers(Admin admin, String grp) throws Exception {
+ return
admin.describeConsumerGroups(Collections.singleton(grp)).all().get().get(grp).members().size();
+ }
+
/** */
private IgniteToKafkaCdcStreamer igniteToKafkaCdcStreamer() {
IgniteToKafkaCdcStreamer streamer = new IgniteToKafkaCdcStreamer()
diff --git
a/modules/cdc-ext/src/test/resources/replication/kafka-to-ignite-client.xml
b/modules/cdc-ext/src/test/resources/replication/kafka-to-ignite-client.xml
index 91f97722..276c00b6 100644
--- a/modules/cdc-ext/src/test/resources/replication/kafka-to-ignite-client.xml
+++ b/modules/cdc-ext/src/test/resources/replication/kafka-to-ignite-client.xml
@@ -43,6 +43,7 @@
<property name="kafkaPartsTo" value="{PARTS_TO}"/>
<property name="threadCount" value="{THREAD_CNT}"/>
<property name="kafkaRequestTimeout" value="{KAFKA_REQ_TIMEOUT}"/>
+ <property name="kafkaConsumerPollTimeout"
value="{KAFKA_POLL_TIMEOUT}"/>
<property name="metricRegistryName" value="{METRIC_REG_NAME}"/>
</bean>
diff --git a/modules/cdc-ext/src/test/resources/replication/kafka-to-ignite.xml
b/modules/cdc-ext/src/test/resources/replication/kafka-to-ignite.xml
index c5d70e42..ca743eff 100644
--- a/modules/cdc-ext/src/test/resources/replication/kafka-to-ignite.xml
+++ b/modules/cdc-ext/src/test/resources/replication/kafka-to-ignite.xml
@@ -57,6 +57,7 @@
<property name="kafkaPartsTo" value="{PARTS_TO}"/>
<property name="threadCount" value="{THREAD_CNT}"/>
<property name="kafkaRequestTimeout" value="{KAFKA_REQ_TIMEOUT}"/>
+ <property name="kafkaConsumerPollTimeout"
value="{KAFKA_POLL_TIMEOUT}"/>
<property name="metricRegistryName" value="{METRIC_REG_NAME}"/>
</bean>