This is an automated email from the ASF dual-hosted git repository.

NSAmelchev pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-extensions.git


The following commit(s) were added to refs/heads/master by this push:
     new 3214a311 IGNITE-29149 Fix flaky testWithExpiryPolicy and metadata 
consumer leak in Kafka to Ignite streamer (#365)
3214a311 is described below

commit 3214a311aa63ba37ad4d52b77e0ca3b375c7e8d7
Author: Nikita Amelchev <[email protected]>
AuthorDate: Wed Oct 28 12:43:07 2026 +0300

    IGNITE-29149 Fix flaky testWithExpiryPolicy and metadata consumer leak in 
Kafka to Ignite streamer (#365)
---
 .../kafka/AbstractKafkaToIgniteCdcStreamer.java    |  3 +++
 .../cdc/kafka/KafkaToIgniteCdcStreamerApplier.java |  2 --
 .../cdc/kafka/KafkaToIgniteMetadataUpdater.java    |  5 +++++
 .../cdc/kafka/CdcKafkaReplicationAbstractTest.java |  4 ++++
 .../kafka/CdcKafkaReplicationAppsAbstractTest.java |  4 ++++
 .../kafka/KafkaToIgniteMetadataUpdaterTest.java    | 26 ++++++++++++++++++++++
 .../replication/kafka-to-ignite-client.xml         |  1 +
 .../test/resources/replication/kafka-to-ignite.xml |  1 +
 8 files changed, 44 insertions(+), 2 deletions(-)

diff --git 
a/modules/cdc-ext/src/main/java/org/apache/ignite/cdc/kafka/AbstractKafkaToIgniteCdcStreamer.java
 
b/modules/cdc-ext/src/main/java/org/apache/ignite/cdc/kafka/AbstractKafkaToIgniteCdcStreamer.java
index 7b14ca65..fa7e64c6 100644
--- 
a/modules/cdc-ext/src/main/java/org/apache/ignite/cdc/kafka/AbstractKafkaToIgniteCdcStreamer.java
+++ 
b/modules/cdc-ext/src/main/java/org/apache/ignite/cdc/kafka/AbstractKafkaToIgniteCdcStreamer.java
@@ -198,6 +198,9 @@ abstract class AbstractKafkaToIgniteCdcStreamer implements 
Runnable {
 
             log.warning("Kafka to Ignite streamer interrupted", e);
         }
+        finally {
+            U.close(metaUpdr, log);
+        }
     }
 
     /**
diff --git 
a/modules/cdc-ext/src/main/java/org/apache/ignite/cdc/kafka/KafkaToIgniteCdcStreamerApplier.java
 
b/modules/cdc-ext/src/main/java/org/apache/ignite/cdc/kafka/KafkaToIgniteCdcStreamerApplier.java
index c28e18ba..95ff98ab 100644
--- 
a/modules/cdc-ext/src/main/java/org/apache/ignite/cdc/kafka/KafkaToIgniteCdcStreamerApplier.java
+++ 
b/modules/cdc-ext/src/main/java/org/apache/ignite/cdc/kafka/KafkaToIgniteCdcStreamerApplier.java
@@ -280,8 +280,6 @@ class KafkaToIgniteCdcStreamerApplier implements Runnable, 
AutoCloseable {
     @Override public void close() {
         log.warning("Close applier!");
 
-        metaUpdr.close();
-
         cnsmrs.forEach(KafkaConsumer::wakeup);
     }
 
diff --git 
a/modules/cdc-ext/src/main/java/org/apache/ignite/cdc/kafka/KafkaToIgniteMetadataUpdater.java
 
b/modules/cdc-ext/src/main/java/org/apache/ignite/cdc/kafka/KafkaToIgniteMetadataUpdater.java
index b6a8bfb2..aa4f15c2 100644
--- 
a/modules/cdc-ext/src/main/java/org/apache/ignite/cdc/kafka/KafkaToIgniteMetadataUpdater.java
+++ 
b/modules/cdc-ext/src/main/java/org/apache/ignite/cdc/kafka/KafkaToIgniteMetadataUpdater.java
@@ -195,7 +195,12 @@ public class KafkaToIgniteMetadataUpdater implements 
AutoCloseable, OffsetCommit
 
     /** {@inheritDoc} */
     @Override public void close() {
+        // Consumer is not thread-safe: wake up updateMetadata() and close 
under its lock.
         cnsmr.wakeup();
+
+        synchronized (this) {
+            cnsmr.close(Duration.ofMillis(kafkaReqTimeout));
+        }
     }
 
     /** {@inheritDoc} */
diff --git 
a/modules/cdc-ext/src/test/java/org/apache/ignite/cdc/kafka/CdcKafkaReplicationAbstractTest.java
 
b/modules/cdc-ext/src/test/java/org/apache/ignite/cdc/kafka/CdcKafkaReplicationAbstractTest.java
index 7b848cff..5051d4e0 100644
--- 
a/modules/cdc-ext/src/test/java/org/apache/ignite/cdc/kafka/CdcKafkaReplicationAbstractTest.java
+++ 
b/modules/cdc-ext/src/test/java/org/apache/ignite/cdc/kafka/CdcKafkaReplicationAbstractTest.java
@@ -70,6 +70,9 @@ public abstract class CdcKafkaReplicationAbstractTest extends 
AbstractReplicatio
     /** */
     public static final int DFLT_PARTS = 16;
 
+    /** */
+    public static final long KAFKA_CONSUMER_POLL_TIMEOUT = 500;
+
     /** */
     private static EmbeddedKafkaCluster KAFKA = null;
 
@@ -314,6 +317,7 @@ public abstract class CdcKafkaReplicationAbstractTest 
extends AbstractReplicatio
         cfg.setTopic(topic);
         cfg.setMetadataTopic(metadataTopic);
         cfg.setKafkaRequestTimeout(DFLT_KAFKA_REQ_TIMEOUT);
+        cfg.setKafkaConsumerPollTimeout(KAFKA_CONSUMER_POLL_TIMEOUT);
 
         cfg.setMetricRegistryName(DFLT_METRICS_REG_NAME + "-" + 
igniteCfg.getIgniteInstanceName());
 
diff --git 
a/modules/cdc-ext/src/test/java/org/apache/ignite/cdc/kafka/CdcKafkaReplicationAppsAbstractTest.java
 
b/modules/cdc-ext/src/test/java/org/apache/ignite/cdc/kafka/CdcKafkaReplicationAppsAbstractTest.java
index 8279ebd9..0275928e 100644
--- 
a/modules/cdc-ext/src/test/java/org/apache/ignite/cdc/kafka/CdcKafkaReplicationAppsAbstractTest.java
+++ 
b/modules/cdc-ext/src/test/java/org/apache/ignite/cdc/kafka/CdcKafkaReplicationAppsAbstractTest.java
@@ -78,6 +78,9 @@ public abstract class CdcKafkaReplicationAppsAbstractTest 
extends CdcKafkaReplic
     /** */
     public static final String KAFKA_REQ_TIMEOUT = "KAFKA_REQ_TIMEOUT";
 
+    /** */
+    public static final String KAFKA_POLL_TIMEOUT = "KAFKA_POLL_TIMEOUT";
+
     /** */
     public static final String PROPS_PATH = "PROPS_PATH";
 
@@ -173,6 +176,7 @@ public abstract class CdcKafkaReplicationAppsAbstractTest 
extends CdcKafkaReplic
         params.put(PARTS_TO, Integer.toString(partTo));
         params.put(THREAD_CNT, Integer.toString((partTo - partFrom) / 3));
         params.put(KAFKA_REQ_TIMEOUT, Long.toString(DFLT_KAFKA_REQ_TIMEOUT));
+        params.put(KAFKA_POLL_TIMEOUT, 
Long.toString(KAFKA_CONSUMER_POLL_TIMEOUT));
         params.put(METRIC_REG_NAME, DFLT_METRICS_REG_NAME + "-" + 
igniteCfg.getIgniteInstanceName());
 
         return runAsync(
diff --git 
a/modules/cdc-ext/src/test/java/org/apache/ignite/cdc/kafka/KafkaToIgniteMetadataUpdaterTest.java
 
b/modules/cdc-ext/src/test/java/org/apache/ignite/cdc/kafka/KafkaToIgniteMetadataUpdaterTest.java
index 8b88df67..3e2893d5 100644
--- 
a/modules/cdc-ext/src/test/java/org/apache/ignite/cdc/kafka/KafkaToIgniteMetadataUpdaterTest.java
+++ 
b/modules/cdc-ext/src/test/java/org/apache/ignite/cdc/kafka/KafkaToIgniteMetadataUpdaterTest.java
@@ -32,6 +32,7 @@ import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.ListeningTestLogger;
 import org.apache.ignite.testframework.LogListener;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.kafka.clients.admin.Admin;
 import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster;
 import org.junit.Test;
 
@@ -144,6 +145,31 @@ public class KafkaToIgniteMetadataUpdaterTest extends 
GridCommonAbstractTest {
         }
     }
 
+    /** */
+    @Test
+    public void testCloseLeavesGroup() throws Exception {
+        KafkaToIgniteCdcStreamerConfiguration cfg = streamerConfiguration();
+
+        String grp = "metadata-group";
+
+        cfg.setMetadataConsumerGroup(grp);
+
+        try (Admin admin = Admin.create(kafkaProperties(kafka))) {
+            try (KafkaToIgniteMetadataUpdater updater = metadataUpdater(cfg)) {
+                updater.updateMetadata();
+
+                assertEquals(1, groupMembers(admin, grp));
+            }
+
+            assertEquals(0, groupMembers(admin, grp));
+        }
+    }
+
+    /** */
+    private static int groupMembers(Admin admin, String grp) throws Exception {
+        return 
admin.describeConsumerGroups(Collections.singleton(grp)).all().get().get(grp).members().size();
+    }
+
     /** */
     private IgniteToKafkaCdcStreamer igniteToKafkaCdcStreamer() {
         IgniteToKafkaCdcStreamer streamer = new IgniteToKafkaCdcStreamer()
diff --git 
a/modules/cdc-ext/src/test/resources/replication/kafka-to-ignite-client.xml 
b/modules/cdc-ext/src/test/resources/replication/kafka-to-ignite-client.xml
index 91f97722..276c00b6 100644
--- a/modules/cdc-ext/src/test/resources/replication/kafka-to-ignite-client.xml
+++ b/modules/cdc-ext/src/test/resources/replication/kafka-to-ignite-client.xml
@@ -43,6 +43,7 @@
         <property name="kafkaPartsTo" value="{PARTS_TO}"/>
         <property name="threadCount" value="{THREAD_CNT}"/>
         <property name="kafkaRequestTimeout" value="{KAFKA_REQ_TIMEOUT}"/>
+        <property name="kafkaConsumerPollTimeout" 
value="{KAFKA_POLL_TIMEOUT}"/>
         <property name="metricRegistryName" value="{METRIC_REG_NAME}"/>
     </bean>
 
diff --git a/modules/cdc-ext/src/test/resources/replication/kafka-to-ignite.xml 
b/modules/cdc-ext/src/test/resources/replication/kafka-to-ignite.xml
index c5d70e42..ca743eff 100644
--- a/modules/cdc-ext/src/test/resources/replication/kafka-to-ignite.xml
+++ b/modules/cdc-ext/src/test/resources/replication/kafka-to-ignite.xml
@@ -57,6 +57,7 @@
         <property name="kafkaPartsTo" value="{PARTS_TO}"/>
         <property name="threadCount" value="{THREAD_CNT}"/>
         <property name="kafkaRequestTimeout" value="{KAFKA_REQ_TIMEOUT}"/>
+        <property name="kafkaConsumerPollTimeout" 
value="{KAFKA_POLL_TIMEOUT}"/>
         <property name="metricRegistryName" value="{METRIC_REG_NAME}"/>
     </bean>
 

Reply via email to