This is an automated email from the ASF dual-hosted git repository.

nizhikov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-extensions.git


The following commit(s) were added to refs/heads/master by this push:
     new 542d9604 IGNITE-29309 Test of CDC Kafka replication with the nodes of 
different versions added (#372)
542d9604 is described below

commit 542d9604955174181c819b3447b4739402c1277f
Author: Nikolay <[email protected]>
AuthorDate: Thu Nov 19 11:11:27 2026 +0300

    IGNITE-29309 Test of CDC Kafka replication with the nodes of different 
versions added (#372)
---
 .../CdcKafkaReplicationCompatibilityTest.java      | 293 +++++++++++++++++++++
 1 file changed, 293 insertions(+)

diff --git 
a/modules/cdc-ext/src/test/java/org/apache/ignite/cdc/kafka/CdcKafkaReplicationCompatibilityTest.java
 
b/modules/cdc-ext/src/test/java/org/apache/ignite/cdc/kafka/CdcKafkaReplicationCompatibilityTest.java
new file mode 100644
index 00000000..f02e697f
--- /dev/null
+++ 
b/modules/cdc-ext/src/test/java/org/apache/ignite/cdc/kafka/CdcKafkaReplicationCompatibilityTest.java
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cdc.kafka;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Properties;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.cdc.CdcConfiguration;
+import 
org.apache.ignite.cdc.conflictresolve.CacheVersionConflictResolverPluginProvider;
+import org.apache.ignite.client.ClientCache;
+import org.apache.ignite.client.IgniteClient;
+import org.apache.ignite.cluster.ClusterState;
+import org.apache.ignite.compatibility.IgniteReleasedVersion;
+import 
org.apache.ignite.compatibility.testframework.junits.IgniteCompatibilityAbstractTest;
+import org.apache.ignite.configuration.ClientConfiguration;
+import org.apache.ignite.configuration.ClientConnectorConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.cdc.CdcMain;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import static 
org.apache.ignite.cdc.kafka.CdcKafkaReplicationAbstractTest.DEST_SRC_META_TOPIC;
+import static 
org.apache.ignite.cdc.kafka.CdcKafkaReplicationAbstractTest.DEST_SRC_TOPIC;
+import static 
org.apache.ignite.cdc.kafka.CdcKafkaReplicationAbstractTest.DFLT_PARTS;
+import static 
org.apache.ignite.cdc.kafka.CdcKafkaReplicationAbstractTest.SRC_DEST_META_TOPIC;
+import static 
org.apache.ignite.cdc.kafka.CdcKafkaReplicationAbstractTest.SRC_DEST_TOPIC;
+import static 
org.apache.ignite.cdc.kafka.CdcKafkaReplicationAbstractTest.initKafka;
+import static 
org.apache.ignite.cdc.kafka.CdcKafkaReplicationAbstractTest.removeKafkaTopicsAndWait;
+import static org.apache.ignite.compatibility.IgniteReleasedVersion.VER_2_18_0;
+import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
+
+/**
+ * Tests active-active CDC replication through Kafka between clusters of 
different Ignite versions.
+ * cdc-ext has no releases coupled with the 2.18 - current cdc-ext used for 
both, old and current nodes.
+ */
+@RunWith(Parameterized.class)
+public class CdcKafkaReplicationCompatibilityTest extends 
IgniteCompatibilityAbstractTest {
+    /** */
+    private static final String VAL_TYPE = "TestValue";
+
+    /** Count of the keys written to each cluster. */
+    private static final int KEYS_CNT = 50;
+
+    /** */
+    private static final int OLD_DISCO_PORT = 47500;
+
+    /** */
+    private static final int CUR_DISCO_PORT = 47600;
+
+    /** */
+    private static final int OLD_CLI_PORT = 10800;
+
+    /** */
+    private static final int CUR_CLI_PORT = 10900;
+
+    /** */
+    private static final byte OLD_CLUSTER_ID = 1;
+
+    /** */
+    private static final byte CUR_CLUSTER_ID = 2;
+
+    /** Timeout to wait for the replication. */
+    private static final long REPLICATION_TIMEOUT = 60_000;
+
+    /** */
+    private static EmbeddedKafkaCluster kafka;
+
+    /** Threads of the replication applications of the current version 
cluster. */
+    private List<Thread> curApps;
+
+    /** Old Ignite version. */
+    @Parameterized.Parameter
+    public String oldVer;
+
+    /** @return Test parameters. */
+    @Parameterized.Parameters(name = "oldVer={0}")
+    public static Collection<String> versions() {
+        return IgniteReleasedVersion.since(VER_2_18_0);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        cleanPersistenceDir();
+
+        kafka = initKafka(kafka);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        if (curApps != null) {
+            curApps.forEach(Thread::interrupt);
+
+            for (Thread app : curApps)
+                app.join(getTestTimeout());
+        }
+
+        stopAllGrids();
+
+        removeKafkaTopicsAndWait(kafka, getTestTimeout());
+
+        cleanPersistenceDir();
+
+        super.afterTest();
+    }
+
+    /** */
+    @Test
+    public void testActiveActiveReplication() throws Exception {
+        String kafkaAddrs = kafka.bootstrapServers();
+
+        startGrid(
+            1,
+            oldVer,
+            cfg -> configure(cfg, "old", OLD_DISCO_PORT, OLD_CLI_PORT, 
OLD_CLUSTER_ID),
+            ign -> startCacheAndReplication(
+                ign,
+                kafkaAddrs,
+                SRC_DEST_TOPIC,
+                SRC_DEST_META_TOPIC,
+                DEST_SRC_TOPIC,
+                DEST_SRC_META_TOPIC,
+                OLD_CLI_PORT
+            )
+        );
+
+        Ignite cur = Ignition.start(
+            configure(getConfiguration(getTestIgniteInstanceName(0)), 
"current", CUR_DISCO_PORT, CUR_CLI_PORT, CUR_CLUSTER_ID)
+        );
+
+        curApps = startCacheAndReplication(
+            cur,
+            kafkaAddrs,
+            DEST_SRC_TOPIC,
+            DEST_SRC_META_TOPIC,
+            SRC_DEST_TOPIC,
+            SRC_DEST_META_TOPIC,
+            CUR_CLI_PORT
+        );
+
+        try (IgniteClient oldCli = Ignition.startClient(new 
ClientConfiguration().setAddresses("127.0.0.1:" + OLD_CLI_PORT))) {
+            ClientCache<Integer, BinaryObject> oldCache = oldCli.<Integer, 
Object>cache(DEFAULT_CACHE_NAME).withKeepBinary();
+            IgniteCache<Integer, BinaryObject> curCache = cur.<Integer, 
Object>cache(DEFAULT_CACHE_NAME).withKeepBinary();
+
+            for (int i = 0; i < KEYS_CNT; i++)
+                oldCache.put(i, 
oldCli.binary().builder(VAL_TYPE).setField("id", i).setField("name", "old-" + 
i).build());
+
+            for (int i = KEYS_CNT; i < 2 * KEYS_CNT; i++)
+                curCache.put(i, cur.binary().builder(VAL_TYPE).setField("id", 
i).setField("name", "current-" + i).build());
+
+            assertTrue("Data must be replicated to both clusters", 
waitForCondition(() -> {
+                for (int i = 0; i < 2 * KEYS_CNT; i++) {
+                    String exp = (i < KEYS_CNT ? "old-" : "current-") + i;
+
+                    if (!hasValue(oldCache.get(i), i, exp) || 
!hasValue(curCache.get(i), i, exp))
+                        return false;
+                }
+
+                return true;
+            }, REPLICATION_TIMEOUT));
+
+            assertEquals(2 * KEYS_CNT, oldCache.size());
+            assertEquals(2 * KEYS_CNT, curCache.size());
+        }
+    }
+
+    /** */
+    private static boolean hasValue(BinaryObject val, int id, String name) {
+        return val != null && Integer.valueOf(id).equals(val.field("id")) && 
name.equals(val.field("name"));
+    }
+
+    /** */
+    private static IgniteConfiguration configure(IgniteConfiguration cfg, 
String consId, int discoPort, int cliPort, byte clusterId) {
+        CacheVersionConflictResolverPluginProvider<?> conflictPlugin = new 
CacheVersionConflictResolverPluginProvider<>();
+
+        conflictPlugin.setClusterId(clusterId);
+        conflictPlugin.setCaches(Collections.singleton(DEFAULT_CACHE_NAME));
+
+        return cfg
+            .setConsistentId(consId)
+            .setLocalHost("127.0.0.1")
+            .setDiscoverySpi(new TcpDiscoverySpi()
+                .setLocalPort(discoPort)
+                .setLocalPortRange(1)
+                .setIpFinder(new 
TcpDiscoveryVmIpFinder().setAddresses(Collections.singleton("127.0.0.1:" + 
discoPort))))
+            .setClientConnectorConfiguration(new 
ClientConnectorConfiguration().setPort(cliPort).setPortRange(0))
+            .setPluginProviders(conflictPlugin)
+            .setDataStorageConfiguration(new DataStorageConfiguration()
+                .setWalForceArchiveTimeout(1_000)
+                .setDefaultDataRegionConfiguration(new 
DataRegionConfiguration()
+                    .setPersistenceEnabled(true)
+                    .setCdcEnabled(true)));
+    }
+
+    /** */
+    private static List<Thread> startCacheAndReplication(
+        Ignite ign,
+        String kafkaAddrs,
+        String outTopic,
+        String outMetaTopic,
+        String inTopic,
+        String inMetaTopic,
+        int cliPort
+    ) {
+        ign.cluster().state(ClusterState.ACTIVE);
+
+        ign.getOrCreateCache(DEFAULT_CACHE_NAME);
+
+        IgniteToKafkaCdcStreamer cdcCnsmr = new IgniteToKafkaCdcStreamer()
+            .setTopic(outTopic)
+            .setMetadataTopic(outMetaTopic)
+            .setKafkaPartitions(DFLT_PARTS)
+            .setCaches(Collections.singleton(DEFAULT_CACHE_NAME))
+            .setKafkaProperties(kafkaProperties(kafkaAddrs, outTopic));
+
+        CdcConfiguration cdcCfg = new CdcConfiguration();
+
+        cdcCfg.setConsumer(cdcCnsmr);
+
+        Thread cdc = runDaemon(new CdcMain(ign.configuration(), null, cdcCfg), 
"ignite-to-kafka-" + outTopic);
+
+        KafkaToIgniteCdcStreamerConfiguration applierCfg = new 
KafkaToIgniteCdcStreamerConfiguration();
+
+        applierCfg.setKafkaPartsFrom(0);
+        applierCfg.setKafkaPartsTo(DFLT_PARTS);
+        applierCfg.setThreadCount(2);
+        applierCfg.setCaches(Collections.singletonList(DEFAULT_CACHE_NAME));
+        applierCfg.setTopic(inTopic);
+        applierCfg.setMetadataTopic(inMetaTopic);
+
+        Thread applier = runDaemon(
+            new KafkaToIgniteClientCdcStreamer(
+                new ClientConfiguration().setAddresses("127.0.0.1:" + cliPort),
+                kafkaProperties(kafkaAddrs, inTopic),
+                applierCfg
+            ),
+            "kafka-to-ignite-" + inTopic
+        );
+
+        return List.of(cdc, applier);
+    }
+
+    /** */
+    private static Thread runDaemon(Runnable r, String name) {
+        Thread t = new Thread(r, name);
+
+        t.setDaemon(true);
+        t.start();
+
+        return t;
+    }
+
+    /** */
+    private static Properties kafkaProperties(String kafkaAddrs, String 
grpSuffix) {
+        Properties props = new Properties();
+
+        props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaAddrs);
+        props.put(ConsumerConfig.GROUP_ID_CONFIG, "kafka-to-ignite-applier-" + 
grpSuffix);
+        props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+        props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
+        props.put(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000");
+
+        return props;
+    }
+}

Reply via email to