maksaska commented on code in PR #13184: URL: https://github.com/apache/ignite/pull/13184#discussion_r3570470664
########## modules/compatibility/src/test/java/org/apache/ignite/compatibility/ru/IgniteRebalanceOnUpgradeTest.java: ########## @@ -0,0 +1,313 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.compatibility.ru; + +import java.io.File; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.Ignition; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.client.ClientCache; +import org.apache.ignite.client.ClientCacheConfiguration; +import org.apache.ignite.client.IgniteClient; +import org.apache.ignite.compatibility.testframework.testcontainers.IgniteClusterContainer; +import org.apache.ignite.compatibility.testframework.testcontainers.IgniteContainer; +import org.apache.ignite.configuration.ClientConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.AfterClass; +import org.junit.Assume; +import org.junit.BeforeClass; +import org.junit.Test; +import org.testcontainers.DockerClientFactory; + +import static org.apache.ignite.compatibility.testframework.testcontainers.IgniteContainer.LOCAL_WORK_DIR_PATH; +import static org.apache.ignite.testframework.GridTestUtils.DFLT_TEST_TIMEOUT; +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; + +/** Smoke test for rolling upgrade with persistence. */ +public class IgniteRebalanceOnUpgradeTest extends GridCommonAbstractTest { + /** Consistent ID's. */ + private static final List<String> CONSISTENT_IDS = List.of( + "ad26bff6-5ff5-49f1-9a61-425a827953ed", + "c1099d16-e7d7-49f4-925c-53329286c444", + "7b880b69-8a9e-4b84-b555-250d365e2e67" + ); + + /** Source version image tag, overridable via {@code -Dru.source.commit.hash}. */ + private static final String SOURCE_COMMIT_HASH = System.getProperty("ru.source.commit.hash"); + + /** Upgrade mode. */ + private static final UpgradeMode UPGRADE_MODE = UpgradeMode.valueOf(System.getProperty("ru.upgrade.mode", + UpgradeMode.DOCKER.name())); + + /** Cache name. */ + private static final String CACHE_NAME = "ru-test-cache"; + + /** Local work directory. */ + private static final File LOCAL_WORK_DIR = new File(LOCAL_WORK_DIR_PATH); + + /** Local host-JVM nodes (LOCAL mode only). */ + private final List<IgniteEx> nodes = new ArrayList<>(); + + /** Consistent ID -> discovery address. */ + private final Map<String, String> addrs = new HashMap<>(); + + /** Thin client. */ + private IgniteClient client; + + /** */ + @BeforeClass + public static void beforeClass() { + Assume.assumeTrue("Docker is required for this test", DockerClientFactory.instance().isDockerAvailable()); + + if (SOURCE_COMMIT_HASH == null) + throw new RuntimeException("Source version image tag must be specified via `-Dru.source.commit.hash`"); + + U.delete(LOCAL_WORK_DIR); + } + + /** */ + @AfterClass + public static void afterClass() { + U.delete(LOCAL_WORK_DIR); + } + + /** {@inheritDoc} */ + @Override protected boolean isMultiJvm() { + return false; + } + + /** {@inheritDoc} */ + @Override protected long getTestTimeout() { + return super.getTestTimeout() * 2; + } + + /** Basic RU test. */ + @Test + public void testRollingUpgrade() throws Exception { + try (IgniteClusterContainer cluster = new IgniteClusterContainer(SOURCE_COMMIT_HASH, CONSISTENT_IDS)) { + cluster.start(); + + ClientCacheConfiguration cfg = new ClientCacheConfiguration() + .setName(CACHE_NAME) + .setBackups(1) + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); + + ClientCache<Integer, Integer> cache = client(cluster.containers().get(0).clientAddress()).createCache(cfg); Review Comment: 2 issues here: 1. If test failed to removed the work dir from previous run, the next try will reuse the data. Hence the errors: ``` [2026-07-13T13:49:58,072][ERROR][main][] Test failed [test=IgniteRebalanceOnUpgradeTest#testRollingUpgrade, duration=99234] org.apache.ignite.client.ClientException: Ignite failed to process request [3]: Failed to start cache (a cache with the same name is already started): ru-test-cache (server status code [1001]) ... org.apache.ignite.testframework.junits.GridAbstractTest$6.run(GridAbstractTest.java:2488) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.ignite.internal.client.thin.ClientServerError: Ignite failed to process request [3]: Failed to start cache (a cache with the same name is already started): ru-test-cache (server status code [1001]) at org.apache.ignite.internal.client.thin.TcpClientChannel.processNextMessage(TcpClientChannel.java:581) at ... ``` 2. I fail to remove the local work dir with my user without root. ########## modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/testcontainers/IgniteContainer.java: ########## @@ -0,0 +1,434 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.compatibility.testframework.testcontainers; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.ZoneId; +import java.util.Arrays; +import java.util.List; +import java.util.jar.JarEntry; +import java.util.jar.JarOutputStream; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; +import com.github.dockerjava.api.model.ContainerNetwork; +import org.apache.ignite.IgniteException; +import org.apache.ignite.cluster.ClusterState; +import org.apache.ignite.compatibility.testframework.plugins.DisabledRollingUpgradeProcessor; +import org.apache.ignite.compatibility.testframework.plugins.DisabledValidationProcessor; +import org.apache.ignite.compatibility.testframework.plugins.TestCompatibilityPluginProvider; +import org.apache.ignite.configuration.ClientConnectorConfiguration; +import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.BindMode; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +import static org.apache.ignite.compatibility.testframework.testcontainers.ContainerAddressResolver.EXT_ADDR_PROP_PREFIX; +import static org.apache.ignite.testframework.GridTestUtils.DFLT_TEST_TIMEOUT; +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; +import static org.junit.Assert.assertTrue; +import static org.testcontainers.utility.MountableFile.forClasspathResource; +import static org.testcontainers.utility.MountableFile.forHostPath; + +/** Ignite container. */ +public class IgniteContainer extends GenericContainer<IgniteContainer> { + /** Local work directory. */ + public static final String LOCAL_WORK_DIR_PATH = System.getProperty("ru.local.work.dir", + U.getIgniteHome() + "/target/test-ignite-work"); + + /** + * {@code true} on Linux, where the host shares the Docker bridge and reaches containers directly. Elsewhere + * (macOS/Windows Docker Desktop) the host talks to containers through a VM proxy, so the address hacks + * (published ports + ContainerAddressResolver + host.docker.internal) are used instead. + */ + public static final boolean LINUX = System.getProperty("os.name", "").toLowerCase().contains("linux"); + + /** Host directory with target-version jars for DOCKER upgrade mode, overridable via {@code -Dru.target.libs.dir}. */ + private static final Path TARGET_LIBS_DIR = Path.of(System.getProperty("ru.target.libs.dir", + U.getIgniteHome() + "/target/ignite-target-libs")); + + /** Logger. */ + private static final Logger LOGGER = LoggerFactory.getLogger(IgniteContainer.class); + + /** Ignite root directory in container. */ + private static final String ROOT_DIR_PATH = "/opt/ignite/apache-ignite/"; + + /** Ignite libs directory in container. */ + private static final String LIBS_DIR_PATH = ROOT_DIR_PATH + "libs/"; + + /** Ignite work directory in container. */ + private static final String WORK_DIR_PATH = ROOT_DIR_PATH + "work"; + + /** Config path in container. */ + private static final String CFG_PATH = ROOT_DIR_PATH + "config/test-config.xml"; + + /** */ + private static final Pattern CLUSTER_STATE_PATTERN = Pattern.compile("Cluster state: (ACTIVE|INACTIVE)"); + + /** Base host port for the published discovery port (node index added). Kept clear of the host-node ports. */ + private static final int DISCO_HOST_PORT_BASE = 50500; + + /** Base host port for the published communication port (node index added). */ + private static final int COMM_HOST_PORT_BASE = 50100; + + /** Base host port for the published thin-client port (node index added). */ + private static final int CLIENT_HOST_PORT_BASE = 50800; + + /** Custom classes (with their nested classes) used by node in containers. */ + private static final List<String> TEST_CLASSES = List.of( + ContainerAddressResolver.class.getName(), + TestCompatibilityPluginProvider.class.getName(), + DisabledRollingUpgradeProcessor.class.getName(), + DisabledValidationProcessor.class.getName() + ); + + /** Jar holding {@link #TEST_CLASSES}, injected so the old image can load it. */ + private static volatile File testClassesJar; + + /** Hostname. */ + private final String hostname; + + /** Consistent ID. */ + private final String consistentId; + + /** Path to work directory. */ + private final String workDirPath; + + /** + * Constructor with a commit hash (image tag). + * Uses {@code apacheignite/ignite:<commitHash>} as the Docker image. + */ + public IgniteContainer(String commitHash, Network net, String hostname, String consistentId, int idx) throws IOException { + super(DockerImageName.parse("apacheignite/ignite:" + commitHash)); + + this.hostname = hostname; + this.consistentId = consistentId; + workDirPath = WORK_DIR_PATH + "/" + hostname; + + int discoHostPort = DISCO_HOST_PORT_BASE + idx; + int commHostPort = COMM_HOST_PORT_BASE + idx; + + withEnv("CONFIG_URI", "file://" + CFG_PATH); + withEnv("IGNITE_QUIET", "false"); + withEnv("IGNITE_WORK_DIR", workDirPath); + withEnv("IGNITE_LOCAL_HOST", "0.0.0.0"); + withEnv("TZ", ZoneId.systemDefault().toString()); + + // node.consistent.id pins the node's consistent id (and thus its persistence folder) so the upgraded host + // node, started with the same consistent id, inherits this node's persisted data. + String jvmOpts = "-Xms512m -Xmx1g -Dnode.consistent.id=" + consistentId; + + // Proxy-networking hosts (macOS/Windows) can't reach container-internal addresses, so each node advertises + // its host-published ports (127.0.0.1:hostPort) via ContainerAddressResolver. On Linux containers are + // directly routable and advertise their real address, so no override is needed. + if (!LINUX) { + jvmOpts += " -D" + EXT_ADDR_PROP_PREFIX + TcpDiscoverySpi.DFLT_PORT + "=127.0.0.1:" + discoHostPort + + " -D" + EXT_ADDR_PROP_PREFIX + TcpCommunicationSpi.DFLT_PORT + "=127.0.0.1:" + commHostPort; + } + + withEnv("JVM_OPTS", jvmOpts); + + withFileSystemBind(LOCAL_WORK_DIR_PATH, WORK_DIR_PATH, BindMode.READ_WRITE); + withCopyFileToContainer(forClasspathResource("docker/test-config.xml"), CFG_PATH); + withCopyFileToContainer(forHostPath(testClassesJar().getAbsolutePath()), LIBS_DIR_PATH + "test-classes.jar"); + + withNetwork(net); + withNetworkAliases(hostname); + + withLogConsumer(frame -> System.out.println("[" + consistentId + "] " + frame.getUtf8String().trim())); + + // Proxy-networking hosts only: publish fixed host ports so the host JVM node can target each container at + // 127.0.0.1:<port>. On Linux the host reaches containers at their bridge IP directly, so nothing is published. + if (!LINUX) { + addFixedExposedPort(CLIENT_HOST_PORT_BASE + idx, ClientConnectorConfiguration.DFLT_PORT); + addFixedExposedPort(commHostPort, TcpCommunicationSpi.DFLT_PORT); + addFixedExposedPort(discoHostPort, TcpDiscoverySpi.DFLT_PORT); + } + + waitingFor(Wait.forLogMessage(".*Node started.*", 1) + .withStartupTimeout(Duration.ofSeconds(600))); + } + + /** {@inheritDoc} */ + @Override public void stop() { + if (isRunning()) { + try { + stopGraceful(); + } + catch (Exception e) { + LOGGER.warn("Graceful shutdown failed for node {}. Proceeding with forceful stop.", hostname, e); + } + } + + super.stop(); + } + + /** In-place upgrade inside Docker: clean libs → graceful stop → swap libs → restart. */ + public void upgradeAndRestart() throws Exception { + LOGGER.info("Cleaning up old libs in container {}", hostname); + + ExecResult result = execInContainer("sh", "-c", "rm -f " + LIBS_DIR_PATH + "*"); Review Comment: ExecResult result = execInContainer("sh", "-c", "rm -rf " + LIBS_DIR_PATH + "*"); ########## modules/compatibility/DEVNOTES.md: ########## @@ -0,0 +1,127 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +# Compatibility Module — Developer Notes + +## Prerequisites + +- [Docker](https://www.docker.com/get-started) (Docker Desktop or Docker Engine) must be installed and running. + +## Running IgniteRebalanceOnUpgradeTest + +This test verifies that data rebalancing works correctly when upgrading Ignite from a specific version to the current codebase. +It supports two upgrade modes: + +- **DOCKER** (default) — all nodes stay in Docker containers; each node is upgraded in-place by swapping its `libs/` directory and restarting. +- **LOCAL** — the source cluster runs in Docker containers, then nodes are upgraded to local host-JVM instances. + +### Step 1. Build a local Docker image for the source (old) version + +Run the following script from the **project root**, passing the commit hash of the version you want to test against: + +```bash +./modules/compatibility/src/test/resources/docker/build_docker_image.sh <commit_hash> +``` + +> **Note:** If you omit `<commit_hash>`, the script will use the hash of the latest commit in the current branch. + +The script will: +1. Checkout the specified commit. +2. Build the project (`./mvnw clean install -T1C -Pall-java,licenses -DskipTests`). +3. Initialize the release (`./mvnw initialize -Prelease`). +4. Build a Docker image tagged as `apacheignite/ignite:<commit_hash>`. +5. Restore the original git state. + +> **Note:** If a distribution archive already exists in `target/bin/`, the build steps will be skipped. + +### Step 2. Run the test + +Run `IgniteRebalanceOnUpgradeTest` from your IDE or via Maven. The source version commit hash **must** be explicitly provided via `-Dru.source.commit.hash`: + +```bash +./mvnw test -pl modules/compatibility -Dtest=IgniteRebalanceOnUpgradeTest \ + -Dru.source.commit.hash=<commit_hash> \ + -Psurefire-fork-count-1 +``` + +--- + +## Upgrade Modes + +### DOCKER mode (default) + +All nodes stay in Docker containers throughout the test. Each node is upgraded in-place: +1. The container is gracefully stopped (`docker stop`). +2. Source jars in `/opt/ignite/apache-ignite/libs/` are replaced by target jars from the host. +3. The container is restarted (`docker start`). + +The Docker image for the source cluster is the same as in LOCAL mode — only one image is needed. +The target-version jars are provided from the host filesystem. + +**Option A: Automatic (recommended)** — use the `compatibility-docker` profile: + +```bash +./mvnw test -pl modules/compatibility -Dtest=IgniteRebalanceOnUpgradeTest \ + -Dru.source.commit.hash=0ad4656eef09acda288cbad96f80f0138732d94a \ + -Psurefire-fork-count-1,compatibility-docker +``` + +The profile will automatically: +1. Check if `project/target/ignite-target-libs` symlink exists. +2. If not, check for a distribution ZIP in `project/target/bin/`. +3. If the ZIP is missing, build the project and distribution (`mvn install` + `mvn initialize -Prelease`). +4. Extract the ZIP into `project/target/bin/` (the distribution lands in `project/target/bin/apache-ignite-*-bin/`). +5. Create a symlink `project/target/ignite-target-libs` → `project/target/bin/apache-ignite-*-bin/libs/`. Review Comment: t would be really convenient for the user if we display the exact creation command right here. ########## modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/testcontainers/IgniteContainer.java: ########## @@ -0,0 +1,434 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.compatibility.testframework.testcontainers; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.ZoneId; +import java.util.Arrays; +import java.util.List; +import java.util.jar.JarEntry; +import java.util.jar.JarOutputStream; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; +import com.github.dockerjava.api.model.ContainerNetwork; +import org.apache.ignite.IgniteException; +import org.apache.ignite.cluster.ClusterState; +import org.apache.ignite.compatibility.testframework.plugins.DisabledRollingUpgradeProcessor; +import org.apache.ignite.compatibility.testframework.plugins.DisabledValidationProcessor; +import org.apache.ignite.compatibility.testframework.plugins.TestCompatibilityPluginProvider; +import org.apache.ignite.configuration.ClientConnectorConfiguration; +import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.BindMode; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +import static org.apache.ignite.compatibility.testframework.testcontainers.ContainerAddressResolver.EXT_ADDR_PROP_PREFIX; +import static org.apache.ignite.testframework.GridTestUtils.DFLT_TEST_TIMEOUT; +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; +import static org.junit.Assert.assertTrue; +import static org.testcontainers.utility.MountableFile.forClasspathResource; +import static org.testcontainers.utility.MountableFile.forHostPath; + +/** Ignite container. */ +public class IgniteContainer extends GenericContainer<IgniteContainer> { + /** Local work directory. */ + public static final String LOCAL_WORK_DIR_PATH = System.getProperty("ru.local.work.dir", + U.getIgniteHome() + "/target/test-ignite-work"); + + /** + * {@code true} on Linux, where the host shares the Docker bridge and reaches containers directly. Elsewhere + * (macOS/Windows Docker Desktop) the host talks to containers through a VM proxy, so the address hacks + * (published ports + ContainerAddressResolver + host.docker.internal) are used instead. + */ + public static final boolean LINUX = System.getProperty("os.name", "").toLowerCase().contains("linux"); + + /** Host directory with target-version jars for DOCKER upgrade mode, overridable via {@code -Dru.target.libs.dir}. */ + private static final Path TARGET_LIBS_DIR = Path.of(System.getProperty("ru.target.libs.dir", + U.getIgniteHome() + "/target/ignite-target-libs")); + + /** Logger. */ + private static final Logger LOGGER = LoggerFactory.getLogger(IgniteContainer.class); + + /** Ignite root directory in container. */ + private static final String ROOT_DIR_PATH = "/opt/ignite/apache-ignite/"; + + /** Ignite libs directory in container. */ + private static final String LIBS_DIR_PATH = ROOT_DIR_PATH + "libs/"; + + /** Ignite work directory in container. */ + private static final String WORK_DIR_PATH = ROOT_DIR_PATH + "work"; + + /** Config path in container. */ + private static final String CFG_PATH = ROOT_DIR_PATH + "config/test-config.xml"; + + /** */ + private static final Pattern CLUSTER_STATE_PATTERN = Pattern.compile("Cluster state: (ACTIVE|INACTIVE)"); + + /** Base host port for the published discovery port (node index added). Kept clear of the host-node ports. */ + private static final int DISCO_HOST_PORT_BASE = 50500; + + /** Base host port for the published communication port (node index added). */ + private static final int COMM_HOST_PORT_BASE = 50100; + + /** Base host port for the published thin-client port (node index added). */ + private static final int CLIENT_HOST_PORT_BASE = 50800; + + /** Custom classes (with their nested classes) used by node in containers. */ + private static final List<String> TEST_CLASSES = List.of( + ContainerAddressResolver.class.getName(), + TestCompatibilityPluginProvider.class.getName(), + DisabledRollingUpgradeProcessor.class.getName(), + DisabledValidationProcessor.class.getName() + ); + + /** Jar holding {@link #TEST_CLASSES}, injected so the old image can load it. */ + private static volatile File testClassesJar; + + /** Hostname. */ + private final String hostname; + + /** Consistent ID. */ + private final String consistentId; + + /** Path to work directory. */ + private final String workDirPath; + + /** + * Constructor with a commit hash (image tag). + * Uses {@code apacheignite/ignite:<commitHash>} as the Docker image. + */ + public IgniteContainer(String commitHash, Network net, String hostname, String consistentId, int idx) throws IOException { + super(DockerImageName.parse("apacheignite/ignite:" + commitHash)); + + this.hostname = hostname; + this.consistentId = consistentId; + workDirPath = WORK_DIR_PATH + "/" + hostname; + + int discoHostPort = DISCO_HOST_PORT_BASE + idx; + int commHostPort = COMM_HOST_PORT_BASE + idx; + + withEnv("CONFIG_URI", "file://" + CFG_PATH); + withEnv("IGNITE_QUIET", "false"); + withEnv("IGNITE_WORK_DIR", workDirPath); + withEnv("IGNITE_LOCAL_HOST", "0.0.0.0"); + withEnv("TZ", ZoneId.systemDefault().toString()); + + // node.consistent.id pins the node's consistent id (and thus its persistence folder) so the upgraded host + // node, started with the same consistent id, inherits this node's persisted data. + String jvmOpts = "-Xms512m -Xmx1g -Dnode.consistent.id=" + consistentId; + + // Proxy-networking hosts (macOS/Windows) can't reach container-internal addresses, so each node advertises + // its host-published ports (127.0.0.1:hostPort) via ContainerAddressResolver. On Linux containers are + // directly routable and advertise their real address, so no override is needed. + if (!LINUX) { + jvmOpts += " -D" + EXT_ADDR_PROP_PREFIX + TcpDiscoverySpi.DFLT_PORT + "=127.0.0.1:" + discoHostPort + + " -D" + EXT_ADDR_PROP_PREFIX + TcpCommunicationSpi.DFLT_PORT + "=127.0.0.1:" + commHostPort; + } + + withEnv("JVM_OPTS", jvmOpts); + + withFileSystemBind(LOCAL_WORK_DIR_PATH, WORK_DIR_PATH, BindMode.READ_WRITE); + withCopyFileToContainer(forClasspathResource("docker/test-config.xml"), CFG_PATH); + withCopyFileToContainer(forHostPath(testClassesJar().getAbsolutePath()), LIBS_DIR_PATH + "test-classes.jar"); + + withNetwork(net); + withNetworkAliases(hostname); + + withLogConsumer(frame -> System.out.println("[" + consistentId + "] " + frame.getUtf8String().trim())); + + // Proxy-networking hosts only: publish fixed host ports so the host JVM node can target each container at + // 127.0.0.1:<port>. On Linux the host reaches containers at their bridge IP directly, so nothing is published. + if (!LINUX) { + addFixedExposedPort(CLIENT_HOST_PORT_BASE + idx, ClientConnectorConfiguration.DFLT_PORT); + addFixedExposedPort(commHostPort, TcpCommunicationSpi.DFLT_PORT); + addFixedExposedPort(discoHostPort, TcpDiscoverySpi.DFLT_PORT); + } + + waitingFor(Wait.forLogMessage(".*Node started.*", 1) + .withStartupTimeout(Duration.ofSeconds(600))); + } + + /** {@inheritDoc} */ + @Override public void stop() { + if (isRunning()) { + try { + stopGraceful(); + } + catch (Exception e) { + LOGGER.warn("Graceful shutdown failed for node {}. Proceeding with forceful stop.", hostname, e); + } + } + + super.stop(); + } + + /** In-place upgrade inside Docker: clean libs → graceful stop → swap libs → restart. */ + public void upgradeAndRestart() throws Exception { + LOGGER.info("Cleaning up old libs in container {}", hostname); + + ExecResult result = execInContainer("sh", "-c", "rm -f " + LIBS_DIR_PATH + "*"); + + if (result.getExitCode() != 0) + throw new IllegalStateException("Failed to clean libs: " + result.getStderr()); + + stopGraceful(); + + restartWithTargetLibs(TARGET_LIBS_DIR); + + assertTrue("Upgraded Docker node is not running", isRunning()); + } + + /** + * Stop the container gracefully <b>without removing it</b> (container stays in "Exited" state). + * Call this before {@link #restartWithTargetLibs(Path)}. + * + * <p>Uses {@code docker stop} (SIGTERM + wait + SIGKILL after timeout) via the Docker API. + * This gives Ignite time to flush persistence data, and falls back to SIGKILL if needed.</p> + */ + private void stopGraceful() { + if (!isRunning()) + return; + + LOGGER.info("Graceful stop of node {}", hostname); + + getDockerClient().stopContainerCmd(getContainerId()) + .withTimeout(30) Review Comment: This approach might cause issues with Ignite's internal lifecycle. We should explicitly stop Ignite first to ensure a predictable shutdown procedure. Currently, the test log show 'Failed to check connection to previous node', which indicates that a backward connection check was initiated during the teardown. ########## modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/testcontainers/IgniteContainer.java: ########## @@ -0,0 +1,434 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.compatibility.testframework.testcontainers; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.ZoneId; +import java.util.Arrays; +import java.util.List; +import java.util.jar.JarEntry; +import java.util.jar.JarOutputStream; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; +import com.github.dockerjava.api.model.ContainerNetwork; +import org.apache.ignite.IgniteException; +import org.apache.ignite.cluster.ClusterState; +import org.apache.ignite.compatibility.testframework.plugins.DisabledRollingUpgradeProcessor; +import org.apache.ignite.compatibility.testframework.plugins.DisabledValidationProcessor; +import org.apache.ignite.compatibility.testframework.plugins.TestCompatibilityPluginProvider; +import org.apache.ignite.configuration.ClientConnectorConfiguration; +import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.BindMode; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +import static org.apache.ignite.compatibility.testframework.testcontainers.ContainerAddressResolver.EXT_ADDR_PROP_PREFIX; +import static org.apache.ignite.testframework.GridTestUtils.DFLT_TEST_TIMEOUT; +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; +import static org.junit.Assert.assertTrue; +import static org.testcontainers.utility.MountableFile.forClasspathResource; +import static org.testcontainers.utility.MountableFile.forHostPath; + +/** Ignite container. */ +public class IgniteContainer extends GenericContainer<IgniteContainer> { + /** Local work directory. */ + public static final String LOCAL_WORK_DIR_PATH = System.getProperty("ru.local.work.dir", + U.getIgniteHome() + "/target/test-ignite-work"); + + /** + * {@code true} on Linux, where the host shares the Docker bridge and reaches containers directly. Elsewhere + * (macOS/Windows Docker Desktop) the host talks to containers through a VM proxy, so the address hacks + * (published ports + ContainerAddressResolver + host.docker.internal) are used instead. + */ + public static final boolean LINUX = System.getProperty("os.name", "").toLowerCase().contains("linux"); + + /** Host directory with target-version jars for DOCKER upgrade mode, overridable via {@code -Dru.target.libs.dir}. */ + private static final Path TARGET_LIBS_DIR = Path.of(System.getProperty("ru.target.libs.dir", + U.getIgniteHome() + "/target/ignite-target-libs")); + + /** Logger. */ + private static final Logger LOGGER = LoggerFactory.getLogger(IgniteContainer.class); + + /** Ignite root directory in container. */ + private static final String ROOT_DIR_PATH = "/opt/ignite/apache-ignite/"; + + /** Ignite libs directory in container. */ + private static final String LIBS_DIR_PATH = ROOT_DIR_PATH + "libs/"; + + /** Ignite work directory in container. */ + private static final String WORK_DIR_PATH = ROOT_DIR_PATH + "work"; + + /** Config path in container. */ + private static final String CFG_PATH = ROOT_DIR_PATH + "config/test-config.xml"; + + /** */ + private static final Pattern CLUSTER_STATE_PATTERN = Pattern.compile("Cluster state: (ACTIVE|INACTIVE)"); + + /** Base host port for the published discovery port (node index added). Kept clear of the host-node ports. */ + private static final int DISCO_HOST_PORT_BASE = 50500; + + /** Base host port for the published communication port (node index added). */ + private static final int COMM_HOST_PORT_BASE = 50100; + + /** Base host port for the published thin-client port (node index added). */ + private static final int CLIENT_HOST_PORT_BASE = 50800; + + /** Custom classes (with their nested classes) used by node in containers. */ + private static final List<String> TEST_CLASSES = List.of( + ContainerAddressResolver.class.getName(), + TestCompatibilityPluginProvider.class.getName(), + DisabledRollingUpgradeProcessor.class.getName(), + DisabledValidationProcessor.class.getName() + ); + + /** Jar holding {@link #TEST_CLASSES}, injected so the old image can load it. */ + private static volatile File testClassesJar; + + /** Hostname. */ + private final String hostname; + + /** Consistent ID. */ + private final String consistentId; + + /** Path to work directory. */ + private final String workDirPath; + + /** + * Constructor with a commit hash (image tag). + * Uses {@code apacheignite/ignite:<commitHash>} as the Docker image. + */ + public IgniteContainer(String commitHash, Network net, String hostname, String consistentId, int idx) throws IOException { + super(DockerImageName.parse("apacheignite/ignite:" + commitHash)); + + this.hostname = hostname; + this.consistentId = consistentId; + workDirPath = WORK_DIR_PATH + "/" + hostname; + + int discoHostPort = DISCO_HOST_PORT_BASE + idx; + int commHostPort = COMM_HOST_PORT_BASE + idx; + + withEnv("CONFIG_URI", "file://" + CFG_PATH); + withEnv("IGNITE_QUIET", "false"); + withEnv("IGNITE_WORK_DIR", workDirPath); + withEnv("IGNITE_LOCAL_HOST", "0.0.0.0"); + withEnv("TZ", ZoneId.systemDefault().toString()); + + // node.consistent.id pins the node's consistent id (and thus its persistence folder) so the upgraded host + // node, started with the same consistent id, inherits this node's persisted data. + String jvmOpts = "-Xms512m -Xmx1g -Dnode.consistent.id=" + consistentId; + + // Proxy-networking hosts (macOS/Windows) can't reach container-internal addresses, so each node advertises + // its host-published ports (127.0.0.1:hostPort) via ContainerAddressResolver. On Linux containers are + // directly routable and advertise their real address, so no override is needed. + if (!LINUX) { + jvmOpts += " -D" + EXT_ADDR_PROP_PREFIX + TcpDiscoverySpi.DFLT_PORT + "=127.0.0.1:" + discoHostPort + + " -D" + EXT_ADDR_PROP_PREFIX + TcpCommunicationSpi.DFLT_PORT + "=127.0.0.1:" + commHostPort; + } + + withEnv("JVM_OPTS", jvmOpts); + + withFileSystemBind(LOCAL_WORK_DIR_PATH, WORK_DIR_PATH, BindMode.READ_WRITE); + withCopyFileToContainer(forClasspathResource("docker/test-config.xml"), CFG_PATH); + withCopyFileToContainer(forHostPath(testClassesJar().getAbsolutePath()), LIBS_DIR_PATH + "test-classes.jar"); + + withNetwork(net); + withNetworkAliases(hostname); + + withLogConsumer(frame -> System.out.println("[" + consistentId + "] " + frame.getUtf8String().trim())); + + // Proxy-networking hosts only: publish fixed host ports so the host JVM node can target each container at + // 127.0.0.1:<port>. On Linux the host reaches containers at their bridge IP directly, so nothing is published. + if (!LINUX) { + addFixedExposedPort(CLIENT_HOST_PORT_BASE + idx, ClientConnectorConfiguration.DFLT_PORT); + addFixedExposedPort(commHostPort, TcpCommunicationSpi.DFLT_PORT); + addFixedExposedPort(discoHostPort, TcpDiscoverySpi.DFLT_PORT); + } + + waitingFor(Wait.forLogMessage(".*Node started.*", 1) + .withStartupTimeout(Duration.ofSeconds(600))); + } + + /** {@inheritDoc} */ + @Override public void stop() { + if (isRunning()) { + try { + stopGraceful(); + } + catch (Exception e) { + LOGGER.warn("Graceful shutdown failed for node {}. Proceeding with forceful stop.", hostname, e); + } + } + + super.stop(); + } + + /** In-place upgrade inside Docker: clean libs → graceful stop → swap libs → restart. */ + public void upgradeAndRestart() throws Exception { + LOGGER.info("Cleaning up old libs in container {}", hostname); + + ExecResult result = execInContainer("sh", "-c", "rm -f " + LIBS_DIR_PATH + "*"); Review Comment: Why we remove libs on alive ignite node? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
