This is an automated email from the ASF dual-hosted git repository.
nizhikov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 0d62fd96ad8 IGNITE-24535 Compatibility test for Ignite snapshot
(#11939)
0d62fd96ad8 is described below
commit 0d62fd96ad8e9608aef4283ce9f84ea3bdb1137e
Author: Vladislav Novikov <[email protected]>
AuthorDate: Mon Mar 31 23:28:13 2025 +0300
IGNITE-24535 Compatibility test for Ignite snapshot (#11939)
---
.../persistence/SnapshotCompatibilityTest.java | 453 +++++++++++++++++++++
.../IgniteCompatibilityBasicTestSuite.java | 2 +
.../cache/persistence/filename/NodeFileTree.java | 10 +-
.../cache/persistence/snapshot/dump/Dump.java | 5 +-
4 files changed, 468 insertions(+), 2 deletions(-)
diff --git
a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/SnapshotCompatibilityTest.java
b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/SnapshotCompatibilityTest.java
new file mode 100644
index 00000000000..18cf026b31c
--- /dev/null
+++
b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/SnapshotCompatibilityTest.java
@@ -0,0 +1,453 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.compatibility.persistence;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import javax.annotation.Nullable;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.binary.BinaryType;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cdc.TypeMapping;
+import org.apache.ignite.cluster.ClusterState;
+import org.apache.ignite.compatibility.IgniteReleasedVersion;
+import
org.apache.ignite.compatibility.testframework.junits.IgniteCompatibilityAbstractTest;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.dump.DumpConsumer;
+import org.apache.ignite.dump.DumpEntry;
+import org.apache.ignite.dump.DumpReader;
+import org.apache.ignite.dump.DumpReaderConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.StoredCacheData;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.jetbrains.annotations.NotNull;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ *
+ */
+@RunWith(Parameterized.class)
+public class SnapshotCompatibilityTest extends IgniteCompatibilityAbstractTest
{
+ /** */
+ private static final String OLD_IGNITE_VERSION =
Arrays.stream(IgniteReleasedVersion.values())
+ .max(Comparator.comparing(IgniteReleasedVersion::version))
+ .map(IgniteReleasedVersion::toString)
+ .orElseThrow(() -> new IllegalStateException("Enum is empty"));
+
+ /** */
+ private static final String SNAPSHOT_NAME = "test_snapshot";
+
+ /** */
+ private static final String CACHE_DUMP_NAME = "test_cache_dump";
+
+ /** */
+ private static final int BASE_CACHE_SIZE = 100;
+
+ /** */
+ private static final int ENTRIES_CNT_FOR_INCREMENT = 100;
+
+ /** */
+ private static final String CUSTOM_SNP_RELATIVE_PATH = "ex_snapshots";
+
+ /** */
+ private static final String CONSISTENT_ID = UUID.randomUUID().toString();
+
+ /** */
+ @Parameterized.Parameter
+ public boolean incSnp;
+
+ /** */
+ @Parameterized.Parameter(1)
+ @Nullable public String consId;
+
+ /** */
+ @Parameterized.Parameter(2)
+ public int oldNodesCnt;
+
+ /** */
+ @Parameterized.Parameter(3)
+ public boolean cacheDump;
+
+ /** */
+ @Parameterized.Parameter(4)
+ public boolean customSnpPath;
+
+ /** */
+ @Parameterized.Parameter(5)
+ public boolean testCacheGrp;
+
+ /** */
+ private CacheGroupInfo cacheGrpInfo;
+
+ /**
+ * The test is parameterized by whether an incremental snapshot is taken
and by consistentId.
+ * Restore incremental snapshot if consistentId is null is fixed in
2.17.0, see here https://issues.apache.org/jira/browse/IGNITE-23222.
+ * Also restoring cache dump and any kind of snapshot is pointless.
+ */
+ @Parameters(name = "incrementalSnp={0}, consistentID={1}, oldNodesCnt={2},
cacheDump={3}, customSnpPath={4}, testCacheGrp={5}")
+ public static Collection<Object[]> data() {
+ List<Object[]> data = new ArrayList<>();
+
+ for (boolean incSnp : Arrays.asList(true, false))
+ for (String consId : Arrays.asList(CONSISTENT_ID, null))
+ for (int oldNodesCnt : Arrays.asList(1, 3))
+ for (boolean cacheDump : Arrays.asList(true, false))
+ for (boolean customSnpPath : Arrays.asList(true,
false))
+ for (boolean testCacheGrp : Arrays.asList(true,
false))
+ if ((!incSnp || !cacheDump) && (!incSnp ||
consId != null))
+ data.add(new Object[]{incSnp, consId,
oldNodesCnt, cacheDump, customSnpPath, testCacheGrp});
+
+ return data;
+ }
+
+ /** */
+ @Before
+ public void setUp() {
+ cacheGrpInfo = new CacheGroupInfo("test-cache", testCacheGrp ? 2 : 1);
+ }
+
+ /** */
+ @Test
+ public void testSnapshotRestore() throws Exception {
+ try {
+ startGrid(
+ oldNodesCnt,
+ OLD_IGNITE_VERSION,
+ new ConfigurationClosure(incSnp, consId, customSnpPath, true,
cacheGrpInfo),
+ new CreateSnapshotClosure(incSnp, cacheDump, cacheGrpInfo)
+ );
+
+ stopAllGrids();
+
+ cleanPersistenceDir(true);
+
+ IgniteEx node = startGrid(currentIgniteConfiguration(incSnp,
consId, customSnpPath));
+
+ node.cluster().state(ClusterState.ACTIVE);
+
+ if (cacheDump)
+ checkCacheDump(node);
+ else if (incSnp)
+ checkIncrementalSnapshot(node);
+ else
+ checkSnapshot(node);
+ }
+ finally {
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+ }
+
+ /** */
+ private void checkSnapshot(IgniteEx node) {
+ node.snapshot().restoreSnapshot(SNAPSHOT_NAME,
Collections.singleton(cacheGrpInfo.name())).get();
+
+ cacheGrpInfo.checkCaches(node, BASE_CACHE_SIZE);
+ }
+
+ /** */
+ private void checkIncrementalSnapshot(IgniteEx node) {
+ node.snapshot().restoreSnapshot(SNAPSHOT_NAME,
Collections.singleton(cacheGrpInfo.name()), 1).get();
+
+ cacheGrpInfo.checkCaches(node, BASE_CACHE_SIZE +
ENTRIES_CNT_FOR_INCREMENT);
+ }
+
+ /** */
+ private void checkCacheDump(IgniteEx node) throws IgniteCheckedException {
+ Map<String, Integer> foundCacheSizes = new ConcurrentHashMap<>();
+
+ Set<String> foundCacheNames = ConcurrentHashMap.newKeySet();
+
+ DumpConsumer consumer = new DumpConsumer() {
+ @Override public void start() {
+ // No-op.
+ }
+
+ @Override public void onMappings(Iterator<TypeMapping> mappings) {
+ // No-op.
+ }
+
+ @Override public void onTypes(Iterator<BinaryType> types) {
+ // No-op.
+ }
+
+ @Override public void onCacheConfigs(Iterator<StoredCacheData>
caches) {
+ assertNotNull(cacheGrpInfo);
+
+ caches.forEachRemaining(cache -> {
+ CacheConfiguration<?, ?> ccfg = cache.config();
+
+ assertNotNull(ccfg);
+
+ assertEquals(cacheGrpInfo.name(), ccfg.getGroupName());
+
+ foundCacheNames.add(ccfg.getName());
+ });
+ }
+
+ @Override public void onPartition(int grp, int part,
Iterator<DumpEntry> data) {
+ assertNotNull(cacheGrpInfo);
+
+ data.forEachRemaining(de -> {
+ assertNotNull(de);
+
+ Integer key = (Integer)de.key();
+ String val = (String)de.value();
+
+ for (String cacheName : cacheGrpInfo.cacheNamesList()) {
+ if (val.startsWith(cacheName)) {
+ assertEquals(calcValue(cacheName, key), val);
+
+ foundCacheSizes.put(cacheName,
foundCacheSizes.getOrDefault(cacheName, 0) + 1);
+
+ break;
+ }
+ }
+ });
+ }
+
+ @Override public void stop() {
+ // No-op.
+ }
+ };
+
+ new DumpReader(new DumpReaderConfiguration(
+ CACHE_DUMP_NAME,
+ customSnpPath ? customSnapshotPath(CUSTOM_SNP_RELATIVE_PATH,
false) : null,
+ node.configuration(),
+ consumer
+ ), log).run();
+
+ cacheGrpInfo.cacheNamesList().forEach(
+ cacheName -> assertEquals(BASE_CACHE_SIZE,
(int)foundCacheSizes.get(cacheName))
+ );
+
+ assertTrue(cacheGrpInfo.cacheNamesList().containsAll(foundCacheNames));
+ assertEquals(cacheGrpInfo.cacheNamesList().size(),
foundCacheNames.size());
+ }
+
+ /** */
+ private @NotNull IgniteConfiguration currentIgniteConfiguration(
+ boolean incSnp,
+ String consId,
+ boolean customSnpPath
+ ) throws Exception {
+ IgniteConfiguration cfg =
getConfiguration(getTestIgniteInstanceName(0));
+
+ // We configure current Ignite version in the same way as the old one.
+ new ConfigurationClosure(incSnp, consId, customSnpPath, false,
cacheGrpInfo).apply(cfg);
+
+ return cfg;
+ }
+
+ /** */
+ private static String customSnapshotPath(String relativePath, boolean
delIfExist) throws IgniteCheckedException {
+ return U.resolveWorkDirectory(U.defaultWorkDirectory(), relativePath,
delIfExist).getAbsolutePath();
+ }
+
+ /** */
+ private static String calcValue(String cacheName, int key) {
+ return cacheName + "-organization-" + key;
+ }
+
+ /**
+ * Configuration closure both for old and current Ignite version.
+ */
+ private static class ConfigurationClosure implements
IgniteInClosure<IgniteConfiguration> {
+ /** */
+ private final String consId;
+
+ /** */
+ private final boolean incSnp;
+
+ /** */
+ private final boolean customSnpPath;
+
+ /** */
+ private final boolean delIfExist;
+
+ /** */
+ private final CacheGroupInfo cacheGrpInfo;
+
+ /** */
+ public ConfigurationClosure(
+ boolean incSnp,
+ String consId,
+ boolean customSnpPath,
+ boolean delIfExist,
+ CacheGroupInfo cacheGrpInfo
+ ) {
+ this.consId = consId;
+ this.incSnp = incSnp;
+ this.customSnpPath = customSnpPath;
+ this.delIfExist = delIfExist;
+ this.cacheGrpInfo = cacheGrpInfo;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void apply(IgniteConfiguration cfg) {
+ DataStorageConfiguration storageCfg = new
DataStorageConfiguration();
+
+
storageCfg.getDefaultDataRegionConfiguration().setPersistenceEnabled(true);
+
+ cfg.setDataStorageConfiguration(storageCfg);
+
+ cfg.setConsistentId(consId);
+
+ storageCfg.setWalCompactionEnabled(incSnp);
+
+ if (delIfExist) {
+ cfg.setCacheConfiguration(
+ cacheGrpInfo.cacheNamesList().stream()
+ .map(cacheName -> new CacheConfiguration<Integer,
String>(cacheName)
+ .setGroupName(cacheGrpInfo.name())
+ .setAffinity(new RendezvousAffinityFunction(false,
10))
+ )
+ .toArray(CacheConfiguration[]::new)
+ );
+ }
+
+ if (customSnpPath) {
+ try {
+
cfg.setSnapshotPath(customSnapshotPath(CUSTOM_SNP_RELATIVE_PATH, delIfExist));
+ }
+ catch (IgniteCheckedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ }
+
+ /**
+ * Snapshot creating closure for old Ignite version.
+ */
+ private static class CreateSnapshotClosure implements
IgniteInClosure<Ignite> {
+ /** */
+ private final boolean incSnp;
+
+ /** */
+ private final boolean cacheDump;
+
+ /** */
+ private final CacheGroupInfo cacheGrpInfo;
+
+ /** */
+ public CreateSnapshotClosure(boolean incSnp, boolean cacheDump,
CacheGroupInfo cacheGrpInfo) {
+ this.incSnp = incSnp;
+ this.cacheDump = cacheDump;
+ this.cacheGrpInfo = cacheGrpInfo;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void apply(Ignite ign) {
+ ign.cluster().state(ClusterState.ACTIVE);
+
+ cacheGrpInfo.addItemsToCacheGrp(ign, 0, BASE_CACHE_SIZE);
+
+ if (cacheDump)
+ ign.snapshot().createDump(CACHE_DUMP_NAME,
Collections.singleton(cacheGrpInfo.name())).get();
+ else
+ ign.snapshot().createSnapshot(SNAPSHOT_NAME).get();
+
+ if (incSnp) {
+ cacheGrpInfo.addItemsToCacheGrp(ign, BASE_CACHE_SIZE,
ENTRIES_CNT_FOR_INCREMENT);
+
+ ign.snapshot().createIncrementalSnapshot(SNAPSHOT_NAME).get();
+ }
+ }
+ }
+
+ /** */
+ private static class CacheGroupInfo {
+ /** */
+ private final String name;
+
+ /** */
+ private final List<String> cacheNames;
+
+ /** */
+ public CacheGroupInfo(String name, int cachesCnt) {
+ this.name = name;
+
+ cacheNames = new ArrayList<>();
+
+ for (int i = 0; i < cachesCnt; ++i)
+ cacheNames.add("test-cache-" + i);
+ }
+
+ /** */
+ public String name() {
+ return name;
+ }
+
+ /** */
+ public List<String> cacheNamesList() {
+ return cacheNames;
+ }
+
+ /** */
+ public void addItemsToCacheGrp(Ignite ign, int startIdx, int cnt) {
+ for (String cacheName : cacheNames)
+ addItemsToCache(ign.cache(cacheName), startIdx, cnt);
+ }
+
+ /** */
+ private void addItemsToCache(IgniteCache<Integer, String> cache, int
startIdx, int cnt) {
+ for (int i = startIdx; i < startIdx + cnt; ++i)
+ cache.put(i, calcValue(cache.getName(), i));
+ }
+
+ /** */
+ public void checkCaches(Ignite ign, int expectedCacheSize) {
+ for (String cacheName : cacheNames) {
+ IgniteCache<Integer, String> cache = ign.cache(cacheName);
+
+ assertNotNull(cache);
+
+ checkCache(cache, expectedCacheSize);
+ }
+ }
+
+ /** */
+ private void checkCache(IgniteCache<Integer, String> cache, int
expectedSize) {
+ assertEquals(expectedSize, cache.size());
+
+ for (int i = 0; i < expectedSize; ++i)
+ assertEquals(calcValue(cache.getName(), i), cache.get(i));
+ }
+ }
+}
diff --git
a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java
b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java
index ee0f1d09ba9..5b422b79f81 100644
---
a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java
+++
b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java
@@ -28,6 +28,7 @@ import
org.apache.ignite.compatibility.persistence.MetaStorageCompatibilityTest;
import
org.apache.ignite.compatibility.persistence.MigratingToWalV2SerializerWithCompactionTest;
import
org.apache.ignite.compatibility.persistence.MoveBinaryMetadataCompatibility;
import
org.apache.ignite.compatibility.persistence.PersistenceBasicCompatibilityTest;
+import org.apache.ignite.compatibility.persistence.SnapshotCompatibilityTest;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
@@ -47,6 +48,7 @@ import org.junit.runners.Suite;
JavaThinCompatibilityTest.class,
IgnitePKIndexesMigrationToUnwrapPkTest.class,
CompoundIndexCompatibilityTest.class,
+ SnapshotCompatibilityTest.class
})
public class IgniteCompatibilityBasicTestSuite {
}
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/NodeFileTree.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/NodeFileTree.java
index f2eb4862bd6..488cb6258b9 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/NodeFileTree.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/NodeFileTree.java
@@ -728,7 +728,7 @@ public class NodeFileTree extends SharedFileTree {
/**
* @param root Root directory.
- * @return Array of cache data files.
+ * @return List of cache data files.
*/
public static List<File> existingCacheConfigFiles(File root) {
if (cacheDir(root)) {
@@ -737,6 +737,14 @@ public class NodeFileTree extends SharedFileTree {
return cfg.exists() ? Collections.singletonList(cfg) :
Collections.emptyList();
}
+ return allExisingConfigFiles(root);
+ }
+
+ /**
+ * @param root Root directory.
+ * @return List of cache data files regardless directory name.
+ */
+ public static List<File> allExisingConfigFiles(File root) {
return
F.asList(root.listFiles(NodeFileTree::cacheOrCacheGroupConfigFile));
}
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/Dump.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/Dump.java
index f14d72a22ba..f04efd1724a 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/Dump.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/Dump.java
@@ -150,7 +150,10 @@ public class Dump implements AutoCloseable {
public List<StoredCacheData> configs(String node, int grp) {
JdkMarshaller marsh = cctx.marshallerContext().jdkMarshaller();
- return
NodeFileTree.existingCacheConfigFiles(sft(node).existingCacheDirectory(grp)).stream().map(f
-> {
+ // Searching for ALL config files regardless directory name.
+ // Initial version of Cache dump contains a bug:
+ // For a group with one cache cache-xxx directory created, but
cacheGroup-xxx expected.
+ return
NodeFileTree.allExisingConfigFiles(sft(node).existingCacheDirectory(grp)).stream().map(f
-> {
try {
return readCacheData(f, marsh, cctx.config());
}
