This is an automated email from the ASF dual-hosted git repository.
nizhikov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new a18c76b631e IGNITE-25389 Explicit index cache path (#12133)
a18c76b631e is described below
commit a18c76b631e615c4da84e3992bede9e307aec07f
Author: Nikolay <[email protected]>
AuthorDate: Wed Jun 18 12:58:35 2025 +0300
IGNITE-25389 Explicit index cache path (#12133)
---
.../ignite/configuration/CacheConfiguration.java | 31 +++
.../processors/cache/ClusterCachesInfo.java | 4 +
.../cache/ValidationOnNodeJoinUtils.java | 35 ++--
.../cache/persistence/filename/FileTreeUtils.java | 25 +++
.../cache/persistence/filename/NodeFileTree.java | 90 ++++++--
.../persistence/filename/SnapshotFileTree.java | 7 +-
.../snapshot/IgniteSnapshotManager.java | 2 +
.../snapshot/SnapshotResponseRemoteFutureTask.java | 25 ++-
.../cache/persistence/snapshot/SnapshotSender.java | 2 +
.../cache/persistence/snapshot/dump/Dump.java | 47 +++--
.../AbstractDataRegionRelativeStoragePathTest.java | 80 ++++++-
.../filename/CacheConfigStoragePathTest.java | 232 ++++++++++++++-------
.../CustomCacheStorageConfigurationSelfTest.java | 38 +++-
.../SnapshotCreationNonDefaultStoragePathTest.java | 49 +++--
.../filename/SQLCacheConfigStoragePathTest.java | 16 ++
15 files changed, 521 insertions(+), 162 deletions(-)
diff --git
a/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java
b/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java
index 2ae9896496d..c033a533b8f 100644
---
a/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java
+++
b/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java
@@ -442,6 +442,14 @@ public class CacheConfiguration<K, V> extends
MutableConfiguration<K, V> {
@IgniteExperimental
@Nullable private String[] storagePaths;
+ /**
+ * Root directory where index file are stored.
+ * @see DataStorageConfiguration#setStoragePath(String)
+ * @see DataStorageConfiguration#setExtraStoragePaths(String[])
+ */
+ @IgniteExperimental
+ @Nullable private String idxPath;
+
/** Empty constructor (all values are initialized to their defaults). */
public CacheConfiguration() {
/* No-op. */
@@ -541,6 +549,7 @@ public class CacheConfiguration<K, V> extends
MutableConfiguration<K, V> {
sqlOnheapCacheMaxSize = cc.getSqlOnheapCacheMaxSize();
evtsDisabled = cc.isEventsDisabled();
storagePaths = cc.getStoragePaths();
+ idxPath = cc.getIndexPath();
}
/**
@@ -2488,6 +2497,28 @@ public class CacheConfiguration<K, V> extends
MutableConfiguration<K, V> {
return this;
}
+ /**
+ * @return A path to the root directory where the Persistent Store for
cache group will persist index.
+ */
+ @IgniteExperimental
+ @Nullable public String getIndexPath() {
+ return idxPath;
+ }
+
+ /**
+ * Sets a path to the root directory where the Persistent Store will
persist index partition.
+ * By default, the Persistent Store's files are located under {@link
DataStorageConfiguration#getStoragePath()}.
+ *
+ * @param idxPath Index path.
+ * @return {@code this} for chaining.
+ */
+ @IgniteExperimental
+ public CacheConfiguration<K, V> setIndexPath(String idxPath) {
+ this.idxPath = idxPath;
+
+ return this;
+ }
+
/** {@inheritDoc} */
@Override public String toString() {
return S.toString(CacheConfiguration.class, this);
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
index 30b92aebb2d..86bce4091ea 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
@@ -2601,6 +2601,10 @@ public class ClusterCachesInfo {
CU.validateCacheGroupsAttributesMismatch(log, cfg, startCfg,
"storagePath", "Storage path",
cfg.getStoragePaths(), startCfg.getStoragePaths(), true);
+
+ CU.validateCacheGroupsAttributesMismatch(log, cfg, startCfg,
+ "indexPath", "Index path",
+ cfg.getIndexPath(), startCfg.getIndexPath(), true);
}
/**
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ValidationOnNodeJoinUtils.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ValidationOnNodeJoinUtils.java
index aee1189ed02..eb3fd275afc 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ValidationOnNodeJoinUtils.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ValidationOnNodeJoinUtils.java
@@ -21,7 +21,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -74,6 +73,7 @@ import static
org.apache.ignite.internal.IgniteNodeAttributes.ATTR_CONSISTENCY_C
import static
org.apache.ignite.internal.IgniteNodeAttributes.ATTR_TX_AWARE_QUERIES_ENABLED;
import static
org.apache.ignite.internal.IgniteNodeAttributes.ATTR_TX_SERIALIZABLE_ENABLED;
import static
org.apache.ignite.internal.processors.cache.GridCacheUtils.isDefaultDataRegionPersistent;
+import static
org.apache.ignite.internal.processors.cache.persistence.filename.FileTreeUtils.nodeStorages;
import static
org.apache.ignite.internal.processors.security.SecurityUtils.nodeSecurityContext;
/**
@@ -393,24 +393,29 @@ public class ValidationOnNodeJoinUtils {
cacheSpec.toString());
}
- if (!F.isEmpty(cc.getStoragePaths()) && !ctx.clientNode()) {
- DataStorageConfiguration dsCfg = c.getDataStorageConfiguration();
+ if (!ctx.clientNode()) {
+ if (!F.isEmpty(cc.getStoragePaths())) {
+ List<String> csp = Arrays.asList(cc.getStoragePaths());
- List<String> csp = Arrays.asList(cc.getStoragePaths());
+ Set<String> nodeStorages =
nodeStorages(c.getDataStorageConfiguration());
- if (dsCfg == null)
- throw new IgniteCheckedException("Data storage must be
configured when cache storage path set: " + csp);
-
- Set<String> nodeStorages = new
HashSet<>(F.asList(dsCfg.getExtraStoragePaths()));
+ if (!nodeStorages.containsAll(csp)) {
+ throw new IgniteCheckedException(
+ "Unknown storage path. Storage path must be from
DataStorageConfiguration " +
+ "[cacheStorage=" + csp + ", nodeStorages=" +
nodeStorages + ']'
+ );
+ }
+ }
- if (!F.isEmpty(dsCfg.getStoragePath()))
- nodeStorages.add(dsCfg.getStoragePath());
+ if (!F.isEmpty(cc.getIndexPath())) {
+ Set<String> nodeStorages =
nodeStorages(c.getDataStorageConfiguration());
- if (!nodeStorages.containsAll(csp)) {
- throw new IgniteCheckedException(
- "Unknown storage path. Storage path must be from
DataStorageConfiguration " +
- "[cacheStorage=" + csp + ", nodeStorages=" +
nodeStorages + ']'
- );
+ if (!nodeStorages.contains(cc.getIndexPath())) {
+ throw new IgniteCheckedException(
+ "Unknown storage path. Storage path must be from
DataStorageConfiguration " +
+ "[indexPath=" + cc.getIndexPath() + ",
nodeStorages=" + nodeStorages + ']'
+ );
+ }
}
}
}
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/FileTreeUtils.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/FileTreeUtils.java
index 5bd538a869e..7d1dd7d7f05 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/FileTreeUtils.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/FileTreeUtils.java
@@ -19,14 +19,19 @@ package
org.apache.ignite.internal.processors.cache.persistence.filename;
import java.io.File;
import java.io.IOException;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.jetbrains.annotations.Nullable;
+import static
org.apache.ignite.internal.pagemem.PageIdAllocator.INDEX_PARTITION;
+
/**
* Utility methods for {@link NodeFileTree}.
*/
@@ -70,6 +75,9 @@ public class FileTreeUtils {
* @return Storage path from config for partition.
*/
public static @Nullable String partitionStorage(CacheConfiguration<?, ?>
ccfg, int part) {
+ if (part == INDEX_PARTITION && !F.isEmpty(ccfg.getIndexPath()))
+ return ccfg.getIndexPath();
+
String[] csp = ccfg.getStoragePaths();
if (F.isEmpty(csp))
@@ -78,6 +86,23 @@ public class FileTreeUtils {
return resolveStorage(csp, part);
}
+ /**
+ * @param dsCfg Data storage configuration.
+ * @return All known storages.
+ * @throws IgniteCheckedException
+ */
+ public static Set<String> nodeStorages(DataStorageConfiguration dsCfg)
throws IgniteCheckedException {
+ if (dsCfg == null)
+ throw new IgniteCheckedException("Data storage must be
configured");
+
+ Set<String> nodeStorages = new
HashSet<>(F.asList(dsCfg.getExtraStoragePaths()));
+
+ if (!F.isEmpty(dsCfg.getStoragePath()))
+ nodeStorages.add(dsCfg.getStoragePath());
+
+ return nodeStorages;
+ }
+
/**
* @param dir Directory to remove
* @param err If {@code true} then operation ends with error.
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/NodeFileTree.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/NodeFileTree.java
index c31869ca3fb..c81daecc1e2 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/NodeFileTree.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/NodeFileTree.java
@@ -78,7 +78,8 @@ import static
org.apache.ignite.internal.processors.cache.persistence.metastorag
* </ul>
*
* Node, user can configure several extra storages by {@link
DataStorageConfiguration#setExtraStoragePaths(String...)}
- * Which can be used by the cache to store data. See {@link
CacheConfiguration#setStoragePaths(String...)}.
+ * Which can be used by the cache to store data. See {@link
CacheConfiguration#setStoragePaths(String...)}
+ * and {@link CacheConfiguration#setIndexPath(String)}.
* Partition files will be spread evenly among all configured storages.
* In case extra storages configured, each storage will repeat structure
described below.
* {@link NodeFileTree#defaultCacheStorage(CacheConfiguration)} will be used
to store cache config.
@@ -567,19 +568,34 @@ public class NodeFileTree extends SharedFileTree {
* @return Store dirs for given cache.
*/
public File[] cacheStorages(CacheConfiguration<?, ?> ccfg) {
- String cacheDirName = ccfg.getGroupName() != null
- ? CACHE_GRP_DIR_PREFIX + ccfg.getGroupName()
- : CACHE_DIR_PREFIX + ccfg.getName();
+ return cacheStorages(ccfg, true);
+ }
+
+ /**
+ * @param ccfg Cache configuration.
+ * @param includeIdxPath If {@code true} then add optional index store to
results.
+ * @return Store dirs for given cache.
+ */
+ private File[] cacheStorages(CacheConfiguration<?, ?> ccfg, boolean
includeIdxPath) {
+ String cacheDirName = cacheDirName(ccfg);
String[] csp = ccfg.getStoragePaths();
+ String idxPath = ccfg.getIndexPath();
+ boolean idxPathEmpty = !includeIdxPath || F.isEmpty(idxPath);
- if (F.isEmpty(csp))
- return new File[] {new File(cacheStorageRoot(null), cacheDirName)};
+ if (F.isEmpty(csp)) {
+ return idxPathEmpty
+ ? new File[]{cacheStorage(null, cacheDirName)}
+ : new File[]{cacheStorage(null, cacheDirName),
cacheStorage(idxPath, cacheDirName)};
+ }
+
+ File[] cs = new File[csp.length + (idxPathEmpty ? 0 : 1)];
- File[] cs = new File[csp.length];
+ for (int i = 0; i < csp.length; i++)
+ cs[i] = cacheStorage(csp[i], cacheDirName);
- for (int i = 0; i < cs.length; i++)
- cs[i] = new File(cacheStorageRoot(csp[i]), cacheDirName);
+ if (!idxPathEmpty)
+ cs[cs.length - 1] = cacheStorage(idxPath, cacheDirName);
return cs;
}
@@ -640,7 +656,14 @@ public class NodeFileTree extends SharedFileTree {
* @return Partition file.
*/
public File partitionFile(CacheConfiguration<?, ?> ccfg, int part) {
- return new File(resolveStorage(cacheStorages(ccfg), part),
partitionFileName(part));
+ if (part == INDEX_PARTITION) {
+ String idxPath = ccfg.getIndexPath();
+
+ if (!F.isEmpty(idxPath))
+ return new File(cacheStorage(idxPath, cacheDirName(ccfg)),
partitionFileName(INDEX_PARTITION));
+ }
+
+ return new File(resolveStorage(cacheStorages(ccfg, false), part),
partitionFileName(part));
}
/**
@@ -648,11 +671,20 @@ public class NodeFileTree extends SharedFileTree {
* @return Store directory for given cache.
*/
public File[] tmpCacheStorages(CacheConfiguration<?, ?> ccfg) {
- File[] cacheStorages = cacheStorages(ccfg);
+ return tmpCacheStorages(ccfg, true);
+ }
+
+ /**
+ * @param ccfg Cache configuration.
+ * @param includeIdxPath If {@code true} then add optional index store to
results.
+ * @return Store directory for given cache.
+ */
+ private File[] tmpCacheStorages(CacheConfiguration<?, ?> ccfg, boolean
includeIdxPath) {
+ File[] cacheStorages = cacheStorages(ccfg, includeIdxPath);
File[] tmpCacheStorages = new File[cacheStorages.length];
for (int i = 0; i < cacheStorages.length; i++)
- tmpCacheStorages[i] = new File(cacheStorages[i].getParentFile(),
TMP_CACHE_DIR_PREFIX + cacheStorages[i].getName());
+ tmpCacheStorages[i] = tmpCacheStorage(cacheStorages[i]);
return tmpCacheStorages;
}
@@ -665,9 +697,10 @@ public class NodeFileTree extends SharedFileTree {
* @param cacheDirName Cache directory name.
* @return Temp store directory for given cache.
* @see CacheConfiguration#getStoragePaths()
+ * @see CacheConfiguration#getIndexPath()
*/
public File tmpCacheStorage(@Nullable String storagePath, String
cacheDirName) {
- return new File(cacheStorageRoot(storagePath), TMP_CACHE_DIR_PREFIX +
cacheDirName);
+ return tmpCacheStorage(cacheStorage(storagePath, cacheDirName));
}
/**
@@ -702,7 +735,17 @@ public class NodeFileTree extends SharedFileTree {
* @return Path to the temp partition file.
*/
public File tmpPartition(CacheConfiguration<?, ?> ccfg, int partId) {
- return new File(resolveStorage(tmpCacheStorages(ccfg), partId),
partitionFileName(partId));
+ if (partId == INDEX_PARTITION) {
+ String idxPath = ccfg.getIndexPath();
+
+ if (!F.isEmpty(idxPath)) {
+ File idxPartFile = partitionFile(ccfg, INDEX_PARTITION);
+
+ return new File(tmpCacheStorage(idxPartFile.getParentFile()),
idxPartFile.getName());
+ }
+ }
+
+ return new File(resolveStorage(tmpCacheStorages(ccfg, false), partId),
partitionFileName(partId));
}
/**
@@ -713,6 +756,7 @@ public class NodeFileTree extends SharedFileTree {
* @param partId partition id.
* @return Path to the temp partition file.
* @see CacheConfiguration#getStoragePaths()
+ * @see CacheConfiguration#getIndexPath()
*/
public File tmpPartition(@Nullable String storagePath, String
cacheDirName, int partId) {
return new File(tmpCacheStorage(storagePath, cacheDirName),
partitionFileName(partId));
@@ -879,11 +923,14 @@ public class NodeFileTree extends SharedFileTree {
/**
* @param storagePath Value from config.
+ * @param cacheDirName Regular or temporary directory name for cache.
* @return File storage.
* @see CacheConfiguration#getStoragePaths()
+ * @see CacheConfiguration#getIndexPath()
+ * @see #cacheDirName(CacheConfiguration)
*/
- private File cacheStorageRoot(@Nullable String storagePath) {
- return storagePath == null ? nodeStorage :
extraStorages.getOrDefault(storagePath, nodeStorage);
+ private File cacheStorage(@Nullable String storagePath, String
cacheDirName) {
+ return new File(storagePath == null ? nodeStorage :
extraStorages.getOrDefault(storagePath, nodeStorage), cacheDirName);
}
/**
@@ -1004,6 +1051,7 @@ public class NodeFileTree extends SharedFileTree {
*/
public static int partId(File part) {
String name = part.getName();
+
if (name.equals(INDEX_FILE_NAME))
return INDEX_PARTITION;
@@ -1046,6 +1094,16 @@ public class NodeFileTree extends SharedFileTree {
return Paths.get(root.getAbsolutePath(), path, folderName).toFile();
}
+ /**
+ * @param ccfg Cache configuration.
+ * @return Cache directory name.
+ */
+ private static String cacheDirName(CacheConfiguration<?, ?> ccfg) {
+ return ccfg.getGroupName() != null
+ ? CACHE_GRP_DIR_PREFIX + ccfg.getGroupName()
+ : CACHE_DIR_PREFIX + ccfg.getName();
+ }
+
/**
* @param typeId Type id.
* @return Binary metadata file name.
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/SnapshotFileTree.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/SnapshotFileTree.java
index e41197b277f..52b12a4d0db 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/SnapshotFileTree.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/SnapshotFileTree.java
@@ -30,7 +30,6 @@ import
org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.dump.DumpReader;
import org.apache.ignite.internal.GridKernalContext;
-import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.A;
import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.S;
@@ -244,10 +243,10 @@ public class SnapshotFileTree extends NodeFileTree {
/**
* @param grpId Cache group id.
- * @return Files that match cache or cache group pattern.
+ * @return Directories that match cache or cache group pattern.
*/
- public File existingCacheDirectory(int grpId) {
- return F.first(existingCacheDirs(true, f -> CU.cacheId(cacheName(f))
== grpId));
+ public List<File> existingCacheDirectories(int grpId) {
+ return existingCacheDirs(true, f -> CU.cacheId(cacheName(f)) == grpId);
}
/**
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java
index e869ce66706..c42c302ad42 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java
@@ -309,6 +309,7 @@ public class IgniteSnapshotManager extends
GridCacheSharedManagerAdapter
/**
* File transmission parameter of a storage path for given group id.
* @see CacheConfiguration#getStoragePaths()
+ * @see CacheConfiguration#getIndexPath()
*/
private static final String SNP_CACHE_STORAGE_PATH_PARAM =
"cacheStoragePath";
@@ -3725,6 +3726,7 @@ public class IgniteSnapshotManager extends
GridCacheSharedManagerAdapter
* @param pair Cache group id with corresponding partition id.
* @return Map of params.
* @see CacheConfiguration#getStoragePaths()
+ * @see CacheConfiguration#getIndexPath()
*/
private Map<String, Serializable> transmissionParams(
String rqId,
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotResponseRemoteFutureTask.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotResponseRemoteFutureTask.java
index b4c9a35fa17..12b589f1d6b 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotResponseRemoteFutureTask.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotResponseRemoteFutureTask.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.processors.cache.persistence.snapshot;
import java.io.File;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
@@ -204,21 +205,25 @@ public class SnapshotResponseRemoteFutureTask extends
AbstractSnapshotFutureTask
if (cacheData.containsKey(grpId))
return cacheData.get(grpId);
- File cacheDir = sft.existingCacheDirectory(grpId);
+ List<File> cacheDirs = sft.existingCacheDirectories(grpId);
- if (cacheDir == null) {
+ if (F.isEmpty(cacheDirs)) {
throw new IgniteException("Cache directory not found
[snpName=" + snpName + ", meta=" + meta +
", grp=" + grpId + ']');
}
- List<StoredCacheData> res =
NodeFileTree.existingCacheConfigFiles(cacheDir).stream().map(f -> {
- try {
- return cctx.cache().configManager().readCacheData(f);
- }
- catch (IgniteCheckedException e) {
- throw new IgniteException(e);
- }
- }).collect(Collectors.toList());
+ List<StoredCacheData> res = new ArrayList<>();
+
+ for (File cacheDir : cacheDirs) {
+
res.addAll(NodeFileTree.existingCacheConfigFiles(cacheDir).stream().map(f -> {
+ try {
+ return cctx.cache().configManager().readCacheData(f);
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
+ }).collect(Collectors.toList()));
+ }
if (res.isEmpty()) {
throw new IgniteException("Cache configs not found [snpName="
+ snpName + ", meta=" + meta +
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotSender.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotSender.java
index be42b5503a9..1b53ef36a8b 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotSender.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotSender.java
@@ -130,6 +130,7 @@ public abstract class SnapshotSender {
* @param pair Group id with partition id pair.
* @param length Partition length.
* @see CacheConfiguration#getStoragePaths()
+ * @see CacheConfiguration#getIndexPath()
*/
public final void sendPart(File from, File to, @Nullable String
storagePath, GroupPartitionId pair, Long length) {
if (!lock.readLock().tryLock())
@@ -197,6 +198,7 @@ public abstract class SnapshotSender {
* @param pair Group id with partition id pair.
* @param length Partition length.
* @see CacheConfiguration#getStoragePaths()
+ * @see CacheConfiguration#getIndexPath()
*/
protected abstract void sendPart0(File from, File to, @Nullable String
storagePath, GroupPartitionId pair, Long length);
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/Dump.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/Dump.java
index 6dcc392fedf..9df3977f2d9 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/Dump.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/Dump.java
@@ -21,6 +21,7 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
@@ -152,19 +153,25 @@ public class Dump implements AutoCloseable {
public List<StoredCacheData> configs(String node, int grp, @Nullable
Set<Integer> cacheIds) {
JdkMarshaller marsh = cctx.marshallerContext().jdkMarshaller();
+ List<StoredCacheData> res = new ArrayList<>();
+
// Searching for ALL config files regardless directory name.
// Initial version of Cache dump contains a bug:
// For a group with one cache cache-xxx directory created, but
cacheGroup-xxx expected.
- return
NodeFileTree.allExisingConfigFiles(sft(node).existingCacheDirectory(grp)).stream().map(f
-> {
- try {
- return readCacheData(f, marsh, cctx.config());
- }
- catch (IgniteCheckedException e) {
- throw new IgniteException(e);
- }
- // Keep only caches from filter.
- }).filter(scd -> cacheIds == null || cacheIds.contains(scd.cacheId()))
- .collect(Collectors.toList());
+ for (File cacheDir : sft(node).existingCacheDirectories(grp)) {
+
res.addAll(NodeFileTree.allExisingConfigFiles(cacheDir).stream().map(f -> {
+ try {
+ return readCacheData(f, marsh, cctx.config());
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
+ // Keep only caches from filter.
+ }).filter(scd -> cacheIds == null ||
cacheIds.contains(scd.cacheId()))
+ .collect(Collectors.toList()));
+ }
+
+ return res;
}
/**
@@ -173,10 +180,10 @@ public class Dump implements AutoCloseable {
* @return Dump iterator.
*/
public List<Integer> partitions(String node, int grp) {
- List<File> parts =
sft(node).existingCachePartitionFiles(sft(node).existingCacheDirectory(grp),
true, comprParts);
+ List<File> parts = new ArrayList<>();
- if (parts == null)
- return Collections.emptyList();
+ for (File cacheDir : sft(node).existingCacheDirectories(grp))
+ parts.addAll(sft(node).existingCachePartitionFiles(cacheDir, true,
comprParts));
return parts.stream()
.map(NodeFileTree::partId)
@@ -197,7 +204,7 @@ public class Dump implements AutoCloseable {
FileIO dumpFile;
try {
- dumpFile = ioFactory.create(new
File(sft(node).existingCacheDirectory(grp), dumpPartFileName(part,
comprParts)));
+ dumpFile = ioFactory.create(dumpFile(node, grp, part));
}
catch (IOException e) {
throw new RuntimeException(e);
@@ -272,6 +279,18 @@ public class Dump implements AutoCloseable {
};
}
+ /** */
+ private File dumpFile(String node, int grp, int part) {
+ for (File cacheDir : sft(node).existingCacheDirectories(grp)) {
+ File partFile = new File(cacheDir, dumpPartFileName(part,
comprParts));
+
+ if (partFile.exists())
+ return partFile;
+ }
+
+ throw new IllegalStateException("Part file not found[node=" + node +
", grp=" + grp + ", part=" + part + ']');
+ }
+
/** @return Dump directories. */
public List<SnapshotFileTree> fileTrees() {
return sfts;
diff --git
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/filename/AbstractDataRegionRelativeStoragePathTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/filename/AbstractDataRegionRelativeStoragePathTest.java
index f40d9e40621..1db94184f8a 100644
---
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/filename/AbstractDataRegionRelativeStoragePathTest.java
+++
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/filename/AbstractDataRegionRelativeStoragePathTest.java
@@ -30,6 +30,8 @@ import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.cluster.ClusterState;
import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.IgnitionEx;
import org.apache.ignite.internal.util.typedef.F;
@@ -50,34 +52,63 @@ public abstract class
AbstractDataRegionRelativeStoragePathTest extends GridComm
/** Second custom storage path. */
static final String STORAGE_PATH_2 = "storage2";
+ /** Custom indexes path. */
+ static final String IDX_PATH = "idxs";
+
/** */
static final String SNP_PATH = "ex_snapshots";
/** */
protected static final int PARTS_CNT = 15;
+ /** */
+ protected static final int GRID_CNT = 3;
+
/** */
@Parameterized.Parameter()
- public boolean absPath;
+ public PathMode pathMode;
/** */
@Parameterized.Parameter(1)
public boolean severalCacheStorages;
/** */
- @Parameterized.Parameters(name = "absPath={0},severalCacheStorages={1}")
+ @Parameterized.Parameter(2)
+ public boolean idxStorage;
+
+ /** */
+ @Parameterized.Parameters(name =
"path={0},severalCacheStorages={1},idxStorage={2}")
public static List<Object[]> params() {
List<Object[]> res = new ArrayList<>();
- for (boolean absPath : new boolean[]{true, false}) {
+ for (PathMode absPathMode : PathMode.values()) {
for (boolean severalCacheStorages : new boolean[]{true, false}) {
- res.add(new Object[] {absPath, severalCacheStorages});
+ for (boolean idxStorage : new boolean[]{true, false})
+ res.add(new Object[]{absPathMode, severalCacheStorages,
idxStorage});
}
}
return res;
}
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String
igniteInstanceName) throws Exception {
+ DataStorageConfiguration dsCfg = dataStorageConfiguration();
+
+ dsCfg.getDefaultDataRegionConfiguration()
+ .setPersistenceEnabled(true);
+
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ return cfg
+ .setConsistentId(consId(cfg))
+ .setDataStorageConfiguration(dsCfg)
+ .setCacheConfiguration(ccfgs())
+ .setWorkDirectory(pathMode == PathMode.SEPARATE_ROOT
+ ? new File(U.defaultWorkDirectory(),
consId(cfg)).getAbsolutePath()
+ : null);
+ }
+
/** {@inheritDoc} */
@Override protected void afterTest() throws Exception {
super.afterTest();
@@ -86,14 +117,22 @@ public abstract class
AbstractDataRegionRelativeStoragePathTest extends GridComm
cleanPersistenceDir();
- if (absPath) {
+ if (pathMode == PathMode.ABS) {
U.delete(new File(storagePath(STORAGE_PATH)).getParentFile());
U.delete(new File(storagePath(STORAGE_PATH_2)).getParentFile());
+ U.delete(new File(storagePath(IDX_PATH)).getParentFile());
}
- else {
+ else if (pathMode == PathMode.RELATIVE) {
U.delete(new File(U.defaultWorkDirectory(), STORAGE_PATH));
U.delete(new File(U.defaultWorkDirectory(), STORAGE_PATH_2));
+ U.delete(new File(U.defaultWorkDirectory(), IDX_PATH));
}
+ else if (pathMode == PathMode.SEPARATE_ROOT) {
+ for (File f : new File(U.defaultWorkDirectory()).listFiles(f ->
!f.getName().equals("log")))
+ U.delete(f);
+ }
+ else
+ throw new IllegalStateException("Unknown path mode");
U.delete(new File(U.defaultWorkDirectory(), SNP_PATH));
}
@@ -116,7 +155,7 @@ public abstract class
AbstractDataRegionRelativeStoragePathTest extends GridComm
ft.extraStorages().values().forEach(U::delete);
});
- U.delete(F.first(fts).db());
+ fts.forEach(ft -> U.delete(ft.db()));
IgniteEx srv = startAndActivate();
@@ -170,7 +209,7 @@ public abstract class
AbstractDataRegionRelativeStoragePathTest extends GridComm
/** */
IgniteEx startAndActivate() throws Exception {
- IgniteEx srv = startGrids(3);
+ IgniteEx srv = startGrids(GRID_CNT);
srv.cluster().state(ClusterState.ACTIVE);
@@ -193,6 +232,9 @@ public abstract class
AbstractDataRegionRelativeStoragePathTest extends GridComm
if (!F.isEmpty(storagePath))
ccfg.setStoragePaths(storagePath);
+ if (idxStorage)
+ ccfg.setIndexPath(storagePath(IDX_PATH));
+
return ccfg;
}
@@ -213,7 +255,7 @@ public abstract class
AbstractDataRegionRelativeStoragePathTest extends GridComm
/** */
String storagePath(String storagePath) {
try {
- return absPath ? new File(U.defaultWorkDirectory(), "abs/" +
storagePath).getAbsolutePath() : storagePath;
+ return pathMode == PathMode.ABS ? new
File(U.defaultWorkDirectory(), "abs/" + storagePath).getAbsolutePath() :
storagePath;
}
catch (IgniteCheckedException e) {
throw new RuntimeException(e);
@@ -223,6 +265,26 @@ public abstract class
AbstractDataRegionRelativeStoragePathTest extends GridComm
/** @param fts Nodes file trees. */
abstract void checkFileTrees(List<NodeFileTree> fts) throws
IgniteCheckedException;
+ /** Data storage configuration. */
+ abstract DataStorageConfiguration dataStorageConfiguration();
+
/** Cache configs. */
abstract CacheConfiguration[] ccfgs();
+
+ /** */
+ public enum PathMode {
+ /** Absolute path for custom directories. */
+ ABS,
+
+ /** Relative path for custom directories. Same root. */
+ RELATIVE,
+
+ /** Relative path for custom directories. Separate root for each node.
*/
+ SEPARATE_ROOT
+ }
+
+ /** */
+ static String consId(IgniteConfiguration cfg) {
+ return U.maskForFileName(cfg.getIgniteInstanceName());
+ }
}
diff --git
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/filename/CacheConfigStoragePathTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/filename/CacheConfigStoragePathTest.java
index b62ab5766c8..d5054060571 100644
---
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/filename/CacheConfigStoragePathTest.java
+++
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/filename/CacheConfigStoragePathTest.java
@@ -21,19 +21,23 @@ import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
+import java.util.function.IntConsumer;
+import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
+import org.apache.ignite.Ignition;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.DataStorageConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.CU;
@@ -41,24 +45,17 @@ import org.apache.ignite.internal.util.typedef.internal.U;
import org.junit.Test;
import static
org.apache.ignite.configuration.IgniteConfiguration.DFLT_SNAPSHOT_DIRECTORY;
+import static
org.apache.ignite.internal.pagemem.PageIdAllocator.INDEX_PARTITION;
import static
org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage.METASTORAGE_CACHE_NAME;
/**
* Test cases when {@link CacheConfiguration#setStoragePaths(String...)} used
to set custom data region storage path.
*/
public class CacheConfigStoragePathTest extends
AbstractDataRegionRelativeStoragePathTest {
-
/** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String
igniteInstanceName) throws Exception {
- DataStorageConfiguration dsCfg = new DataStorageConfiguration()
- .setExtraStoragePaths(storagePath(STORAGE_PATH),
storagePath(STORAGE_PATH_2));
-
- dsCfg.getDefaultDataRegionConfiguration().setPersistenceEnabled(true);
-
- return super.getConfiguration(igniteInstanceName)
- .setConsistentId(U.maskForFileName(igniteInstanceName))
- .setDataStorageConfiguration(dsCfg)
- .setCacheConfiguration(ccfgs());
+ @Override protected DataStorageConfiguration dataStorageConfiguration() {
+ return new DataStorageConfiguration()
+ .setExtraStoragePaths(storagePath(STORAGE_PATH),
storagePath(STORAGE_PATH_2), storagePath(IDX_PATH));
}
/** {@inheritDoc} */
@@ -75,6 +72,39 @@ public class CacheConfigStoragePathTest extends
AbstractDataRegionRelativeStorag
};
}
+ /** Sanity checks - all paths for all partitions are different and contain
cacheDir. */
+ @Test
+ public void testPathGeneration() throws Exception {
+ IgniteEx srv = startAndActivate();
+
+ NodeFileTree ft = srv.context().pdsFolderResolver().fileTree();
+
+ IntConsumer checkPart = i -> {
+ Set<File> parts = new HashSet<>();
+
+ Set<String> grps = new HashSet<>();
+
+ for (CacheConfiguration<?, ?> ccfg : ccfgs()) {
+ if (!grps.add(CU.cacheOrGroupName(ccfg)))
+ continue;
+
+ File part = ft.partitionFile(ccfg, i);
+
+
assertTrue(Arrays.asList(ft.cacheStorages(ccfg)).contains(part.getParentFile()));
+ assertTrue(parts.add(part));
+ }
+
+ assertEquals(grpCount(), grps.size());
+ };
+
+ for (int i = 0; i < PARTS_CNT; i++)
+ checkPart.accept(i);
+
+ checkPart.accept(INDEX_PARTITION);
+
+ stopAllGrids();
+ }
+
/** */
@Test
public void testCaches() throws Exception {
@@ -129,110 +159,152 @@ public class CacheConfigStoragePathTest extends
AbstractDataRegionRelativeStorag
* @param path Snapshot path.
*/
private void checkSnapshotFiles(String name, String path) {
- NodeFileTree ft = grid(0).context().pdsFolderResolver().fileTree();
+ // Only in "separate root" mode snapshots for nodes will have
different roots.
+ List<NodeFileTree> fts = pathMode == PathMode.SEPARATE_ROOT
+ ? Ignition.allGrids().stream().map(n ->
((IgniteEx)n).context().pdsFolderResolver().fileTree()).collect(Collectors.toList())
+ :
Collections.singletonList(grid(0).context().pdsFolderResolver().fileTree());
- Function<String, File> snpRootF = storage -> {
- File snpRoot;
+ Map<Integer, Set<Integer>> partsMap = new HashMap<>();
- if (path != null)
- snpRoot = new File(path);
- else if (storage == null)
- snpRoot = ft.snapshotsRoot();
- else {
- File nodeStorage = absPath
- ? new File(storage)
- : new File(ft.root(), storage);
+ for (NodeFileTree ft : fts) {
+ Function<String, File> snpRootF = storage -> {
+ File snpRoot;
+
+ if (path != null)
+ snpRoot = new File(path);
+ else if (storage == null)
+ snpRoot = ft.snapshotsRoot();
+ else {
+ File nodeStorage = pathMode == PathMode.ABS
+ ? new File(storage)
+ : new File(ft.root(), storage);
+
+ snpRoot = new File(nodeStorage, DFLT_SNAPSHOT_DIRECTORY);
+ }
- snpRoot = new File(nodeStorage, DFLT_SNAPSHOT_DIRECTORY);
- }
+ return new File(snpRoot, name);
+ };
- return new File(snpRoot, name);
- };
+ // Snapshot root directories.
+ Set<File> roots = new HashSet<>(Arrays.asList(
+ snpRootF.apply(null),
+ snpRootF.apply(storagePath(STORAGE_PATH)),
+ snpRootF.apply(storagePath(STORAGE_PATH_2))
+ ));
- // Snapshot root directories.
- Set<File> roots = new HashSet<>(Arrays.asList(
- snpRootF.apply(null),
- snpRootF.apply(storagePath(STORAGE_PATH)),
- snpRootF.apply(storagePath(STORAGE_PATH_2))
- ));
+ boolean idxPathUsed = idxStorage && idxPartMustExistsInSnapshot();
- assertEquals(path != null ? 1 : 3, roots.size());
+ if (idxPathUsed)
+ roots.add(snpRootF.apply(storagePath(IDX_PATH)));
- // Root -> cache -> partition set.
- Map<File, Map<String, Set<Integer>>> snpFiles = new HashMap<>();
+ // Sanity check storagePath returns different paths.
+ assertEquals(path != null ? 1 : (idxPathUsed ? 4 : 3),
roots.size());
- // Collecting all partition files under each snapshot root.
- roots.forEach(snpRoot -> {
- assertTrue(snpRoot.exists());
- assertTrue(snpRoot.isDirectory());
+ // Root -> cache -> partition set.
+ Map<File, Map<String, Set<Integer>>> snpFiles = new HashMap<>();
- try (Stream<Path> files = Files.walk(snpRoot.toPath())) {
- files.filter(p ->
NodeFileTree.partitionFile(p.toFile())).forEach(partFile -> {
- File root = roots.stream().filter(r ->
partFile.startsWith(r.toPath())).findFirst().orElseThrow();
+ // Collecting all partition files under each snapshot root.
+ for (File snpRoot : roots) {
+ assertTrue(snpRoot.exists());
+ assertTrue(snpRoot.isDirectory());
- String cacheName =
NodeFileTree.cacheName(partFile.getParent().toFile());
+ Predicate<Path> pathPredicate = p ->
NodeFileTree.partitionFile(p.toFile())
+ ||
p.getFileName().toString().equals(NodeFileTree.partitionFileName(INDEX_PARTITION));
- if (cacheName.equals(METASTORAGE_CACHE_NAME))
- return;
+ try (Stream<Path> files = Files.walk(snpRoot.toPath())) {
+ files.filter(pathPredicate).forEach(partFile -> {
+ File root = roots.stream().filter(r ->
partFile.startsWith(r.toPath())).findFirst().orElseThrow();
- int part = NodeFileTree.partId(partFile.toFile());
+ String cacheName =
NodeFileTree.cacheName(partFile.getParent().toFile());
- String[] cs = Arrays.stream(ccfgs())
- .filter(ccfg ->
CU.cacheOrGroupName(ccfg).equals(cacheName))
- .findFirst().orElseThrow().getStoragePaths();
+ if (cacheName.equals(METASTORAGE_CACHE_NAME))
+ return;
- File expStorage = snpRootF.apply(F.isEmpty(cs) ? null :
cs[part % cs.length]);
+ int part = NodeFileTree.partId(partFile.toFile());
- assertEquals(expStorage, root);
+ String[] cs = Arrays.stream(ccfgs())
+ .filter(ccfg ->
CU.cacheOrGroupName(ccfg).equals(cacheName))
+ .findFirst().orElseThrow().getStoragePaths();
- snpFiles
- .computeIfAbsent(root, r -> new HashMap<>())
- .computeIfAbsent(cacheName, c -> new HashSet<>())
- .add(part);
- });
- }
- catch (IOException e) {
- throw new RuntimeException(e);
+ File expStorage = (idxPathUsed && part ==
INDEX_PARTITION)
+ ? snpRootF.apply(storagePath(IDX_PATH))
+ : snpRootF.apply(F.isEmpty(cs) ? null : cs[part %
cs.length]);
+
+ assertEquals(expStorage, root);
+
+ snpFiles
+ .computeIfAbsent(root, r -> new HashMap<>())
+ .computeIfAbsent(cacheName, c -> new HashSet<>())
+ .add(part);
+ });
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
}
- });
- Set<String> seenGrps = new HashSet<>();
+ Set<String> seenGrps = new HashSet<>();
- Arrays.stream(ccfgs()).forEach(ccfg -> {
- String cname = CU.cacheOrGroupName(ccfg);
+ Arrays.stream(ccfgs()).forEach(ccfg -> {
+ String cname = CU.cacheOrGroupName(ccfg);
- if (seenGrps.contains(cname))
- return;
+ if (seenGrps.contains(cname))
+ return;
- seenGrps.add(cname);
+ seenGrps.add(cname);
- Set<Integer> parts = new HashSet<>();
+ Set<Integer> parts = new HashSet<>();
- String[] storagePaths = ccfg.getStoragePaths();
+ List<String> storagePaths = new
ArrayList<>(F.isEmpty(ccfg.getStoragePaths())
+ ? Collections.singletonList(null)
+ : Arrays.asList(ccfg.getStoragePaths()));
- if (F.isEmpty(storagePaths)) {
- storagePaths = new String[1];
- }
+ if (!F.isEmpty(ccfg.getIndexPath()) &&
idxPartMustExistsInSnapshot())
+ storagePaths.add(ccfg.getIndexPath());
- for (String storagePath : storagePaths) {
- File expRoot = snpRootF.apply(storagePath);
+ for (String storagePath : storagePaths) {
+ File expRoot = snpRootF.apply(storagePath);
- assertTrue(cname + " must be found",
snpFiles.containsKey(expRoot));
+ assertTrue(cname + " must be found",
snpFiles.containsKey(expRoot));
- parts.addAll(snpFiles.get(expRoot).get(cname));
- }
+ parts.addAll(snpFiles.get(expRoot).get(cname));
+ }
+
+ assertFalse(cname + " partitions must be found",
parts.isEmpty());
+
+ partsMap.computeIfAbsent(CU.cacheGroupId(ccfg), key -> new
HashSet<>()).addAll(parts);
+ });
+
+ assertEquals(grpCount(), seenGrps.size());
+ }
+
+ Arrays.stream(ccfgs()).forEach(ccfg -> {
+ String cname = CU.cacheOrGroupName(ccfg);
+
+ Set<Integer> parts = partsMap.get(CU.cacheGroupId(ccfg));
assertFalse(cname + " partitions must be found", parts.isEmpty());
- assertEquals("All partitions for " + cname + " must be found",
PARTS_CNT, parts.size());
+
+ assertEquals(
+ "All partitions for " + cname + " must be found",
+ PARTS_CNT + (idxPartMustExistsInSnapshot() ? 1 : 0),
+ parts.size()
+ );
IntStream.range(0, PARTS_CNT).forEach(i -> assertTrue(i + "
partition must be found", parts.contains(i)));
- });
- assertEquals(grpCount(), seenGrps.size());
+ if (idxPartMustExistsInSnapshot())
+ assertTrue(parts.contains(INDEX_PARTITION));
+ });
}
/** */
private int grpCount() {
return
Arrays.stream(ccfgs()).map(CU::cacheOrGroupName).collect(Collectors.toSet()).size();
}
+
+ /** */
+ protected boolean idxPartMustExistsInSnapshot() {
+ return false;
+ }
}
diff --git
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/filename/CustomCacheStorageConfigurationSelfTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/filename/CustomCacheStorageConfigurationSelfTest.java
index ff0f145b1a5..e52e6cb322c 100644
---
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/filename/CustomCacheStorageConfigurationSelfTest.java
+++
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/filename/CustomCacheStorageConfigurationSelfTest.java
@@ -111,6 +111,11 @@ public class CustomCacheStorageConfigurationSelfTest
extends GridCommonAbstractT
() -> srv.createCache(new
CacheConfiguration<>("my-cache").setStoragePaths("other")),
IgniteCheckedException.class
);
+
+ assertThrowsWithCause(
+ () -> srv.createCache(new
CacheConfiguration<>("my-cache").setIndexPath("other")),
+ IgniteCheckedException.class
+ );
};
try (IgniteEx srv = startGrid(new
IgniteConfiguration().setDataStorageConfiguration(new DataStorageConfiguration()
@@ -157,6 +162,13 @@ public class CustomCacheStorageConfigurationSelfTest
extends GridCommonAbstractT
IgniteCheckedException.class
);
+ assertThrowsWithCause(
+ () -> srv.createCache(new
CacheConfiguration<>("my-cache2").setGroupName("grp")
+ .setStoragePaths(myPath3.getAbsolutePath())
+ .setIndexPath(myPath.getAbsolutePath())),
+ IgniteCheckedException.class
+ );
+
srv.createCache(new CacheConfiguration<>("my-cache2")
.setGroupName("grp-2"));
@@ -174,6 +186,13 @@ public class CustomCacheStorageConfigurationSelfTest
extends GridCommonAbstractT
IgniteCheckedException.class
);
+ assertThrowsWithCause(
+ () -> srv.createCache(new CacheConfiguration<>("my-cache3")
+ .setGroupName("grp-2")
+ .setIndexPath(myPath.getAbsolutePath())),
+ IgniteCheckedException.class
+ );
+
srv.createCache(new CacheConfiguration<>("my-cache3")
.setStoragePaths(myPath2.getAbsolutePath(),
myPath3.getAbsolutePath())
.setGroupName("grp-3"));
@@ -191,6 +210,14 @@ public class CustomCacheStorageConfigurationSelfTest
extends GridCommonAbstractT
.setStoragePaths(myPath3.getAbsolutePath(),
myPath2.getAbsolutePath())),
IgniteCheckedException.class
);
+
+ assertThrowsWithCause(
+ () -> srv.createCache(new CacheConfiguration<>("my-cache4")
+ .setGroupName("grp-3")
+ .setStoragePaths(myPath2.getAbsolutePath(),
myPath3.getAbsolutePath())
+ .setIndexPath(myPath.getAbsolutePath())),
+ IgniteCheckedException.class
+ );
};
try (IgniteEx srv = startGrid(new
IgniteConfiguration().setDataStorageConfiguration(new DataStorageConfiguration()
@@ -227,6 +254,14 @@ public class CustomCacheStorageConfigurationSelfTest
extends GridCommonAbstractT
srv.createCache(new CacheConfiguration<>("my-cache2")
.setGroupName("grp").setStoragePaths(myPath3.getAbsolutePath()));
+
+ srv.createCache(new CacheConfiguration<>("my-cache3")
+
.setGroupName("grp2").setStoragePaths(myPath3.getAbsolutePath())
+ .setIndexPath(myPath.getAbsolutePath()));
+
+ srv.createCache(new CacheConfiguration<>("my-cache4")
+
.setGroupName("grp2").setStoragePaths(myPath3.getAbsolutePath())
+ .setIndexPath(myPath.getAbsolutePath()));
}
}
@@ -238,7 +273,8 @@ public class CustomCacheStorageConfigurationSelfTest
extends GridCommonAbstractT
.setDefaultDataRegionConfiguration(new
DataRegionConfiguration().setPersistenceEnabled(true));
CacheConfiguration<Object, Object> ccfg = new
CacheConfiguration<>(DEFAULT_CACHE_NAME)
- .setStoragePaths(myPath.getAbsolutePath());
+ .setStoragePaths(myPath.getAbsolutePath())
+ .setIndexPath(myPath.getAbsolutePath());
try (IgniteEx srv = startGrid(getConfiguration("srv")
.setDataStorageConfiguration(dsCfg)
diff --git
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/filename/SnapshotCreationNonDefaultStoragePathTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/filename/SnapshotCreationNonDefaultStoragePathTest.java
index e22d7a0cd11..b670774e5b1 100644
---
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/filename/SnapshotCreationNonDefaultStoragePathTest.java
+++
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/filename/SnapshotCreationNonDefaultStoragePathTest.java
@@ -18,18 +18,21 @@
package org.apache.ignite.internal.processors.cache.persistence.filename;
import java.io.File;
+import java.nio.file.Path;
import java.util.List;
import java.util.function.BiConsumer;
+import org.apache.commons.io.FileUtils;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteException;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.DataStorageConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.util.typedef.G;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.junit.Test;
+import static
org.apache.ignite.configuration.IgniteConfiguration.DFLT_SNAPSHOT_DIRECTORY;
+import static
org.apache.ignite.testframework.GridTestUtils.assertThrowsAnyCause;
import static
org.apache.ignite.testframework.GridTestUtils.assertThrowsWithCause;
/**
@@ -37,18 +40,10 @@ import static
org.apache.ignite.testframework.GridTestUtils.assertThrowsWithCaus
*/
public class SnapshotCreationNonDefaultStoragePathTest extends
AbstractDataRegionRelativeStoragePathTest {
/** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String
igniteInstanceName) throws Exception {
- DataStorageConfiguration dsCfg = new DataStorageConfiguration()
+ @Override protected DataStorageConfiguration dataStorageConfiguration() {
+ return new DataStorageConfiguration()
.setStoragePath(storagePath(STORAGE_PATH))
- .setExtraStoragePaths(storagePath(STORAGE_PATH_2));
-
- dsCfg.getDefaultDataRegionConfiguration()
- .setPersistenceEnabled(true);
-
- return super.getConfiguration(igniteInstanceName)
- .setConsistentId(U.maskForFileName(igniteInstanceName))
- .setDataStorageConfiguration(dsCfg)
- .setCacheConfiguration(ccfgs());
+ .setExtraStoragePaths(storagePath(STORAGE_PATH_2),
storagePath(IDX_PATH));
}
/** {@inheritDoc} */
@@ -126,6 +121,8 @@ public class SnapshotCreationNonDefaultStoragePathTest
extends AbstractDataRegio
srv.context().cache().context().snapshotMgr().createSnapshot("mysnp2",
fullPathSnp.getAbsolutePath(), false, false).get();
+ String grid1ConsId = consId(grid(1).configuration());
+
stopGrid(1);
resetBaselineTopology();
@@ -146,6 +143,32 @@ public class SnapshotCreationNonDefaultStoragePathTest
extends AbstractDataRegio
checkDataExists();
};
+ if (pathMode == PathMode.SEPARATE_ROOT) {
+ // Must move snapshot file from grid1 to other so it can be
restored.
+ assertThrowsAnyCause(
+ log,
+ () -> {
+ check.accept("mysnp", null);
+ return null;
+ },
+ IgniteException.class,
+ "No snapshot metadatas found for the baseline nodes with
consistent ids: "
+ );
+
+ Path[] copyStoppedNodeData = new Path[] {
+ Path.of(DFLT_SNAPSHOT_DIRECTORY, "mysnp"),
+ Path.of(STORAGE_PATH_2, DFLT_SNAPSHOT_DIRECTORY, "mysnp"),
+ Path.of(IDX_PATH, DFLT_SNAPSHOT_DIRECTORY, "mysnp")
+ };
+
+ for (Path copyPath : copyStoppedNodeData) {
+ FileUtils.copyDirectory(
+ Path.of(U.defaultWorkDirectory(), grid1ConsId,
copyPath.toString()).toFile(),
+ Path.of(U.defaultWorkDirectory(),
consId(grid(0).configuration()), copyPath.toString()).toFile()
+ );
+ }
+ }
+
check.accept("mysnp", null);
check.accept("mysnp2", fullPathSnp.getAbsolutePath());
}
@@ -158,7 +181,7 @@ public class SnapshotCreationNonDefaultStoragePathTest
extends AbstractDataRegio
for (String cs : ccfg.getStoragePaths()) {
- File customRoot = ensureExists(absPath
+ File customRoot = ensureExists(pathMode == PathMode.ABS
? new File(cs)
: new File(ft.root(), cs)
);
diff --git
a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/filename/SQLCacheConfigStoragePathTest.java
b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/filename/SQLCacheConfigStoragePathTest.java
index eb48daab65d..7517d4dea0d 100644
---
a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/filename/SQLCacheConfigStoragePathTest.java
+++
b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/filename/SQLCacheConfigStoragePathTest.java
@@ -89,6 +89,22 @@ public class SQLCacheConfigStoragePathTest extends
CacheConfigStoragePathTest {
}
+ /** {@inheritDoc} */
+ @Override void restoreAndCheck(String name, String path) throws Exception {
+ super.restoreAndCheck(name, path);
+
+ for (int i = 0; i < GRID_CNT; i++) {
+ List<List<?>> rows = executeSql(grid(0), "SELECT TABLE_NAME FROM
SYS.TABLES WHERE IS_INDEX_REBUILD_IN_PROGRESS = 'TRUE'");
+
+ assertTrue(rows.isEmpty());
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean idxPartMustExistsInSnapshot() {
+ return true;
+ }
+
/** */
private static String dep(int i) {
return i + " dep";
