This is an automated email from the ASF dual-hosted git repository.
petrov-mg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 89bccc08fa8 IGNITE-29131 Fixed idle_verify failure with
CorruptedTreeException during node shutdown or cache stop (#13686)
89bccc08fa8 is described below
commit 89bccc08fa86cb050d42af2d389ed6fb8f0319c1
Author: Vladimir Steshin <[email protected]>
AuthorDate: Thu Oct 29 12:19:54 2026 +0300
IGNITE-29131 Fixed idle_verify failure with CorruptedTreeException during
node shutdown or cache stop (#13686)
---
.../cache/VerifyBackupPartitionsTask.java | 200 ++++++++++-----
.../cache/IgniteCacheOffheapManager.java | 15 ++
.../cache/IgniteCacheOffheapManagerImpl.java | 10 +
.../cache/persistence/pagemem/PageMemoryImpl.java | 30 +--
.../snapshot/SnapshotPartitionsVerifyHandler.java | 8 +-
.../processors/cache/verify/IdleVerifyUtility.java | 23 +-
.../internal/processors/pool/PoolProcessor.java | 7 +-
.../NodeOrCacheStoppingDuringIdleVerifyTest.java | 274 +++++++++++++++++++++
.../junits/common/GridCommonAbstractTest.java | 9 +-
.../ignite/testsuites/IgnitePdsTestSuite3.java | 2 +
10 files changed, 483 insertions(+), 95 deletions(-)
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/management/cache/VerifyBackupPartitionsTask.java
b/modules/core/src/main/java/org/apache/ignite/internal/management/cache/VerifyBackupPartitionsTask.java
index 2330c00334e..4913a0690e2 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/management/cache/VerifyBackupPartitionsTask.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/management/cache/VerifyBackupPartitionsTask.java
@@ -85,12 +85,21 @@ public class VerifyBackupPartitionsTask extends
ComputeTaskAdapter<CacheIdleVeri
/** */
private static final long serialVersionUID = 0L;
+ /** */
+ private static final int PARTITION_FUTURE_TIMEOUT = 100;
+
+ /** */
+ private static final long LOG_PROGRESS_TIMEOUT = 3 * 60 * 1000L;
+
/** Error thrown when idle_verify is called on an inactive cluster with
persistence. */
public static final String IDLE_VERIFY_ON_INACTIVE_CLUSTER_ERROR_MESSAGE =
"Cannot perform the operation because " +
"the cluster is inactive.";
/** */
- public static final String CACL_PART_HASH_ERR_MSG = "Can't calculate
partition hash";
+ public static final String CALC_PART_HASH_ERR_MSG = "Can't calculate
partition hash";
+
+ /** */
+ private static final String CALC_CANCELLED_ERR_MSG = "Partition hash
calculation has been cancelled";
/** Checkpoint reason. */
public static final String CP_REASON = "VerifyBackupPartitions";
@@ -206,7 +215,7 @@ public class VerifyBackupPartitionsTask extends
ComputeTaskAdapter<CacheIdleVeri
}
if (isCancelled())
- throw new IgniteException(getClass().getSimpleName() + " was
cancelled.");
+ throw new IgniteException(CALC_CANCELLED_ERR_MSG);
Set<Integer> grpIds = getGroupIds();
@@ -217,81 +226,102 @@ public class VerifyBackupPartitionsTask extends
ComputeTaskAdapter<CacheIdleVeri
completionCntr.set(0);
- List<Future<Map<PartitionKey, PartitionHashRecord>>>
partHashCalcFuts =
- calcPartitionHashAsync(grpIds);
+ IgniteException jobFailure = null;
+ List<IgniteException> exceptions = null;
+ boolean interrupted = false;
Map<PartitionKey, PartitionHashRecord> res = new HashMap<>();
- List<IgniteException> exceptions = new ArrayList<>();
-
long lastProgressLogTs = U.currentTimeMillis();
- for (int i = 0; i < partHashCalcFuts.size(); ) {
- if (isCancelled()) {
- cancelFuts(i, partHashCalcFuts);
-
- throw new IgniteException(getClass().getSimpleName() + "
was cancelled.");
- }
+ List<Future<Map<PartitionKey, PartitionHashRecord>>>
partHashCalcFuts = calcPartitionHashAsync(grpIds);
+ for (int i = 0; i < partHashCalcFuts.size(); i++) {
Future<Map<PartitionKey, PartitionHashRecord>> fut =
partHashCalcFuts.get(i);
try {
- Map<PartitionKey, PartitionHashRecord> partHash =
fut.get(100, TimeUnit.MILLISECONDS);
+ Map<PartitionKey, PartitionHashRecord> partHash =
fut.get(PARTITION_FUTURE_TIMEOUT, TimeUnit.MILLISECONDS);
- res.putAll(partHash);
-
- i++;
+ if (jobFailure == null)
+ res.putAll(partHash);
}
- catch (InterruptedException | ExecutionException e) {
+ catch (ExecutionException e) {
if (e.getCause() instanceof IgniteException &&
!(e.getCause() instanceof GridNotIdleException)) {
- exceptions.add((IgniteException)e.getCause());
+ if (exceptions == null)
+ exceptions = new ArrayList<>();
- i++;
+ exceptions.add((IgniteException)e.getCause());
+ }
+ else if (jobFailure == null) {
+ jobFailure = new IgniteException("Failed to execute a
partition hash calculation [grpIds=" +
+ grpIds + ", reason=" + e.getCause().getMessage() +
']', e.getCause());
- continue;
+ cancel();
}
+ else
+ jobFailure.addSuppressed(e.getCause());
+ }
+ catch (InterruptedException e) {
+ interrupted = true;
- cancelFuts(i, partHashCalcFuts);
+ if (jobFailure == null) {
+ jobFailure = new IgniteInterruptedException("Partition
hash calculation has been interrupted " +
+ "[grpIds=" + grpIds + "].", e);
- if (e instanceof InterruptedException)
- throw new
IgniteInterruptedException((InterruptedException)e);
+ cancel();
+ }
else
- throw new IgniteException(e.getCause());
+ jobFailure.addSuppressed(e);
}
- catch (TimeoutException ignored) {
- if (U.currentTimeMillis() - lastProgressLogTs > 3 * 60 *
1000L) {
+ catch (TimeoutException e) {
+ if (U.currentTimeMillis() - lastProgressLogTs >
LOG_PROGRESS_TIMEOUT) {
lastProgressLogTs = U.currentTimeMillis();
log.warning("idle_verify is still running, processed "
+ completionCntr.get() + " of " +
partHashCalcFuts.size() + " local partitions");
}
+
+ --i;
}
}
- if (log.isInfoEnabled())
- log.info("Idle verify procedure has finished.");
+ if (interrupted)
+ Thread.currentThread().interrupt();
+
+ if (jobFailure != null)
+ throw jobFailure;
+
+ if (isCancelled())
+ throw new IgniteException(CALC_CANCELLED_ERR_MSG + " [grpIds="
+ grpIds + "].");
+
+ if (!F.isEmpty(exceptions)) {
+ if (ignite.context().isStopping()) {
+ throw new IgniteException(CALC_CANCELLED_ERR_MSG + "
because the local node is stopping [grpIds=" +
+ grpIds + "].");
+ }
+
+ Integer stoppingGrpId = F.find(grpIds, null,
this::isGroupStopping);
+
+ if (stoppingGrpId != null) {
+ throw new IgniteException(CALC_CANCELLED_ERR_MSG + "
because the cache group is stopping [grpId=" +
+ stoppingGrpId + "].");
+ }
- if (!F.isEmpty(exceptions))
throw new IdleVerifyException(exceptions);
+ }
- return res;
- }
+ if (log.isInfoEnabled()) {
+ log.info("Idle verify procedure has finished [skipZeros=" +
arg.skipZeros() + ", checkCrc=" +
+ arg.checkCrc() + ", grpIds=" + grpIds + "].");
+ }
- /**
- * Cancels futures from given index.
- *
- * @param start Index to start from.
- * @param partHashCalcFuts Partitions hash calculation futures to
cancel.
- */
- private void cancelFuts(int start, List<Future<Map<PartitionKey,
PartitionHashRecord>>> partHashCalcFuts) {
- for (int i = start; i < partHashCalcFuts.size(); i++)
- partHashCalcFuts.get(i).cancel(false);
+ return res;
}
/**
* Class that processes cache filtering chain.
*/
- private class CachesFiltering {
+ private static class CachesFiltering {
/**
* Initially all cache descriptors are included.
*/
@@ -489,6 +519,9 @@ public class VerifyBackupPartitionsTask extends
ComputeTaskAdapter<CacheIdleVeri
final BooleanSupplier cancelled
) {
return pool.submit(() -> {
+ if (isCancelledOrStopped(cancelled, gctx))
+ throw cancellationException(gctx, part);
+
Map<PartitionKey, PartitionHashRecord> res = emptyMap();
if (!part.reserve())
@@ -497,30 +530,42 @@ public class VerifyBackupPartitionsTask extends
ComputeTaskAdapter<CacheIdleVeri
try {
PartitionUpdateCounter updCntr =
part.dataStore().partUpdateCounter();
PartitionUpdateCounter updateCntrBefore = updCntr == null
? null : updCntr.copy();
-
- if (arg.checkCrc() && gctx.persistenceEnabled()) {
- FilePageStoreManager pageStoreMgr =
-
(FilePageStoreManager)ignite.context().cache().context().pageStore();
-
- checkPartitionsPageCrcSum(
- () ->
(FilePageStore)pageStoreMgr.getStore(gctx.groupId(), part.id()),
- part.id(),
- FLAG_DATA,
- cancelled
+ PartitionKey key = new PartitionKey(gctx.groupId(),
part.id(), gctx.cacheOrGroupName());
+ PartitionHashRecord hash;
+
+ // The cache group can be destroyed concurrently.
+ if (!gctx.offheap().enterBusy())
+ throw cancellationException(gctx, part);
+
+ try {
+ // Also hold the lock during the read to avoid a
concurrent deletion of the partition
+ // file when the cache is destroyed.
+ if (arg.checkCrc() && gctx.persistenceEnabled()) {
+ FilePageStoreManager pageStoreMgr =
+
(FilePageStoreManager)ignite.context().cache().context().pageStore();
+
+ checkPartitionsPageCrcSum(
+ () ->
(FilePageStore)pageStoreMgr.getStore(gctx.groupId(), part.id()),
+ part.id(),
+ FLAG_DATA,
+ () -> isCancelledOrStopped(cancelled, gctx)
+ );
+ }
+
+ hash = calculatePartitionHash(
+ key,
+ updateCntrBefore == null ? 0 :
updateCntrBefore.comparableState(),
+
ignite.context().discovery().localNode().consistentId(),
+ part.state(),
+
part.primary(gctx.topology().readyTopologyVersion()),
+ part.dataStore().fullSize(),
+ gctx.offheap().partitionIterator(part.id()),
+ () -> isCancelledOrStopped(cancelled, gctx)
);
}
-
- PartitionKey key = new PartitionKey(gctx.groupId(),
part.id(), gctx.cacheOrGroupName());
-
- PartitionHashRecord hash = calculatePartitionHash(key,
- updateCntrBefore == null ? 0 :
updateCntrBefore.comparableState(),
-
ignite.context().discovery().localNode().consistentId(),
- part.state(),
- part.primary(gctx.topology().readyTopologyVersion()),
- part.dataStore().fullSize(),
- gctx.offheap().partitionIterator(part.id()),
- cancelled
- );
+ finally {
+ gctx.offheap().leaveBusy();
+ }
if (hash != null)
res = Collections.singletonMap(key, hash);
@@ -534,7 +579,8 @@ public class VerifyBackupPartitionsTask extends
ComputeTaskAdapter<CacheIdleVeri
}
}
catch (IgniteCheckedException e) {
- String msg = CACL_PART_HASH_ERR_MSG + " [grpId=" +
gctx.groupId() + ", partId=" + part.id() + "]";
+ String msg = CALC_PART_HASH_ERR_MSG + " [grpId=" +
gctx.groupId() + ", partId=" + part.id() +
+ ", reason=" + e.getMessage() + "]";
U.error(log, msg, e);
@@ -549,5 +595,31 @@ public class VerifyBackupPartitionsTask extends
ComputeTaskAdapter<CacheIdleVeri
return res;
});
}
+
+ /**
+ * Checks whether the work should be stopped: the task was cancelled,
the local node is stopping,
+ * or the cache group is stopping or being destroyed.
+ *
+ * @param cancelled Supplier that returns {@code true} if the task
cancellation was requested.
+ * @param gctx Cache group context.
+ * @return {@code true} if the work should be stopped.
+ */
+ private boolean isCancelledOrStopped(BooleanSupplier cancelled,
CacheGroupContext gctx) {
+ return cancelled.getAsBoolean() || gctx.isPreparedToStop() ||
ignite.context().isStopping();
+ }
+
+ /** */
+ private boolean isGroupStopping(int grpId) {
+ CacheGroupContext grp = ignite.context().cache().cacheGroup(grpId);
+
+ return grp == null || grp.isPreparedToStop();
+ }
+
+ /** */
+ private static IgniteException cancellationException(CacheGroupContext
gctx, GridDhtLocalPartition part) {
+ return new IgniteException(CALC_CANCELLED_ERR_MSG + ". The task
has been cancelled, or the local node is stopping, " +
+ "or the cache group is stopping [groupId=" + gctx.groupId() +
+ ", groupName=" + gctx.cacheOrGroupName() + ", partId=" +
part.id() + "].");
+ }
}
}
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
index 54884f20bb7..2ddd55889e6 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
@@ -408,6 +408,21 @@ public interface IgniteCacheOffheapManager {
*/
public void preloadPartition(int pardId) throws IgniteCheckedException;
+ /**
+ * Tries to lock the manager against incompatible state like node or cache
group stopping.
+ *
+ * @return {@code true} if successful, {@code false} if the manager is
already in an incompatible state.
+ * @see #leaveBusy()
+ */
+ public boolean enterBusy();
+
+ /**
+ * Releases the 'busy' state.
+ *
+ * @see #enterBusy()
+ */
+ public void leaveBusy();
+
/**
*
*/
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
index 31a01344897..e99e2347b71 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
@@ -190,6 +190,16 @@ public class IgniteCacheOffheapManagerImpl implements
IgniteCacheOffheapManager
// No-op.
}
+ /** {@inheritDoc} */
+ @Override public boolean enterBusy() {
+ return busyLock.enterBusy();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void leaveBusy() {
+ busyLock.leaveBusy();
+ }
+
/** {@inheritDoc} */
@Override public void stopCache(int cacheId, boolean destroy) {
if (destroy && grp.affinityNode())
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index 6508c71da3b..36bf64cc1f0 100755
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -474,7 +474,7 @@ public class PageMemoryImpl implements PageMemoryEx {
/** {@inheritDoc} */
@Override public void releasePage(int grpId, long pageId, long page) {
- assert started;
+ assert started : "Page memory is stopped";
Segment seg = segment(grpId, pageId);
@@ -490,35 +490,35 @@ public class PageMemoryImpl implements PageMemoryEx {
/** {@inheritDoc} */
@Override public long readLock(int grpId, long pageId, long page) {
- assert started;
+ assert started : "Page memory is stopped";
return readLock(page, pageId, false);
}
/** {@inheritDoc} */
@Override public void readUnlock(int grpId, long pageId, long page) {
- assert started;
+ assert started : "Page memory is stopped";
readUnlockPage(page);
}
/** {@inheritDoc} */
@Override public long writeLock(int grpId, long pageId, long page) {
- assert started;
+ assert started : "Page memory is stopped";
return writeLock(grpId, pageId, page, false);
}
/** {@inheritDoc} */
@Override public long writeLock(int grpId, long pageId, long page, boolean
restore) {
- assert started;
+ assert started : "Page memory is stopped";
return writeLockPage(page, new FullPageId(pageId, grpId), !restore);
}
/** {@inheritDoc} */
@Override public long tryWriteLock(int grpId, long pageId, long page) {
- assert started;
+ assert started : "Page memory is stopped";
return tryWriteLockPage(page, new FullPageId(pageId, grpId), true);
}
@@ -526,7 +526,7 @@ public class PageMemoryImpl implements PageMemoryEx {
/** {@inheritDoc} */
@Override public void writeUnlock(int grpId, long pageId, long page,
Boolean walPlc,
boolean dirtyFlag) {
- assert started;
+ assert started : "Page memory is stopped";
writeUnlock(grpId, pageId, page, walPlc, dirtyFlag, false);
}
@@ -534,14 +534,14 @@ public class PageMemoryImpl implements PageMemoryEx {
/** {@inheritDoc} */
@Override public void writeUnlock(int grpId, long pageId, long page,
Boolean walPlc,
boolean dirtyFlag, boolean restore) {
- assert started;
+ assert started : "Page memory is stopped";
writeUnlockPage(page, new FullPageId(pageId, grpId), walPlc,
dirtyFlag, restore);
}
/** {@inheritDoc} */
@Override public boolean isDirty(int grpId, long pageId, long page) {
- assert started;
+ assert started : "Page memory is stopped";
return isDirty(page);
}
@@ -552,7 +552,7 @@ public class PageMemoryImpl implements PageMemoryEx {
flags == PageIdAllocator.FLAG_IDX && partId ==
PageIdAllocator.INDEX_PARTITION :
"flags = " + flags + ", partId = " + partId;
- assert started;
+ assert started : "Page memory is stopped";
assert stateChecker.checkpointLockIsHeldByThread();
if (isThrottlingEnabled())
@@ -697,7 +697,7 @@ public class PageMemoryImpl implements PageMemoryEx {
/** {@inheritDoc} */
@Override public long partitionMetaPageId(int grpId, int partId) {
- assert started;
+ assert started : "Page memory is stopped";
return PageIdUtils.pageId(partId, PageIdAllocator.FLAG_DATA, 0);
}
@@ -710,7 +710,7 @@ public class PageMemoryImpl implements PageMemoryEx {
/** {@inheritDoc} */
@Override public long acquirePage(int grpId, long pageId,
IoStatisticsHolder statHolder) throws IgniteCheckedException {
- assert started;
+ assert started : "Page memory is stopped";
return acquirePage(grpId, pageId, statHolder, false);
}
@@ -735,7 +735,7 @@ public class PageMemoryImpl implements PageMemoryEx {
*/
private long acquirePage(int grpId, long pageId, IoStatisticsHolder
statHolder,
boolean restore, @Nullable AtomicBoolean pageAllocated) throws
IgniteCheckedException {
- assert started;
+ assert started : "Page memory is stopped";
int partId = PageIdUtils.partId(pageId);
@@ -1582,7 +1582,7 @@ public class PageMemoryImpl implements PageMemoryEx {
/** {@inheritDoc} */
@Override public long readLock(long absPtr, long pageId, boolean force,
boolean touch) {
- assert started;
+ assert started : "Page memory is stopped";
int tag = force ? -1 : PageIdUtils.tag(pageId);
@@ -1601,7 +1601,7 @@ public class PageMemoryImpl implements PageMemoryEx {
/** {@inheritDoc} */
@Override public long readLockForce(int grpId, long pageId, long page) {
- assert started;
+ assert started : "Page memory is stopped";
return readLock(page, pageId, true);
}
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyHandler.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyHandler.java
index 84c79b88cc6..16562996ef9 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyHandler.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyHandler.java
@@ -304,15 +304,17 @@ public class SnapshotPartitionsVerifyHandler implements
SnapshotHandler<Snapshot
// There is no `primary` partitions for snapshot.
PartitionKey key = new PartitionKey(grpId, partId,
grpName);
- PartitionHashRecord hash = calculatePartitionHash(key,
+ PartitionHashRecord hash = calculatePartitionHash(
+ key,
updateCntr,
meta.consistentId(),
GridDhtPartitionState.OWNING,
false,
size,
- skipHash() ? F.emptyIterator()
+ skipHash()
+ ? F.emptyIterator()
: snpMgr.partitionRowIterator(snpCtx, grpName,
partId, pageStore),
- null
+ () -> cctx.kernalContext().isStopping()
);
assert hash != null : "OWNING must have hash: " + key;
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java
index 3cccbb3e72a..6a42f7138b2 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java
@@ -122,7 +122,7 @@ public class IdleVerifyUtility {
for (int pageNo = 0; pageNo < pageStore.pages(); pageId++,
pageNo++) {
if (cancelled != null && cancelled.getAsBoolean())
- throw new IgniteException("Checking of partitions page CRC
sum has been cancelled.");
+ throw new IgniteException("Checking of partitions page CRC
sum has been cancelled [partId=" + partId + ']');
buf.clear();
@@ -288,17 +288,18 @@ public class IdleVerifyUtility {
boolean isPrimary,
long partSize,
GridIterator<CacheDataRow> it,
- @Nullable BooleanSupplier cancelled
+ BooleanSupplier cancelled
) throws IgniteCheckedException {
if (state == GridDhtPartitionState.MOVING || state ==
GridDhtPartitionState.LOST) {
- return new PartitionHashRecord(partKey,
+ return new PartitionHashRecord(
+ partKey,
isPrimary,
consId,
updCntr,
- state == GridDhtPartitionState.MOVING ?
- PartitionHashRecord.MOVING_PARTITION_SIZE : 0,
- state == GridDhtPartitionState.MOVING ?
- PartitionHashRecord.PartitionState.MOVING :
PartitionHashRecord.PartitionState.LOST,
+ state == GridDhtPartitionState.MOVING ?
PartitionHashRecord.MOVING_PARTITION_SIZE : 0,
+ state == GridDhtPartitionState.MOVING
+ ? PartitionHashRecord.PartitionState.MOVING
+ : PartitionHashRecord.PartitionState.LOST,
new VerifyPartitionContext()
);
}
@@ -308,10 +309,9 @@ public class IdleVerifyUtility {
VerifyPartitionContext ctx = new VerifyPartitionContext();
- while (it.hasNextX()) {
- if (cancelled != null && cancelled.getAsBoolean())
- throw new IgniteCheckedException("Partition hash calculation
has been cancelled.");
+ boolean cancelled0;
+ while (!(cancelled0 = cancelled.getAsBoolean()) && it.hasNextX()) {
CacheDataRow row = it.nextX();
if (row.expireTime() > 0)
@@ -320,6 +320,9 @@ public class IdleVerifyUtility {
ctx.update(row.key(), row.value(), row.version());
}
+ if (cancelled0)
+ throw new IgniteException("Partition hash calculation has been
cancelled [part=" + partKey + ']');
+
return new PartitionHashRecord(
partKey,
isPrimary,
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/pool/PoolProcessor.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/pool/PoolProcessor.java
index bedbb96bf66..5f264a7ec81 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/pool/PoolProcessor.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/pool/PoolProcessor.java
@@ -145,10 +145,13 @@ public class PoolProcessor extends GridProcessorAdapter {
*/
public static final long DFLT_PERIODIC_STARVATION_CHECK_FREQ = 30 * 1000L;
+ /** */
+ public static final String POOL_NAME_IDLE_VERIFY = "idleVerify";
+
/** */
@SystemProperty(value = "Idle verify utility thread pool size.", type =
Integer.class,
defaults = "Total available CPUs minus 2, but at least 4 threads.")
- public static final String IDLE_VERIFY_POOL_SIZE_PROPERTY =
"IGNITE_IDLE_VERIFY_POOL_SIZE";
+ private static final String IDLE_VERIFY_POOL_SIZE_PROPERTY =
"IGNITE_IDLE_VERIFY_POOL_SIZE";
/** */
private static final int DFLT_IDLE_VERIFY_POOL_SIZE = Math.max(4,
IgniteConfiguration.AVAILABLE_PROC_CNT - 2);
@@ -395,7 +398,7 @@ public class PoolProcessor extends GridProcessorAdapter {
validateThreadPoolSize(idleVerifyPoolSz, "idle verify");
idleVerifyExecSvc = new IgniteThreadPoolExecutor(
- "idleVerify",
+ POOL_NAME_IDLE_VERIFY,
cfg.getIgniteInstanceName(),
idleVerifyPoolSz,
idleVerifyPoolSz,
diff --git
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/NodeOrCacheStoppingDuringIdleVerifyTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/NodeOrCacheStoppingDuringIdleVerifyTest.java
new file mode 100644
index 00000000000..6409082fffb
--- /dev/null
+++
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/NodeOrCacheStoppingDuringIdleVerifyTest.java
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.OpenOption;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cluster.ClusterState;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.TestFailureHandler;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.management.cache.IdleVerifyResult;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import
org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator;
+import
org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import
org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
+import org.apache.ignite.internal.processors.cache.verify.GridNotIdleException;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import static
org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree.partitionFileName;
+import static
org.apache.ignite.internal.processors.pool.PoolProcessor.POOL_NAME_IDLE_VERIFY;
+import static org.apache.ignite.testframework.GridTestUtils.runAsync;
+import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
+
+/** */
+@RunWith(Parameterized.class)
+public class NodeOrCacheStoppingDuringIdleVerifyTest extends
GridCommonAbstractTest {
+ /** */
+ private static final int PARTS = 4;
+
+ /** */
+ private static final int KEYS = PARTS * 200;
+
+ /** */
+ private static final int NOT_IDLE_PART = 0;
+
+ /** */
+ private static volatile CountDownLatch partitionReadBlocked;
+
+ /** */
+ private static volatile CountDownLatch partitionReadUnblocked;
+
+ /** */
+ private static volatile CountDownLatch notIdlePartReadUnblocked;
+
+ /** */
+ @Parameterized.Parameter
+ public boolean checkCrc;
+
+ /** */
+ private TestFailureHandler failureHnd;
+
+ /** */
+ @Parameterized.Parameters(name = "checkCrc={0}")
+ public static Iterable<Object> parameters() {
+ return F.asList(false, true);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String
igniteInstanceName) throws Exception {
+ return super.getConfiguration(igniteInstanceName)
+ .setDataStorageConfiguration(new DataStorageConfiguration()
+ .setFileIOFactory(new IdleVerifyThreadsBlockingFileIOFactory())
+ .setDefaultDataRegionConfiguration(new
DataRegionConfiguration()
+ .setPersistenceEnabled(true)))
+ .setFailureHandler(failureHnd);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ cleanPersistenceDir();
+
+ failureHnd = new TestFailureHandler(false);
+
+ partitionReadBlocked = new CountDownLatch(PARTS);
+ partitionReadUnblocked = new CountDownLatch(1);
+ notIdlePartReadUnblocked = new CountDownLatch(1);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ unblockPartitionRead();
+
+ stopAllGrids();
+
+ cleanPersistenceDir();
+
+ super.afterTest();
+ }
+
+ /** */
+ @Test
+ public void testWithCacheDestruction() throws Exception {
+ prepareCluster();
+
+ CacheGroupContext grp =
grid(0).context().cache().cacheGroup(CU.cacheId(DEFAULT_CACHE_NAME));
+
+ IgniteInternalFuture<IdleVerifyResult> idleFut = startIdleVerify();
+
+ IgniteInternalFuture<?> destroyFut = runAsync(() ->
grid(0).destroyCache(DEFAULT_CACHE_NAME));
+
+ assertTrue(waitForCondition(grp::isPreparedToStop, getTestTimeout()));
+
+ assertFalse(waitForCondition(destroyFut::isDone, 1_000));
+
+ unblockPartitionRead();
+
+ destroyFut.get(getTestTimeout());
+
+ assertFalse(F.isEmpty(idleFut.get(getTestTimeout()).exceptions()));
+ assertNull(failureHnd.failureContext());
+ }
+
+ /** */
+ @Test
+ public void testWithDeactivation() throws Exception {
+ prepareCluster();
+
+ CacheGroupContext grp =
grid(0).context().cache().cacheGroup(CU.cacheId(DEFAULT_CACHE_NAME));
+
+ IgniteInternalFuture<IdleVerifyResult> idleFut = startIdleVerify();
+
+ IgniteInternalFuture<?> deactivateFut = runAsync(() ->
grid(0).cluster().state(ClusterState.INACTIVE));
+
+ assertTrue(waitForCondition(grp::isPreparedToStop, getTestTimeout()));
+
+ assertFalse(waitForCondition(deactivateFut::isDone, 1_000));
+
+ unblockPartitionRead();
+
+ deactivateFut.get(getTestTimeout());
+
+ assertFalse(F.isEmpty(idleFut.get(getTestTimeout()).exceptions()));
+
+ grid(0).cluster().state(ClusterState.ACTIVE);
+
+ assertEquals(KEYS, grid(0).cache(DEFAULT_CACHE_NAME).size());
+ assertNull(failureHnd.failureContext());
+ }
+
+ /** */
+ @Test
+ public void testWithNodeStoppage() throws Exception {
+ prepareCluster();
+
+ IgniteCache<Integer, Integer> cache =
grid(0).cache(DEFAULT_CACHE_NAME);
+
+ int key = partitionKeys(cache, NOT_IDLE_PART, 1, 0).get(0);
+
+ cache.get(key);
+
+ IgniteInternalFuture<IdleVerifyResult> idleFut = startIdleVerify();
+
+ cache.put(key, -1);
+
+ notIdlePartReadUnblocked.countDown();
+
+ assertTrue(waitForCondition(
+ () ->
grid(0).context().pools().getIdleVerifyExecutorService().getCompletedTaskCount()
> 0,
+ getTestTimeout()
+ ));
+
+ GridKernalContext kctx = grid(0).context();
+
+ IgniteInternalFuture<?> stopFut = runAsync(() -> stopGrid(0, true));
+
+ assertTrue(waitForCondition(kctx::isStopping, getTestTimeout()));
+
+ unblockPartitionRead();
+
+ stopFut.get(getTestTimeout());
+
+ IdleVerifyResult res = idleFut.get(getTestTimeout());
+
+ assertTrue(X.hasCause(F.first(res.exceptions().values()),
GridNotIdleException.class));
+ assertNull(failureHnd.failureContext());
+ }
+
+ /** */
+ private void prepareCluster() throws Exception {
+ startGrid(0);
+
+ grid(0).cluster().state(ClusterState.ACTIVE);
+
+ IgniteCache<Integer, Integer> cache = grid(0).createCache(new
CacheConfiguration<Integer, Integer>(DEFAULT_CACHE_NAME)
+ .setAffinity(new RendezvousAffinityFunction(false, PARTS)));
+
+ for (int k = 0; k < KEYS; k++)
+ cache.put(k, k);
+
+ stopGrid(0);
+
+ startGrid(0);
+
+ grid(0).cluster().state(ClusterState.ACTIVE);
+ }
+
+ /** */
+ private IgniteInternalFuture<IdleVerifyResult> startIdleVerify() throws
Exception {
+ IgniteInternalFuture<IdleVerifyResult> fut = runAsync(() ->
idleVerify(grid(0), checkCrc, DEFAULT_CACHE_NAME));
+
+ assertTrue(partitionReadBlocked.await(getTestTimeout(),
TimeUnit.MILLISECONDS));
+
+ return fut;
+ }
+
+ /** */
+ private static void unblockPartitionRead() {
+ notIdlePartReadUnblocked.countDown();
+ partitionReadUnblocked.countDown();
+ }
+
+ /** */
+ private static class IdleVerifyThreadsBlockingFileIOFactory implements
FileIOFactory {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private final FileIOFactory delegate = new RandomAccessFileIOFactory();
+
+ /** {@inheritDoc} */
+ @Override public FileIO create(File file, OpenOption... modes) throws
IOException {
+ return new FileIODecorator(delegate.create(file, modes)) {
+ @Override public int readFully(ByteBuffer destBuf, long
position) throws IOException {
+ int res = super.readFully(destBuf, position);
+
+ if
(Thread.currentThread().getName().startsWith(POOL_NAME_IDLE_VERIFY)) {
+ partitionReadBlocked.countDown();
+
+
U.awaitQuiet(file.getName().equals(partitionFileName(NOT_IDLE_PART))
+ ? notIdlePartReadUnblocked
+ : partitionReadUnblocked);
+ }
+
+ return res;
+ }
+ };
+ }
+ }
+}
diff --git
a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
index 2ad409701d0..b923e6bd1d0 100755
---
a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
+++
b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
@@ -2209,11 +2209,12 @@ public abstract class GridCommonAbstractTest extends
GridAbstractTest {
* concurrently updated.
*
* @param ig Ignite instance.
+ * @param checkCrc Whether to check the CRC sums of partition pages.
* @param caches Cache names (if null, all user caches will be verified).
* @return Conflicts result.
* @throws IgniteException If none caches or node found.
*/
- protected IdleVerifyResult idleVerify(Ignite ig, @Nullable String...
caches) throws Exception {
+ protected IdleVerifyResult idleVerify(Ignite ig, boolean checkCrc,
@Nullable String... caches) throws Exception {
log.info("Starting idleVerify ...");
IgniteEx ig0 = (IgniteEx)ig;
@@ -2236,6 +2237,7 @@ public abstract class GridCommonAbstractTest extends
GridAbstractTest {
CacheIdleVerifyCommandArg taskArg = new CacheIdleVerifyCommandArg();
taskArg.caches(cacheNames.toArray(U.EMPTY_STRS));
+ taskArg.checkCrc(checkCrc);
return ((VisorTaskResult<IdleVerifyResult>)ig.compute().execute(
IdleVerifyTask.class.getName(),
@@ -2243,6 +2245,11 @@ public abstract class GridCommonAbstractTest extends
GridAbstractTest {
)).result();
}
+ /** @see #idleVerify(Ignite, boolean, String...) */
+ protected IdleVerifyResult idleVerify(Ignite ig, @Nullable String...
caches) throws Exception {
+ return idleVerify(ig, false, caches);
+ }
+
/**
* Checks if all txs and mvcc futures are finished.
*/
diff --git
a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite3.java
b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite3.java
index 3b93a139c37..2e2c6234f79 100644
---
a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite3.java
+++
b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite3.java
@@ -23,6 +23,7 @@ import java.util.List;
import
org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTest;
import
org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTestWithExpiryPolicy;
import
org.apache.ignite.internal.processors.cache.persistence.IgnitePdsNodeJoinWithCachesStopping;
+import
org.apache.ignite.internal.processors.cache.persistence.NodeOrCacheStoppingDuringIdleVerifyTest;
import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.DynamicSuite;
import org.junit.runner.RunWith;
@@ -47,6 +48,7 @@ public class IgnitePdsTestSuite3 {
GridTestUtils.addTestIfNeeded(suite,
IgnitePdsContinuousRestartTest.class, ignoredTests);
GridTestUtils.addTestIfNeeded(suite,
IgnitePdsContinuousRestartTestWithExpiryPolicy.class, ignoredTests);
GridTestUtils.addTestIfNeeded(suite,
IgnitePdsNodeJoinWithCachesStopping.class, ignoredTests);
+ GridTestUtils.addTestIfNeeded(suite,
NodeOrCacheStoppingDuringIdleVerifyTest.class, ignoredTests);
return suite;
}