This is an automated email from the ASF dual-hosted git repository.

petrov-mg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git


The following commit(s) were added to refs/heads/master by this push:
     new 89bccc08fa8 IGNITE-29131 Fixed idle_verify failure with 
CorruptedTreeException during node shutdown or cache stop (#13686)
89bccc08fa8 is described below

commit 89bccc08fa86cb050d42af2d389ed6fb8f0319c1
Author: Vladimir Steshin <[email protected]>
AuthorDate: Thu Oct 29 12:19:54 2026 +0300

    IGNITE-29131 Fixed idle_verify failure with CorruptedTreeException during 
node shutdown or cache stop (#13686)
---
 .../cache/VerifyBackupPartitionsTask.java          | 200 ++++++++++-----
 .../cache/IgniteCacheOffheapManager.java           |  15 ++
 .../cache/IgniteCacheOffheapManagerImpl.java       |  10 +
 .../cache/persistence/pagemem/PageMemoryImpl.java  |  30 +--
 .../snapshot/SnapshotPartitionsVerifyHandler.java  |   8 +-
 .../processors/cache/verify/IdleVerifyUtility.java |  23 +-
 .../internal/processors/pool/PoolProcessor.java    |   7 +-
 .../NodeOrCacheStoppingDuringIdleVerifyTest.java   | 274 +++++++++++++++++++++
 .../junits/common/GridCommonAbstractTest.java      |   9 +-
 .../ignite/testsuites/IgnitePdsTestSuite3.java     |   2 +
 10 files changed, 483 insertions(+), 95 deletions(-)

diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/management/cache/VerifyBackupPartitionsTask.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/management/cache/VerifyBackupPartitionsTask.java
index 2330c00334e..4913a0690e2 100644
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/management/cache/VerifyBackupPartitionsTask.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/management/cache/VerifyBackupPartitionsTask.java
@@ -85,12 +85,21 @@ public class VerifyBackupPartitionsTask extends 
ComputeTaskAdapter<CacheIdleVeri
     /** */
     private static final long serialVersionUID = 0L;
 
+    /** */
+    private static final int PARTITION_FUTURE_TIMEOUT = 100;
+
+    /** */
+    private static final long LOG_PROGRESS_TIMEOUT = 3 * 60 * 1000L;
+
     /** Error thrown when idle_verify is called on an inactive cluster with 
persistence. */
     public static final String IDLE_VERIFY_ON_INACTIVE_CLUSTER_ERROR_MESSAGE = 
"Cannot perform the operation because " +
         "the cluster is inactive.";
 
     /** */
-    public static final String CACL_PART_HASH_ERR_MSG = "Can't calculate 
partition hash";
+    public static final String CALC_PART_HASH_ERR_MSG = "Can't calculate 
partition hash";
+
+    /** */
+    private static final String CALC_CANCELLED_ERR_MSG = "Partition hash 
calculation has been cancelled";
 
     /** Checkpoint reason. */
     public static final String CP_REASON = "VerifyBackupPartitions";
@@ -206,7 +215,7 @@ public class VerifyBackupPartitionsTask extends 
ComputeTaskAdapter<CacheIdleVeri
             }
 
             if (isCancelled())
-                throw new IgniteException(getClass().getSimpleName() + " was 
cancelled.");
+                throw new IgniteException(CALC_CANCELLED_ERR_MSG);
 
             Set<Integer> grpIds = getGroupIds();
 
@@ -217,81 +226,102 @@ public class VerifyBackupPartitionsTask extends 
ComputeTaskAdapter<CacheIdleVeri
 
             completionCntr.set(0);
 
-            List<Future<Map<PartitionKey, PartitionHashRecord>>> 
partHashCalcFuts =
-                calcPartitionHashAsync(grpIds);
+            IgniteException jobFailure = null;
+            List<IgniteException> exceptions = null;
+            boolean interrupted = false;
 
             Map<PartitionKey, PartitionHashRecord> res = new HashMap<>();
 
-            List<IgniteException> exceptions = new ArrayList<>();
-
             long lastProgressLogTs = U.currentTimeMillis();
 
-            for (int i = 0; i < partHashCalcFuts.size(); ) {
-                if (isCancelled()) {
-                    cancelFuts(i, partHashCalcFuts);
-
-                    throw new IgniteException(getClass().getSimpleName() + " 
was cancelled.");
-                }
+            List<Future<Map<PartitionKey, PartitionHashRecord>>> 
partHashCalcFuts = calcPartitionHashAsync(grpIds);
 
+            for (int i = 0; i < partHashCalcFuts.size(); i++) {
                 Future<Map<PartitionKey, PartitionHashRecord>> fut = 
partHashCalcFuts.get(i);
 
                 try {
-                    Map<PartitionKey, PartitionHashRecord> partHash = 
fut.get(100, TimeUnit.MILLISECONDS);
+                    Map<PartitionKey, PartitionHashRecord> partHash = 
fut.get(PARTITION_FUTURE_TIMEOUT, TimeUnit.MILLISECONDS);
 
-                    res.putAll(partHash);
-
-                    i++;
+                    if (jobFailure == null)
+                        res.putAll(partHash);
                 }
-                catch (InterruptedException | ExecutionException e) {
+                catch (ExecutionException e) {
                     if (e.getCause() instanceof IgniteException && 
!(e.getCause() instanceof GridNotIdleException)) {
-                        exceptions.add((IgniteException)e.getCause());
+                        if (exceptions == null)
+                            exceptions = new ArrayList<>();
 
-                        i++;
+                        exceptions.add((IgniteException)e.getCause());
+                    }
+                    else if (jobFailure == null) {
+                        jobFailure = new IgniteException("Failed to execute a 
partition hash calculation [grpIds=" +
+                            grpIds + ", reason=" + e.getCause().getMessage() + 
']', e.getCause());
 
-                        continue;
+                        cancel();
                     }
+                    else
+                        jobFailure.addSuppressed(e.getCause());
+                }
+                catch (InterruptedException e) {
+                    interrupted = true;
 
-                    cancelFuts(i, partHashCalcFuts);
+                    if (jobFailure == null) {
+                        jobFailure = new IgniteInterruptedException("Partition 
hash calculation has been interrupted " +
+                            "[grpIds=" + grpIds + "].", e);
 
-                    if (e instanceof InterruptedException)
-                        throw new 
IgniteInterruptedException((InterruptedException)e);
+                        cancel();
+                    }
                     else
-                        throw new IgniteException(e.getCause());
+                        jobFailure.addSuppressed(e);
                 }
-                catch (TimeoutException ignored) {
-                    if (U.currentTimeMillis() - lastProgressLogTs > 3 * 60 * 
1000L) {
+                catch (TimeoutException e) {
+                    if (U.currentTimeMillis() - lastProgressLogTs > 
LOG_PROGRESS_TIMEOUT) {
                         lastProgressLogTs = U.currentTimeMillis();
 
                         log.warning("idle_verify is still running, processed " 
+ completionCntr.get() + " of " +
                             partHashCalcFuts.size() + " local partitions");
                     }
+
+                    --i;
                 }
             }
 
-            if (log.isInfoEnabled())
-                log.info("Idle verify procedure has finished.");
+            if (interrupted)
+                Thread.currentThread().interrupt();
+
+            if (jobFailure != null)
+                throw jobFailure;
+
+            if (isCancelled())
+                throw new IgniteException(CALC_CANCELLED_ERR_MSG + " [grpIds=" 
+ grpIds + "].");
+
+            if (!F.isEmpty(exceptions)) {
+                if (ignite.context().isStopping()) {
+                    throw new IgniteException(CALC_CANCELLED_ERR_MSG + " 
because the local node is stopping [grpIds=" +
+                        grpIds + "].");
+                }
+
+                Integer stoppingGrpId = F.find(grpIds, null, 
this::isGroupStopping);
+
+                if (stoppingGrpId != null) {
+                    throw new IgniteException(CALC_CANCELLED_ERR_MSG + " 
because the cache group is stopping [grpId=" +
+                        stoppingGrpId + "].");
+                }
 
-            if (!F.isEmpty(exceptions))
                 throw new IdleVerifyException(exceptions);
+            }
 
-            return res;
-        }
+            if (log.isInfoEnabled()) {
+                log.info("Idle verify procedure has finished [skipZeros=" + 
arg.skipZeros() + ", checkCrc=" +
+                    arg.checkCrc() + ", grpIds=" + grpIds + "].");
+            }
 
-        /**
-         * Cancels futures from given index.
-         *
-         * @param start Index to start from.
-         * @param partHashCalcFuts Partitions hash calculation futures to 
cancel.
-         */
-        private void cancelFuts(int start, List<Future<Map<PartitionKey, 
PartitionHashRecord>>> partHashCalcFuts) {
-            for (int i = start; i < partHashCalcFuts.size(); i++)
-                partHashCalcFuts.get(i).cancel(false);
+            return res;
         }
 
         /**
          * Class that processes cache filtering chain.
          */
-        private class CachesFiltering {
+        private static class CachesFiltering {
             /**
              * Initially all cache descriptors are included.
              */
@@ -489,6 +519,9 @@ public class VerifyBackupPartitionsTask extends 
ComputeTaskAdapter<CacheIdleVeri
             final BooleanSupplier cancelled
         ) {
             return pool.submit(() -> {
+                if (isCancelledOrStopped(cancelled, gctx))
+                    throw cancellationException(gctx, part);
+
                 Map<PartitionKey, PartitionHashRecord> res = emptyMap();
 
                 if (!part.reserve())
@@ -497,30 +530,42 @@ public class VerifyBackupPartitionsTask extends 
ComputeTaskAdapter<CacheIdleVeri
                 try {
                     PartitionUpdateCounter updCntr = 
part.dataStore().partUpdateCounter();
                     PartitionUpdateCounter updateCntrBefore = updCntr == null 
? null : updCntr.copy();
-
-                    if (arg.checkCrc() && gctx.persistenceEnabled()) {
-                        FilePageStoreManager pageStoreMgr =
-                            
(FilePageStoreManager)ignite.context().cache().context().pageStore();
-
-                        checkPartitionsPageCrcSum(
-                            () -> 
(FilePageStore)pageStoreMgr.getStore(gctx.groupId(), part.id()),
-                            part.id(),
-                            FLAG_DATA,
-                            cancelled
+                    PartitionKey key = new PartitionKey(gctx.groupId(), 
part.id(), gctx.cacheOrGroupName());
+                    PartitionHashRecord hash;
+
+                    // The cache group can be destroyed concurrently.
+                    if (!gctx.offheap().enterBusy())
+                        throw cancellationException(gctx, part);
+
+                    try {
+                        // Also hold the lock during the read to avoid a 
concurrent deletion of the partition
+                        // file when the cache is destroyed.
+                        if (arg.checkCrc() && gctx.persistenceEnabled()) {
+                            FilePageStoreManager pageStoreMgr =
+                                
(FilePageStoreManager)ignite.context().cache().context().pageStore();
+
+                            checkPartitionsPageCrcSum(
+                                () -> 
(FilePageStore)pageStoreMgr.getStore(gctx.groupId(), part.id()),
+                                part.id(),
+                                FLAG_DATA,
+                                () -> isCancelledOrStopped(cancelled, gctx)
+                            );
+                        }
+
+                        hash = calculatePartitionHash(
+                            key,
+                            updateCntrBefore == null ? 0 : 
updateCntrBefore.comparableState(),
+                            
ignite.context().discovery().localNode().consistentId(),
+                            part.state(),
+                            
part.primary(gctx.topology().readyTopologyVersion()),
+                            part.dataStore().fullSize(),
+                            gctx.offheap().partitionIterator(part.id()),
+                            () -> isCancelledOrStopped(cancelled, gctx)
                         );
                     }
-
-                    PartitionKey key = new PartitionKey(gctx.groupId(), 
part.id(), gctx.cacheOrGroupName());
-
-                    PartitionHashRecord hash = calculatePartitionHash(key,
-                        updateCntrBefore == null ? 0 : 
updateCntrBefore.comparableState(),
-                        
ignite.context().discovery().localNode().consistentId(),
-                        part.state(),
-                        part.primary(gctx.topology().readyTopologyVersion()),
-                        part.dataStore().fullSize(),
-                        gctx.offheap().partitionIterator(part.id()),
-                        cancelled
-                    );
+                    finally {
+                        gctx.offheap().leaveBusy();
+                    }
 
                     if (hash != null)
                         res = Collections.singletonMap(key, hash);
@@ -534,7 +579,8 @@ public class VerifyBackupPartitionsTask extends 
ComputeTaskAdapter<CacheIdleVeri
                     }
                 }
                 catch (IgniteCheckedException e) {
-                    String msg = CACL_PART_HASH_ERR_MSG + " [grpId=" + 
gctx.groupId() + ", partId=" + part.id() + "]";
+                    String msg = CALC_PART_HASH_ERR_MSG + " [grpId=" + 
gctx.groupId() + ", partId=" + part.id() +
+                        ", reason=" + e.getMessage() + "]";
 
                     U.error(log, msg, e);
 
@@ -549,5 +595,31 @@ public class VerifyBackupPartitionsTask extends 
ComputeTaskAdapter<CacheIdleVeri
                 return res;
             });
         }
+
+        /**
+         * Checks whether the work should be stopped: the task was cancelled, 
the local node is stopping,
+         * or the cache group is stopping or being destroyed.
+         *
+         * @param cancelled Supplier that returns {@code true} if the task 
cancellation was requested.
+         * @param gctx Cache group context.
+         * @return {@code true} if the work should be stopped.
+         */
+        private boolean isCancelledOrStopped(BooleanSupplier cancelled, 
CacheGroupContext gctx) {
+            return cancelled.getAsBoolean() || gctx.isPreparedToStop() || 
ignite.context().isStopping();
+        }
+
+        /** */
+        private boolean isGroupStopping(int grpId) {
+            CacheGroupContext grp = ignite.context().cache().cacheGroup(grpId);
+
+            return grp == null || grp.isPreparedToStop();
+        }
+
+        /** */
+        private static IgniteException cancellationException(CacheGroupContext 
gctx, GridDhtLocalPartition part) {
+            return new IgniteException(CALC_CANCELLED_ERR_MSG + ". The task 
has been cancelled, or the local node is stopping, " +
+                "or the cache group is stopping [groupId=" + gctx.groupId() +
+                ", groupName=" + gctx.cacheOrGroupName() + ", partId=" + 
part.id() + "].");
+        }
     }
 }
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
index 54884f20bb7..2ddd55889e6 100644
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
@@ -408,6 +408,21 @@ public interface IgniteCacheOffheapManager {
      */
     public void preloadPartition(int pardId) throws IgniteCheckedException;
 
+    /**
+     * Tries to lock the manager against incompatible state like node or cache 
group stopping.
+     *
+     * @return {@code true} if successful, {@code false} if the manager is 
already in an incompatible state.
+     * @see #leaveBusy()
+     */
+    public boolean enterBusy();
+
+    /**
+     * Releases the 'busy' state.
+     *
+     * @see #enterBusy()
+     */
+    public void leaveBusy();
+
     /**
      *
      */
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
index 31a01344897..e99e2347b71 100644
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
@@ -190,6 +190,16 @@ public class IgniteCacheOffheapManagerImpl implements 
IgniteCacheOffheapManager
         // No-op.
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean enterBusy() {
+        return busyLock.enterBusy();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void leaveBusy() {
+        busyLock.leaveBusy();
+    }
+
     /** {@inheritDoc} */
     @Override public void stopCache(int cacheId, boolean destroy) {
         if (destroy && grp.affinityNode())
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index 6508c71da3b..36bf64cc1f0 100755
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -474,7 +474,7 @@ public class PageMemoryImpl implements PageMemoryEx {
 
     /** {@inheritDoc} */
     @Override public void releasePage(int grpId, long pageId, long page) {
-        assert started;
+        assert started : "Page memory is stopped";
 
         Segment seg = segment(grpId, pageId);
 
@@ -490,35 +490,35 @@ public class PageMemoryImpl implements PageMemoryEx {
 
     /** {@inheritDoc} */
     @Override public long readLock(int grpId, long pageId, long page) {
-        assert started;
+        assert started : "Page memory is stopped";
 
         return readLock(page, pageId, false);
     }
 
     /** {@inheritDoc} */
     @Override public void readUnlock(int grpId, long pageId, long page) {
-        assert started;
+        assert started : "Page memory is stopped";
 
         readUnlockPage(page);
     }
 
     /** {@inheritDoc} */
     @Override public long writeLock(int grpId, long pageId, long page) {
-        assert started;
+        assert started : "Page memory is stopped";
 
         return writeLock(grpId, pageId, page, false);
     }
 
     /** {@inheritDoc} */
     @Override public long writeLock(int grpId, long pageId, long page, boolean 
restore) {
-        assert started;
+        assert started : "Page memory is stopped";
 
         return writeLockPage(page, new FullPageId(pageId, grpId), !restore);
     }
 
     /** {@inheritDoc} */
     @Override public long tryWriteLock(int grpId, long pageId, long page) {
-        assert started;
+        assert started : "Page memory is stopped";
 
         return tryWriteLockPage(page, new FullPageId(pageId, grpId), true);
     }
@@ -526,7 +526,7 @@ public class PageMemoryImpl implements PageMemoryEx {
     /** {@inheritDoc} */
     @Override public void writeUnlock(int grpId, long pageId, long page, 
Boolean walPlc,
         boolean dirtyFlag) {
-        assert started;
+        assert started : "Page memory is stopped";
 
         writeUnlock(grpId, pageId, page, walPlc, dirtyFlag, false);
     }
@@ -534,14 +534,14 @@ public class PageMemoryImpl implements PageMemoryEx {
     /** {@inheritDoc} */
     @Override public void writeUnlock(int grpId, long pageId, long page, 
Boolean walPlc,
         boolean dirtyFlag, boolean restore) {
-        assert started;
+        assert started : "Page memory is stopped";
 
         writeUnlockPage(page, new FullPageId(pageId, grpId), walPlc, 
dirtyFlag, restore);
     }
 
     /** {@inheritDoc} */
     @Override public boolean isDirty(int grpId, long pageId, long page) {
-        assert started;
+        assert started : "Page memory is stopped";
 
         return isDirty(page);
     }
@@ -552,7 +552,7 @@ public class PageMemoryImpl implements PageMemoryEx {
             flags == PageIdAllocator.FLAG_IDX && partId == 
PageIdAllocator.INDEX_PARTITION :
             "flags = " + flags + ", partId = " + partId;
 
-        assert started;
+        assert started : "Page memory is stopped";
         assert stateChecker.checkpointLockIsHeldByThread();
 
         if (isThrottlingEnabled())
@@ -697,7 +697,7 @@ public class PageMemoryImpl implements PageMemoryEx {
 
     /** {@inheritDoc} */
     @Override public long partitionMetaPageId(int grpId, int partId) {
-        assert started;
+        assert started : "Page memory is stopped";
 
         return PageIdUtils.pageId(partId, PageIdAllocator.FLAG_DATA, 0);
     }
@@ -710,7 +710,7 @@ public class PageMemoryImpl implements PageMemoryEx {
     /** {@inheritDoc} */
     @Override public long acquirePage(int grpId, long pageId,
         IoStatisticsHolder statHolder) throws IgniteCheckedException {
-        assert started;
+        assert started : "Page memory is stopped";
 
         return acquirePage(grpId, pageId, statHolder, false);
     }
@@ -735,7 +735,7 @@ public class PageMemoryImpl implements PageMemoryEx {
      */
     private long acquirePage(int grpId, long pageId, IoStatisticsHolder 
statHolder,
         boolean restore, @Nullable AtomicBoolean pageAllocated) throws 
IgniteCheckedException {
-        assert started;
+        assert started : "Page memory is stopped";
 
         int partId = PageIdUtils.partId(pageId);
 
@@ -1582,7 +1582,7 @@ public class PageMemoryImpl implements PageMemoryEx {
 
     /** {@inheritDoc} */
     @Override public long readLock(long absPtr, long pageId, boolean force, 
boolean touch) {
-        assert started;
+        assert started : "Page memory is stopped";
 
         int tag = force ? -1 : PageIdUtils.tag(pageId);
 
@@ -1601,7 +1601,7 @@ public class PageMemoryImpl implements PageMemoryEx {
 
     /** {@inheritDoc} */
     @Override public long readLockForce(int grpId, long pageId, long page) {
-        assert started;
+        assert started : "Page memory is stopped";
 
         return readLock(page, pageId, true);
     }
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyHandler.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyHandler.java
index 84c79b88cc6..16562996ef9 100644
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyHandler.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyHandler.java
@@ -304,15 +304,17 @@ public class SnapshotPartitionsVerifyHandler implements 
SnapshotHandler<Snapshot
                         // There is no `primary` partitions for snapshot.
                         PartitionKey key = new PartitionKey(grpId, partId, 
grpName);
 
-                        PartitionHashRecord hash = calculatePartitionHash(key,
+                        PartitionHashRecord hash = calculatePartitionHash(
+                            key,
                             updateCntr,
                             meta.consistentId(),
                             GridDhtPartitionState.OWNING,
                             false,
                             size,
-                            skipHash() ? F.emptyIterator()
+                            skipHash()
+                                ? F.emptyIterator()
                                 : snpMgr.partitionRowIterator(snpCtx, grpName, 
partId, pageStore),
-                            null
+                            () -> cctx.kernalContext().isStopping()
                         );
 
                         assert hash != null : "OWNING must have hash: " + key;
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java
index 3cccbb3e72a..6a42f7138b2 100644
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java
@@ -122,7 +122,7 @@ public class IdleVerifyUtility {
 
             for (int pageNo = 0; pageNo < pageStore.pages(); pageId++, 
pageNo++) {
                 if (cancelled != null && cancelled.getAsBoolean())
-                    throw new IgniteException("Checking of partitions page CRC 
sum has been cancelled.");
+                    throw new IgniteException("Checking of partitions page CRC 
sum has been cancelled [partId=" + partId + ']');
 
                 buf.clear();
 
@@ -288,17 +288,18 @@ public class IdleVerifyUtility {
         boolean isPrimary,
         long partSize,
         GridIterator<CacheDataRow> it,
-        @Nullable BooleanSupplier cancelled
+        BooleanSupplier cancelled
     ) throws IgniteCheckedException {
         if (state == GridDhtPartitionState.MOVING || state == 
GridDhtPartitionState.LOST) {
-            return new PartitionHashRecord(partKey,
+            return new PartitionHashRecord(
+                partKey,
                 isPrimary,
                 consId,
                 updCntr,
-                state == GridDhtPartitionState.MOVING ?
-                    PartitionHashRecord.MOVING_PARTITION_SIZE : 0,
-                state == GridDhtPartitionState.MOVING ?
-                    PartitionHashRecord.PartitionState.MOVING : 
PartitionHashRecord.PartitionState.LOST,
+                state == GridDhtPartitionState.MOVING ? 
PartitionHashRecord.MOVING_PARTITION_SIZE : 0,
+                state == GridDhtPartitionState.MOVING
+                    ? PartitionHashRecord.PartitionState.MOVING
+                    : PartitionHashRecord.PartitionState.LOST,
                 new VerifyPartitionContext()
             );
         }
@@ -308,10 +309,9 @@ public class IdleVerifyUtility {
 
         VerifyPartitionContext ctx = new VerifyPartitionContext();
 
-        while (it.hasNextX()) {
-            if (cancelled != null && cancelled.getAsBoolean())
-                throw new IgniteCheckedException("Partition hash calculation 
has been cancelled.");
+        boolean cancelled0;
 
+        while (!(cancelled0 = cancelled.getAsBoolean()) && it.hasNextX()) {
             CacheDataRow row = it.nextX();
 
             if (row.expireTime() > 0)
@@ -320,6 +320,9 @@ public class IdleVerifyUtility {
             ctx.update(row.key(), row.value(), row.version());
         }
 
+        if (cancelled0)
+            throw new IgniteException("Partition hash calculation has been 
cancelled [part=" + partKey + ']');
+
         return new PartitionHashRecord(
             partKey,
             isPrimary,
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/pool/PoolProcessor.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/pool/PoolProcessor.java
index bedbb96bf66..5f264a7ec81 100644
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/pool/PoolProcessor.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/pool/PoolProcessor.java
@@ -145,10 +145,13 @@ public class PoolProcessor extends GridProcessorAdapter {
      */
     public static final long DFLT_PERIODIC_STARVATION_CHECK_FREQ = 30 * 1000L;
 
+    /** */
+    public static final String POOL_NAME_IDLE_VERIFY = "idleVerify";
+
     /** */
     @SystemProperty(value = "Idle verify utility thread pool size.", type = 
Integer.class,
         defaults = "Total available CPUs minus 2, but at least 4 threads.")
-    public static final String IDLE_VERIFY_POOL_SIZE_PROPERTY = 
"IGNITE_IDLE_VERIFY_POOL_SIZE";
+    private static final String IDLE_VERIFY_POOL_SIZE_PROPERTY = 
"IGNITE_IDLE_VERIFY_POOL_SIZE";
 
     /** */
     private static final int DFLT_IDLE_VERIFY_POOL_SIZE = Math.max(4, 
IgniteConfiguration.AVAILABLE_PROC_CNT - 2);
@@ -395,7 +398,7 @@ public class PoolProcessor extends GridProcessorAdapter {
         validateThreadPoolSize(idleVerifyPoolSz, "idle verify");
 
         idleVerifyExecSvc = new IgniteThreadPoolExecutor(
-            "idleVerify",
+            POOL_NAME_IDLE_VERIFY,
             cfg.getIgniteInstanceName(),
             idleVerifyPoolSz,
             idleVerifyPoolSz,
diff --git 
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/NodeOrCacheStoppingDuringIdleVerifyTest.java
 
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/NodeOrCacheStoppingDuringIdleVerifyTest.java
new file mode 100644
index 00000000000..6409082fffb
--- /dev/null
+++ 
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/NodeOrCacheStoppingDuringIdleVerifyTest.java
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.OpenOption;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cluster.ClusterState;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.TestFailureHandler;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.management.cache.IdleVerifyResult;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import 
org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator;
+import 
org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import 
org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
+import org.apache.ignite.internal.processors.cache.verify.GridNotIdleException;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import static 
org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree.partitionFileName;
+import static 
org.apache.ignite.internal.processors.pool.PoolProcessor.POOL_NAME_IDLE_VERIFY;
+import static org.apache.ignite.testframework.GridTestUtils.runAsync;
+import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
+
+/** */
+@RunWith(Parameterized.class)
+public class NodeOrCacheStoppingDuringIdleVerifyTest extends 
GridCommonAbstractTest {
+    /** */
+    private static final int PARTS = 4;
+
+    /** */
+    private static final int KEYS = PARTS * 200;
+
+    /** */
+    private static final int NOT_IDLE_PART = 0;
+
+    /** */
+    private static volatile CountDownLatch partitionReadBlocked;
+
+    /** */
+    private static volatile CountDownLatch partitionReadUnblocked;
+
+    /** */
+    private static volatile CountDownLatch notIdlePartReadUnblocked;
+
+    /** */
+    @Parameterized.Parameter
+    public boolean checkCrc;
+
+    /** */
+    private TestFailureHandler failureHnd;
+
+    /** */
+    @Parameterized.Parameters(name = "checkCrc={0}")
+    public static Iterable<Object> parameters() {
+        return F.asList(false, true);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String 
igniteInstanceName) throws Exception {
+        return super.getConfiguration(igniteInstanceName)
+            .setDataStorageConfiguration(new DataStorageConfiguration()
+                .setFileIOFactory(new IdleVerifyThreadsBlockingFileIOFactory())
+                .setDefaultDataRegionConfiguration(new 
DataRegionConfiguration()
+                    .setPersistenceEnabled(true)))
+            .setFailureHandler(failureHnd);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        cleanPersistenceDir();
+
+        failureHnd = new TestFailureHandler(false);
+
+        partitionReadBlocked = new CountDownLatch(PARTS);
+        partitionReadUnblocked = new CountDownLatch(1);
+        notIdlePartReadUnblocked = new CountDownLatch(1);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        unblockPartitionRead();
+
+        stopAllGrids();
+
+        cleanPersistenceDir();
+
+        super.afterTest();
+    }
+
+    /** */
+    @Test
+    public void testWithCacheDestruction() throws Exception {
+        prepareCluster();
+
+        CacheGroupContext grp = 
grid(0).context().cache().cacheGroup(CU.cacheId(DEFAULT_CACHE_NAME));
+
+        IgniteInternalFuture<IdleVerifyResult> idleFut = startIdleVerify();
+
+        IgniteInternalFuture<?> destroyFut = runAsync(() -> 
grid(0).destroyCache(DEFAULT_CACHE_NAME));
+
+        assertTrue(waitForCondition(grp::isPreparedToStop, getTestTimeout()));
+
+        assertFalse(waitForCondition(destroyFut::isDone, 1_000));
+
+        unblockPartitionRead();
+
+        destroyFut.get(getTestTimeout());
+
+        assertFalse(F.isEmpty(idleFut.get(getTestTimeout()).exceptions()));
+        assertNull(failureHnd.failureContext());
+    }
+
+    /** */
+    @Test
+    public void testWithDeactivation() throws Exception {
+        prepareCluster();
+
+        CacheGroupContext grp = 
grid(0).context().cache().cacheGroup(CU.cacheId(DEFAULT_CACHE_NAME));
+
+        IgniteInternalFuture<IdleVerifyResult> idleFut = startIdleVerify();
+
+        IgniteInternalFuture<?> deactivateFut = runAsync(() -> 
grid(0).cluster().state(ClusterState.INACTIVE));
+
+        assertTrue(waitForCondition(grp::isPreparedToStop, getTestTimeout()));
+
+        assertFalse(waitForCondition(deactivateFut::isDone, 1_000));
+
+        unblockPartitionRead();
+
+        deactivateFut.get(getTestTimeout());
+
+        assertFalse(F.isEmpty(idleFut.get(getTestTimeout()).exceptions()));
+
+        grid(0).cluster().state(ClusterState.ACTIVE);
+
+        assertEquals(KEYS, grid(0).cache(DEFAULT_CACHE_NAME).size());
+        assertNull(failureHnd.failureContext());
+    }
+
+    /** */
+    @Test
+    public void testWithNodeStoppage() throws Exception {
+        prepareCluster();
+
+        IgniteCache<Integer, Integer> cache = 
grid(0).cache(DEFAULT_CACHE_NAME);
+
+        int key = partitionKeys(cache, NOT_IDLE_PART, 1, 0).get(0);
+
+        cache.get(key);
+
+        IgniteInternalFuture<IdleVerifyResult> idleFut = startIdleVerify();
+
+        cache.put(key, -1);
+
+        notIdlePartReadUnblocked.countDown();
+
+        assertTrue(waitForCondition(
+            () -> 
grid(0).context().pools().getIdleVerifyExecutorService().getCompletedTaskCount()
 > 0,
+            getTestTimeout()
+        ));
+
+        GridKernalContext kctx = grid(0).context();
+
+        IgniteInternalFuture<?> stopFut = runAsync(() -> stopGrid(0, true));
+
+        assertTrue(waitForCondition(kctx::isStopping, getTestTimeout()));
+
+        unblockPartitionRead();
+
+        stopFut.get(getTestTimeout());
+
+        IdleVerifyResult res = idleFut.get(getTestTimeout());
+
+        assertTrue(X.hasCause(F.first(res.exceptions().values()), 
GridNotIdleException.class));
+        assertNull(failureHnd.failureContext());
+    }
+
+    /** */
+    private void prepareCluster() throws Exception {
+        startGrid(0);
+
+        grid(0).cluster().state(ClusterState.ACTIVE);
+
+        IgniteCache<Integer, Integer> cache = grid(0).createCache(new 
CacheConfiguration<Integer, Integer>(DEFAULT_CACHE_NAME)
+            .setAffinity(new RendezvousAffinityFunction(false, PARTS)));
+
+        for (int k = 0; k < KEYS; k++)
+            cache.put(k, k);
+
+        stopGrid(0);
+
+        startGrid(0);
+
+        grid(0).cluster().state(ClusterState.ACTIVE);
+    }
+
+    /** */
+    private IgniteInternalFuture<IdleVerifyResult> startIdleVerify() throws 
Exception {
+        IgniteInternalFuture<IdleVerifyResult> fut = runAsync(() -> 
idleVerify(grid(0), checkCrc, DEFAULT_CACHE_NAME));
+
+        assertTrue(partitionReadBlocked.await(getTestTimeout(), 
TimeUnit.MILLISECONDS));
+
+        return fut;
+    }
+
+    /** */
+    private static void unblockPartitionRead() {
+        notIdlePartReadUnblocked.countDown();
+        partitionReadUnblocked.countDown();
+    }
+
+    /** */
+    private static class IdleVerifyThreadsBlockingFileIOFactory implements 
FileIOFactory {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        private final FileIOFactory delegate = new RandomAccessFileIOFactory();
+
+        /** {@inheritDoc} */
+        @Override public FileIO create(File file, OpenOption... modes) throws 
IOException {
+            return new FileIODecorator(delegate.create(file, modes)) {
+                @Override public int readFully(ByteBuffer destBuf, long 
position) throws IOException {
+                    int res = super.readFully(destBuf, position);
+
+                    if 
(Thread.currentThread().getName().startsWith(POOL_NAME_IDLE_VERIFY)) {
+                        partitionReadBlocked.countDown();
+
+                        
U.awaitQuiet(file.getName().equals(partitionFileName(NOT_IDLE_PART))
+                            ? notIdlePartReadUnblocked
+                            : partitionReadUnblocked);
+                    }
+
+                    return res;
+                }
+            };
+        }
+    }
+}
diff --git 
a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
 
b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
index 2ad409701d0..b923e6bd1d0 100755
--- 
a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
+++ 
b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
@@ -2209,11 +2209,12 @@ public abstract class GridCommonAbstractTest extends 
GridAbstractTest {
      * concurrently updated.
      *
      * @param ig Ignite instance.
+     * @param checkCrc Whether to check the CRC sums of partition pages.
      * @param caches Cache names (if null, all user caches will be verified).
      * @return Conflicts result.
      * @throws IgniteException If none caches or node found.
      */
-    protected IdleVerifyResult idleVerify(Ignite ig, @Nullable String... 
caches) throws Exception {
+    protected IdleVerifyResult idleVerify(Ignite ig, boolean checkCrc, 
@Nullable String... caches) throws Exception {
         log.info("Starting idleVerify ...");
 
         IgniteEx ig0 = (IgniteEx)ig;
@@ -2236,6 +2237,7 @@ public abstract class GridCommonAbstractTest extends 
GridAbstractTest {
         CacheIdleVerifyCommandArg taskArg = new CacheIdleVerifyCommandArg();
 
         taskArg.caches(cacheNames.toArray(U.EMPTY_STRS));
+        taskArg.checkCrc(checkCrc);
 
         return ((VisorTaskResult<IdleVerifyResult>)ig.compute().execute(
             IdleVerifyTask.class.getName(),
@@ -2243,6 +2245,11 @@ public abstract class GridCommonAbstractTest extends 
GridAbstractTest {
         )).result();
     }
 
+    /** @see #idleVerify(Ignite, boolean, String...) */
+    protected IdleVerifyResult idleVerify(Ignite ig, @Nullable String... 
caches) throws Exception {
+        return idleVerify(ig, false, caches);
+    }
+
     /**
      * Checks if all txs and mvcc futures are finished.
      */
diff --git 
a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite3.java
 
b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite3.java
index 3b93a139c37..2e2c6234f79 100644
--- 
a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite3.java
+++ 
b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite3.java
@@ -23,6 +23,7 @@ import java.util.List;
 import 
org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTest;
 import 
org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTestWithExpiryPolicy;
 import 
org.apache.ignite.internal.processors.cache.persistence.IgnitePdsNodeJoinWithCachesStopping;
+import 
org.apache.ignite.internal.processors.cache.persistence.NodeOrCacheStoppingDuringIdleVerifyTest;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.DynamicSuite;
 import org.junit.runner.RunWith;
@@ -47,6 +48,7 @@ public class IgnitePdsTestSuite3 {
         GridTestUtils.addTestIfNeeded(suite, 
IgnitePdsContinuousRestartTest.class, ignoredTests);
         GridTestUtils.addTestIfNeeded(suite, 
IgnitePdsContinuousRestartTestWithExpiryPolicy.class, ignoredTests);
         GridTestUtils.addTestIfNeeded(suite, 
IgnitePdsNodeJoinWithCachesStopping.class, ignoredTests);
+        GridTestUtils.addTestIfNeeded(suite, 
NodeOrCacheStoppingDuringIdleVerifyTest.class, ignoredTests);
 
         return suite;
     }

Reply via email to