Repository: ignite
Updated Branches:
  refs/heads/master 859b37cc8 -> d7af98538


IGNITE-10446 Fix control.sh --cache idle_verify fail with NPE when node left 
grid

Signed-off-by: Dmitriy Govorukhin <[email protected]>


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/d7af9853
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/d7af9853
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/d7af9853

Branch: refs/heads/master
Commit: d7af985383d9e82608fd77904415960492ed6290
Parents: 859b37c
Author: Vladislav Pyatkov <[email protected]>
Authored: Thu Dec 13 12:26:41 2018 +0300
Committer: Dmitriy Govorukhin <[email protected]>
Committed: Thu Dec 13 12:26:41 2018 +0300

----------------------------------------------------------------------
 .../internal/commandline/CommandHandler.java    |   4 +-
 .../cache/verify/IdleVerifyResultV2.java        |  34 ++-
 .../verify/VerifyBackupPartitionsDumpTask.java  |  28 +++
 .../verify/VerifyBackupPartitionsTaskV2.java    |  31 ++-
 .../ignite/util/GridCommandHandlerTest.java     | 214 +++++++++++++------
 5 files changed, 237 insertions(+), 74 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/d7af9853/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
----------------------------------------------------------------------
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index 2cfce37..8dcceed 100644
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -855,7 +855,7 @@ public class CommandHandler {
             log("Contention check failed on nodes:");
 
             for (Map.Entry<UUID, Exception> e : res.exceptions().entrySet()) {
-                log("Node ID = " + e.getKey());
+                log("Node ID: " + e.getKey());
 
                 log("Exception message:");
                 log(e.getValue().getMessage());
@@ -890,7 +890,7 @@ public class CommandHandler {
             log("Index validation failed on nodes:");
 
             for (Map.Entry<UUID, Exception> e : 
taskRes.exceptions().entrySet()) {
-                log(i("Node ID = " + e.getKey()));
+                log(i("Node ID: " + e.getKey()));
 
                 log(i("Exception message:"));
                 log(i(e.getValue().getMessage(), 2));

http://git-wip-us.apache.org/repos/asf/ignite/blob/d7af9853/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java
----------------------------------------------------------------------
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java
index c31d662..a153063 100644
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java
@@ -21,6 +21,7 @@ import java.io.ObjectInput;
 import java.io.ObjectOutput;
 import java.util.List;
 import java.util.Map;
+import java.util.UUID;
 import java.util.function.Consumer;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -43,6 +44,9 @@ public class IdleVerifyResultV2 extends 
VisorDataTransferObject {
     /** Moving partitions. */
     private Map<PartitionKeyV2, List<PartitionHashRecordV2>> movingPartitions;
 
+    /** Exceptions. */
+    private Map<UUID, Exception> exceptions;
+
     /**
      * @param cntrConflicts Counter conflicts.
      * @param hashConflicts Hash conflicts.
@@ -51,11 +55,13 @@ public class IdleVerifyResultV2 extends 
VisorDataTransferObject {
     public IdleVerifyResultV2(
         Map<PartitionKeyV2, List<PartitionHashRecordV2>> cntrConflicts,
         Map<PartitionKeyV2, List<PartitionHashRecordV2>> hashConflicts,
-        Map<PartitionKeyV2, List<PartitionHashRecordV2>> movingPartitions
+        Map<PartitionKeyV2, List<PartitionHashRecordV2>> movingPartitions,
+        Map<UUID, Exception> exceptions
     ) {
         this.cntrConflicts = cntrConflicts;
         this.hashConflicts = hashConflicts;
         this.movingPartitions = movingPartitions;
+        this.exceptions = exceptions;
     }
 
     /**
@@ -65,10 +71,16 @@ public class IdleVerifyResultV2 extends 
VisorDataTransferObject {
     }
 
     /** {@inheritDoc} */
+    @Override public byte getProtocolVersion() {
+        return V2;
+    }
+
+    /** {@inheritDoc} */
     @Override protected void writeExternalData(ObjectOutput out) throws 
IOException {
         U.writeMap(out, cntrConflicts);
         U.writeMap(out, hashConflicts);
         U.writeMap(out, movingPartitions);
+        U.writeMap(out, exceptions);
     }
 
     /** {@inheritDoc} */
@@ -77,6 +89,9 @@ public class IdleVerifyResultV2 extends 
VisorDataTransferObject {
         cntrConflicts = U.readMap(in);
         hashConflicts = U.readMap(in);
         movingPartitions = U.readMap(in);
+
+        if (protoVer >= V2)
+            exceptions = U.readMap(in);
     }
 
     /**
@@ -108,6 +123,13 @@ public class IdleVerifyResultV2 extends 
VisorDataTransferObject {
     }
 
     /**
+     * @return Exceptions on nodes.
+     */
+    public Map<UUID, Exception> exceptions() {
+        return exceptions;
+    }
+
+    /**
      * Print formatted result to given printer.
      *
      * @param printer Consumer for handle formatted result.
@@ -159,6 +181,16 @@ public class IdleVerifyResultV2 extends 
VisorDataTransferObject {
 
             printer.accept("\n");
         }
+
+        if (!F.isEmpty(exceptions())) {
+            printer.accept("Idle verify failed on nodes:\n");
+
+            for (Map.Entry<UUID, Exception> e : exceptions().entrySet()) {
+                printer.accept("Node ID: " + e.getKey() + "\n");
+                printer.accept("Exception message:" + "\n");
+                printer.accept(e.getValue().getMessage() + "\n");
+            }
+        }
     }
 
     /** {@inheritDoc} */

http://git-wip-us.apache.org/repos/asf/ignite/blob/d7af9853/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsDumpTask.java
----------------------------------------------------------------------
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsDumpTask.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsDumpTask.java
index 7bbd9f1..5945f4d 100644
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsDumpTask.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsDumpTask.java
@@ -30,15 +30,18 @@ import java.util.Map;
 import java.util.TreeMap;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.compute.ComputeJob;
 import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.compute.ComputeJobResultPolicy;
 import org.apache.ignite.compute.ComputeTaskAdapter;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.visor.verify.VisorIdleVerifyDumpTaskArg;
 import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskArg;
 import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.resources.LoggerResource;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
@@ -68,6 +71,10 @@ public class VerifyBackupPartitionsDumpTask extends 
ComputeTaskAdapter<VisorIdle
     @IgniteInstanceResource
     private Ignite ignite;
 
+    /** Injected logger. */
+    @LoggerResource
+    private IgniteLogger log;
+
     /** {@inheritDoc} */
     @Nullable @Override public Map<? extends ComputeJob, ClusterNode> map(
         List<ClusterNode> subgrid, VisorIdleVerifyTaskArg arg) throws 
IgniteException {
@@ -83,6 +90,9 @@ public class VerifyBackupPartitionsDumpTask extends 
ComputeTaskAdapter<VisorIdle
         Map<PartitionKeyV2, List<PartitionHashRecordV2>> clusterHashes = new 
TreeMap<>(buildPartitionKeyComparator());
 
         for (ComputeJobResult res : results) {
+            if (res.getException() != null)
+                continue;
+
             Map<PartitionKeyV2, PartitionHashRecordV2> nodeHashes = 
res.getData();
 
             for (Map.Entry<PartitionKeyV2, PartitionHashRecordV2> e : 
nodeHashes.entrySet()) {
@@ -111,6 +121,22 @@ public class VerifyBackupPartitionsDumpTask extends 
ComputeTaskAdapter<VisorIdle
         return writeHashes(partitions, delegate.reduce(results), 
skippedRecords);
     }
 
+    /** {@inheritDoc} */
+    @Override public ComputeJobResultPolicy result(ComputeJobResult res, 
List<ComputeJobResult> rcvd) throws
+        IgniteException {
+        ComputeJobResultPolicy superRes = super.result(res, rcvd);
+
+        // Deny failover.
+        if (superRes == ComputeJobResultPolicy.FAILOVER) {
+            superRes = ComputeJobResultPolicy.WAIT;
+
+            log.warning("VerifyBackupPartitionsJobV2 failed on node " +
+                "[consistentId=" + res.getNode().consistentId() + "]", 
res.getException());
+        }
+
+        return superRes;
+    }
+
     /**
      * Checking conditions for adding given record to result.
      *
@@ -140,6 +166,8 @@ public class VerifyBackupPartitionsDumpTask extends 
ComputeTaskAdapter<VisorIdle
 
     /**
      * @param partitions Dump result.
+     * @param conflictRes Conflict results.
+     * @param skippedRecords Number of empty partitions.
      * @return Path where results are written.
      * @throws IgniteException If failed to write the file.
      */

http://git-wip-us.apache.org/repos/asf/ignite/blob/d7af9853/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
----------------------------------------------------------------------
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
index 14995cb..5e872eb 100644
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
@@ -25,6 +25,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.UUID;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ForkJoinPool;
@@ -41,6 +42,7 @@ import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.compute.ComputeJob;
 import org.apache.ignite.compute.ComputeJobAdapter;
 import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.compute.ComputeJobResultPolicy;
 import org.apache.ignite.compute.ComputeTaskAdapter;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
@@ -77,6 +79,10 @@ public class VerifyBackupPartitionsTaskV2 extends 
ComputeTaskAdapter<VisorIdleVe
     /** First version of Ignite that is capable of executing Idle Verify V2. */
     public static final IgniteProductVersion V2_SINCE_VER = 
IgniteProductVersion.fromString("2.5.3");
 
+    /** Injected logger. */
+    @LoggerResource
+    private IgniteLogger log;
+
     /** */
     private static final long serialVersionUID = 0L;
 
@@ -95,8 +101,15 @@ public class VerifyBackupPartitionsTaskV2 extends 
ComputeTaskAdapter<VisorIdleVe
     @Nullable @Override public IdleVerifyResultV2 
reduce(List<ComputeJobResult> results)
         throws IgniteException {
         Map<PartitionKeyV2, List<PartitionHashRecordV2>> clusterHashes = new 
HashMap<>();
+        Map<UUID, Exception> exceptions = new HashMap<>();
 
         for (ComputeJobResult res : results) {
+            if (res.getException() != null) {
+                exceptions.put(res.getNode().id(), res.getException());
+
+                continue;
+            }
+
             Map<PartitionKeyV2, PartitionHashRecordV2> nodeHashes = 
res.getData();
 
             for (Map.Entry<PartitionKeyV2, PartitionHashRecordV2> e : 
nodeHashes.entrySet()) {
@@ -141,7 +154,23 @@ public class VerifyBackupPartitionsTaskV2 extends 
ComputeTaskAdapter<VisorIdleVe
             }
         }
 
-        return new IdleVerifyResultV2(updateCntrConflicts, hashConflicts, 
movingParts);
+        return new IdleVerifyResultV2(updateCntrConflicts, hashConflicts, 
movingParts, exceptions);
+    }
+
+    /** {@inheritDoc} */
+    @Override public ComputeJobResultPolicy result(ComputeJobResult res, 
List<ComputeJobResult> rcvd) throws
+        IgniteException {
+        ComputeJobResultPolicy superRes = super.result(res, rcvd);
+
+        // Deny failover.
+        if (superRes == ComputeJobResultPolicy.FAILOVER) {
+            superRes = ComputeJobResultPolicy.WAIT;
+
+            log.warning("VerifyBackupPartitionsJobV2 failed on node " +
+                "[consistentId=" + res.getNode().consistentId() + "]", 
res.getException());
+        }
+
+        return superRes;
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/d7af9853/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
----------------------------------------------------------------------
diff --git 
a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java 
b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index e40d648..318b837 100644
--- 
a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ 
b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -32,6 +32,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
+import java.util.UUID;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -47,6 +48,7 @@ import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteAtomicSequence;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.AtomicConfiguration;
@@ -56,6 +58,7 @@ import 
org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.GridJobExecuteResponse;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.TestRecordingCommunicationSpi;
@@ -922,13 +925,7 @@ public class GridCommandHandlerTest extends 
GridCommonAbstractTest {
 
         ignite.cluster().active(true);
 
-        IgniteCache<Object, Object> cache = ignite.createCache(new 
CacheConfiguration<>()
-            .setAffinity(new RendezvousAffinityFunction(false, 32))
-            .setBackups(1)
-            .setName(DEFAULT_CACHE_NAME));
-
-        for (int i = 0; i < 100; i++)
-            cache.put(i, i);
+        createCacheAndPreload(ignite, 100);
 
         injectTestSystemOut();
 
@@ -956,15 +953,7 @@ public class GridCommandHandlerTest extends 
GridCommonAbstractTest {
 
         ignite.cluster().active(true);
 
-        int parts = 32;
-
-        IgniteCache<Object, Object> cache = ignite.createCache(new 
CacheConfiguration<>()
-            .setAffinity(new RendezvousAffinityFunction(false, parts))
-            .setBackups(1)
-            .setName(DEFAULT_CACHE_NAME));
-
-        for (int i = 0; i < 100; i++)
-            cache.put(i, i);
+        createCacheAndPreload(ignite, 100);
 
         injectTestSystemOut();
 
@@ -976,7 +965,7 @@ public class GridCommandHandlerTest extends 
GridCommonAbstractTest {
 
         corruptDataEntry(cacheCtx, 1, true, false);
 
-        corruptDataEntry(cacheCtx, 1 + parts / 2, false, true);
+        corruptDataEntry(cacheCtx, 1 + 
cacheCtx.config().getAffinity().partitions() / 2, false, true);
 
         assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify"));
 
@@ -994,12 +983,11 @@ public class GridCommandHandlerTest extends 
GridCommonAbstractTest {
 
         ignite.cluster().active(true);
 
-        int parts = 32;
+        int keysCount = 20;//less than parts number for ability to check 
skipZeros flag.
 
-        IgniteCache<Object, Object> cache = ignite.createCache(new 
CacheConfiguration<>()
-            .setAffinity(new RendezvousAffinityFunction(false, parts))
-            .setBackups(1)
-            .setName(DEFAULT_CACHE_NAME));
+        createCacheAndPreload(ignite, keysCount);
+
+        int parts = ignite.affinity(DEFAULT_CACHE_NAME).partitions();
 
         ignite.createCache(new CacheConfiguration<>()
             .setAffinity(new RendezvousAffinityFunction(false, parts))
@@ -1008,11 +996,6 @@ public class GridCommandHandlerTest extends 
GridCommonAbstractTest {
 
         injectTestSystemOut();
 
-        int keysCount = 20;//less than parts number for ability to check 
skipZeros flag.
-
-        for (int i = 0; i < keysCount; i++)
-            cache.put(i, i);
-
         assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", 
DEFAULT_CACHE_NAME));
 
         assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", 
"--skipZeros", DEFAULT_CACHE_NAME));
@@ -1081,31 +1064,25 @@ public class GridCommandHandlerTest extends 
GridCommonAbstractTest {
 
         ignite.cluster().active(true);
 
-        int parts = 32;
-
-        IgniteCache<Object, Object> cache = ignite.createCache(new 
CacheConfiguration<>()
-            .setAffinity(new RendezvousAffinityFunction(false, parts))
-            .setBackups(1)
-            .setName(DEFAULT_CACHE_NAME));
+        createCacheAndPreload(ignite, 100);
 
-        for (int i = 0; i < 100; i++)
-            cache.put(i, i);
+        injectTestSystemOut();
 
-        corruptingAndCheckDefaultCache(ignite, parts, CacheFilterEnum.ALL);
+        corruptingAndCheckDefaultCache(ignite, CacheFilterEnum.ALL);
     }
 
     /**
      * @param ignite Ignite.
-     * @param parts Parts.
+     * @param cacheFilterEnum Filter enum.
      */
-    private void corruptingAndCheckDefaultCache(IgniteEx ignite, int parts, 
CacheFilterEnum cacheFilterEnum) throws IOException {
+    private void corruptingAndCheckDefaultCache(IgniteEx ignite, 
CacheFilterEnum cacheFilterEnum) throws IOException {
         injectTestSystemOut();
 
         GridCacheContext<Object, Object> cacheCtx = 
ignite.cachex(DEFAULT_CACHE_NAME).context();
 
         corruptDataEntry(cacheCtx, 0, true, false);
 
-        corruptDataEntry(cacheCtx, parts / 2, false, true);
+        corruptDataEntry(cacheCtx, 
cacheCtx.config().getAffinity().partitions() / 2, false, true);
 
         assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", 
"--cacheFilter", cacheFilterEnum.toString()));
 
@@ -1121,6 +1098,130 @@ public class GridCommandHandlerTest extends 
GridCommonAbstractTest {
     }
 
     /**
+     * Tests that idle verify print partitions info when node failing.
+     *
+     * @throws Exception If failed.
+     */
+    public void testCacheIdleVerifyDumpWhenNodeFailing() throws Exception {
+        Ignite ignite = startGrids(3);
+
+        Ignite unstable = startGrid("unstable");
+
+        ignite.cluster().active(true);
+
+        createCacheAndPreload(ignite, 100);
+
+        for (int i = 0; i < 3; i++) {
+            
TestRecordingCommunicationSpi.spi(unstable).blockMessages(GridJobExecuteResponse.class,
+                getTestIgniteInstanceName(i));
+        }
+
+        injectTestSystemOut();
+
+        IgniteInternalFuture fut = GridTestUtils.runAsync(() -> {
+            assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", 
"--dump"));
+        });
+
+        TestRecordingCommunicationSpi.spi(unstable).waitForBlocked();
+
+        UUID unstableNodeId = unstable.cluster().localNode().id();
+
+        unstable.close();
+
+        fut.get();
+
+        checkExceptionMessageOnReport(unstableNodeId);
+    }
+
+    /**
+     * Tests that idle verify print partitions info when several nodes failing 
at same time.
+     *
+     * @throws Exception If failed.
+     */
+    public void testCacheIdleVerifyDumpWhenSeveralNodesFailing() throws 
Exception {
+        int nodes = 6;
+
+        Ignite ignite = startGrids(nodes);
+
+        List<Ignite> unstableNodes = new ArrayList<>(nodes / 2);
+
+        for (int i = 0; i < nodes; i++) {
+            if (i % 2 == 1)
+                unstableNodes.add(ignite(i));
+        }
+
+        ignite.cluster().active(true);
+
+        createCacheAndPreload(ignite, 100);
+
+        for (Ignite unstable : unstableNodes) {
+            for (int i = 0; i < nodes; i++) {
+                
TestRecordingCommunicationSpi.spi(unstable).blockMessages(GridJobExecuteResponse.class,
+                    getTestIgniteInstanceName(i));
+            }
+        }
+
+        injectTestSystemOut();
+
+        IgniteInternalFuture fut = GridTestUtils.runAsync(() -> {
+            assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", 
"--dump"));
+        });
+
+        List<UUID> unstableNodeIds = new ArrayList<>(nodes / 2);
+
+        for (Ignite unstable : unstableNodes) {
+            TestRecordingCommunicationSpi.spi(unstable).waitForBlocked();
+
+            unstableNodeIds.add(unstable.cluster().localNode().id());
+
+            unstable.close();
+        }
+
+        fut.get();
+
+        for (UUID unstableId : unstableNodeIds)
+            checkExceptionMessageOnReport(unstableId);
+    }
+
+    /**
+     * Creates default cache and preload some data entries.
+     *
+     * @param ignite Ignite.
+     * @param countEntries Count of entries.
+     */
+    private void createCacheAndPreload(Ignite ignite, int countEntries) {
+        ignite.createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME)
+            .setAffinity(new RendezvousAffinityFunction(false, 32))
+            .setBackups(1));
+
+        try (IgniteDataStreamer streamer = 
ignite.dataStreamer(DEFAULT_CACHE_NAME)) {
+            for (int i = 0; i < countEntries; i++)
+                streamer.addData(i, i);
+        }
+    }
+
+    /**
+     * Try to finds node failed exception message on output report.
+     *
+     * @param unstableNodeId Unstable node id.
+     */
+    private void checkExceptionMessageOnReport(UUID unstableNodeId) throws 
IOException {
+        Matcher fileNameMatcher = dumpFileNameMatcher();
+
+        if (fileNameMatcher.find()) {
+            String dumpWithConflicts = new 
String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1))));
+
+            assertTrue(dumpWithConflicts.contains("Idle verify failed on 
nodes:"));
+
+            assertTrue(dumpWithConflicts.contains("Node ID: " + unstableNodeId 
+ "\n" +
+                "Exception message:\n" +
+                "Node has left grid: " + unstableNodeId));
+        }
+        else
+            fail("Should be found dump with conflicts");
+    }
+
+    /**
      * Tests that idle verify print partitions info over system caches.
      *
      * @throws Exception If failed.
@@ -1188,28 +1289,13 @@ public class GridCommandHandlerTest extends 
GridCommonAbstractTest {
      */
     @Test
     public void 
testCacheIdleVerifyDumpForCorruptedDataOnPersistenceClientCache() throws 
Exception {
-        int parts = 32;
-
-        dataRegionConfiguration = new DataRegionConfiguration()
-            .setName("persistence-region")
-            .setMaxSize(100L * 1024 * 1024)
-            .setPersistenceEnabled(true);
-
         IgniteEx ignite = (IgniteEx)startGrids(3);
 
         ignite.cluster().active(true);
 
-        IgniteCache<Object, Object> cache = ignite.createCache(new 
CacheConfiguration<>()
-            .setAffinity(new RendezvousAffinityFunction(false, parts))
-            .setBackups(2)
-            .setName(DEFAULT_CACHE_NAME)
-            .setDataRegionName("persistence-region"));
-
-        // Adding some assignments without deployments.
-        for (int i = 0; i < 100; i++)
-            cache.put(i, i);
+        createCacheAndPreload(ignite, 100);
 
-        corruptingAndCheckDefaultCache(ignite, parts, 
CacheFilterEnum.PERSISTENT);
+        corruptingAndCheckDefaultCache(ignite, CacheFilterEnum.PERSISTENT);
     }
 
     /**
@@ -1572,13 +1658,7 @@ public class GridCommandHandlerTest extends 
GridCommonAbstractTest {
 
         ignite.cluster().active(true);
 
-        IgniteCache<Object, Object> cache = ignite.createCache(new 
CacheConfiguration<>()
-            .setAffinity(new RendezvousAffinityFunction(false, 32))
-            .setBackups(1)
-            .setName(DEFAULT_CACHE_NAME));
-
-        for (int i = 0; i < 100; i++)
-            cache.put(i, i);
+        createCacheAndPreload(ignite, 100);
 
         injectTestSystemOut();
 
@@ -1623,13 +1703,7 @@ public class GridCommandHandlerTest extends 
GridCommonAbstractTest {
 
         ignite.cluster().active(true);
 
-        IgniteCache<Object, Object> cache = ignite.createCache(new 
CacheConfiguration<>()
-            .setAffinity(new RendezvousAffinityFunction(false, 32))
-            .setBackups(1)
-            .setName(DEFAULT_CACHE_NAME));
-
-        for (int i = 0; i < 100; i++)
-            cache.put(i, i);
+        createCacheAndPreload(ignite, 100);
 
         injectTestSystemOut();
 

Reply via email to