This is an automated email from the ASF dual-hosted git repository. amashenkov pushed a commit to branch gg-18540 in repository https://gitbox.apache.org/repos/asf/ignite.git
commit 85643a8babca72c334291ecaaaf236a8c95b1e56 Author: Ilya Kasnacheev <[email protected]> AuthorDate: Fri Apr 26 14:39:25 2019 +0300 GG-17344 Idle Verify --cache-filter ALL should include system caches. Also adds USER and DEFAULT values, improves help messages. --- .../internal/commandline/CommandHandler.java | 22 ++++++++---- .../internal/commandline/cache/CacheArguments.java | 2 +- .../cache/verify/IdleVerifyResultV2.java | 3 +- .../verify/VerifyBackupPartitionsDumpTask.java | 26 +++++++++++--- .../cache/verify/VerifyBackupPartitionsTaskV2.java | 36 +++++-------------- .../internal/visor/verify/CacheFilterEnum.java | 12 +++++-- .../visor/verify/VisorIdleVerifyDumpTaskArg.java | 2 +- .../apache/ignite/util/GridCommandHandlerTest.java | 42 ++++++++++++++++++---- 8 files changed, 93 insertions(+), 52 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java index 2d0fcec..a113141 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java @@ -180,6 +180,11 @@ import static org.apache.ignite.internal.commandline.cache.argument.ListCommandA import static org.apache.ignite.internal.commandline.cache.argument.ListCommandArg.SEQUENCE; import static org.apache.ignite.internal.commandline.cache.argument.ValidateIndexesCommandArg.CHECK_FIRST; import static org.apache.ignite.internal.commandline.cache.argument.ValidateIndexesCommandArg.CHECK_THROUGH; +import static org.apache.ignite.internal.visor.verify.CacheFilterEnum.ALL; +import static org.apache.ignite.internal.visor.verify.CacheFilterEnum.NOT_PERSISTENT; +import static org.apache.ignite.internal.visor.verify.CacheFilterEnum.PERSISTENT; +import static org.apache.ignite.internal.visor.verify.CacheFilterEnum.SYSTEM; +import static org.apache.ignite.internal.visor.verify.CacheFilterEnum.USER; import static org.apache.ignite.internal.visor.verify.VisorViewCacheCmd.CACHES; import static org.apache.ignite.internal.visor.verify.VisorViewCacheCmd.GROUPS; import static org.apache.ignite.internal.visor.verify.VisorViewCacheCmd.SEQ; @@ -862,8 +867,8 @@ public class CommandHandler { usageCache(LIST, "regexPattern", op(or(GROUP, SEQUENCE)), OP_NODE_ID, op(CONFIG), op(OUTPUT_FORMAT, MULTI_LINE)); usageCache(CONTENTION, "minQueueSize", OP_NODE_ID, op("maxPrint")); - usageCache(IDLE_VERIFY, op(DUMP), op(SKIP_ZEROS), op(CHECK_CRC), - op(EXCLUDE_CACHES, CACHES), op(CACHE_FILTER, or(CacheFilterEnum.values())), op(CACHES)); + usageCache(IDLE_VERIFY, op(DUMP), op(SKIP_ZEROS), op(CHECK_CRC), op(EXCLUDE_CACHES, CACHES), + op(CACHE_FILTER, or(ALL, USER, SYSTEM, PERSISTENT, NOT_PERSISTENT)), op(CACHES)); usageCache(VALIDATE_INDEXES, op(CACHES), OP_NODE_ID, op(or(CHECK_FIRST + " N", CHECK_THROUGH + " K"))); usageCache(DISTRIBUTION, or(NODE_ID, NULL), op(CACHES), op(USER_ATTRIBUTES, "attrName1,...,attrNameN")); usageCache(RESET_LOST_PARTITIONS, CACHES); @@ -1927,11 +1932,14 @@ public class CommandHandler { return "Show the keys that are point of contention for multiple transactions."; case IDLE_VERIFY: - return "Verify counters and hash sums of primary and backup partitions for the specified caches/cache groups on an idle cluster and print out the differences, if any. " + - "Cache filtering options configure the set of caches that will be processed by " + IDLE_VERIFY + " command. " + - "Default value for the set of cache names (or cache group names) is all cache groups. Default value for " + EXCLUDE_CACHES + " is empty set. " + - "Default value for " + CACHE_FILTER + " is no filtering. Therefore, the set of all caches is sequently filtered by cache name " + - "regexps, by cache type and after all by exclude regexps."; + return "Verify counters and hash sums of primary and backup partitions for the specified caches/cache " + + "groups on an idle cluster and print out the differences, if any. When no parameters are specified, " + + "all user caches are verified. Cache filtering options configure the set of caches that will be " + + "processed by " + IDLE_VERIFY + " command. If cache names are specified, in form of regular " + + "expressions, only matching caches will be verified. Caches matched by regexes specified after " + + EXCLUDE_CACHES + " parameter will be excluded from verification. Using parameter " + CACHE_FILTER + + " you can verify: only " + USER + " caches, only user " + PERSISTENT + " caches, only user " + + NOT_PERSISTENT + " caches, only " + SYSTEM + " caches, or " + ALL + " of the above."; case VALIDATE_INDEXES: return "Validate indexes on an idle cluster and print out the keys that are missing in the indexes."; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java index d2190d1..7fbf419 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java @@ -75,7 +75,7 @@ public class CacheArguments { private boolean fullConfig; /** Cache filter. */ - private CacheFilterEnum cacheFilterEnum = CacheFilterEnum.ALL; + private CacheFilterEnum cacheFilterEnum = CacheFilterEnum.DEFAULT; /** Check CRC sum on idle verify. */ private boolean idleCheckCrc; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java index 3dc5ac3..c583a2f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java @@ -189,13 +189,14 @@ public class IdleVerifyResultV2 extends VisorDataTransferObject { private void print(Consumer<String> printer, boolean printExceptionMessages) { boolean noMatchingCaches = false; - for (Exception e : exceptions.values()) + for (Exception e : exceptions.values()) { if (e instanceof NoMatchingCachesException) { noMatchingCaches = true; succeeded = false; break; } + } if (succeeded) { if (!F.isEmpty(exceptions)) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsDumpTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsDumpTask.java index e61b1d2..35a7c717 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsDumpTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsDumpTask.java @@ -37,6 +37,7 @@ import org.apache.ignite.compute.ComputeJobResultPolicy; import org.apache.ignite.compute.ComputeTaskAdapter; import org.apache.ignite.internal.processors.task.GridInternal; import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.visor.verify.CacheFilterEnum; import org.apache.ignite.internal.visor.verify.VisorIdleVerifyDumpTaskArg; import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskArg; import org.apache.ignite.resources.IgniteInstanceResource; @@ -202,13 +203,30 @@ public class VerifyBackupPartitionsDumpTask extends ComputeTaskAdapter<VisorIdle int skippedRecords, PrintWriter writer ) { - if (!F.isEmpty(conflictRes.exceptions())) { - int size = conflictRes.exceptions().size(); + Map<ClusterNode, Exception> exceptions = conflictRes.exceptions(); + + if (!F.isEmpty(exceptions)) { + boolean noMatchingCaches = false; + + for (Exception e : exceptions.values()) { + if (e instanceof NoMatchingCachesException) { + noMatchingCaches = true; + + break; + } + } + + int size = exceptions.size(); writer.write("idle_verify failed on " + size + " node" + (size == 1 ? "" : "s") + ".\n"); + + if (noMatchingCaches) + writer.write("There are no caches matching given filter options."); } - writer.write("idle_verify check has finished, found " + partitions.size() + " partitions\n"); + if (!partitions.isEmpty()) + writer.write("idle_verify check has finished, found " + partitions.size() + " partitions\n"); + writer.write("idle_verify task was executed with the following args: " + taskArgsAsCmd() + "\n"); if (skippedRecords > 0) @@ -244,7 +262,7 @@ public class VerifyBackupPartitionsDumpTask extends ComputeTaskAdapter<VisorIdle } } - if (taskArg.getCacheFilterEnum() != null) { + if (taskArg.getCacheFilterEnum() != null && taskArg.getCacheFilterEnum() != CacheFilterEnum.DEFAULT) { result.append(CACHE_FILTER); result.append(" "); result.append(taskArg.getCacheFilterEnum()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java index 945d8ef..ce2539f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java @@ -61,7 +61,6 @@ import org.apache.ignite.internal.util.lang.GridIterator; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.SB; import org.apache.ignite.internal.util.typedef.internal.U; -import org.apache.ignite.internal.visor.verify.CacheFilterEnum; import org.apache.ignite.internal.visor.verify.VisorIdleVerifyDumpTaskArg; import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskArg; import org.apache.ignite.lang.IgniteInClosure; @@ -439,19 +438,7 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe * @param cachesToFilter cache groups to filter */ private void filterByCacheFilter(Set<CacheGroupContext> cachesToFilter) { - if (onlySpecificCaches()) - cachesToFilter.removeIf(grp -> !doesGrpMatchFilter(grp)); - - boolean excludeSysCaches; - - if (arg instanceof VisorIdleVerifyDumpTaskArg) { - CacheFilterEnum filter = ((VisorIdleVerifyDumpTaskArg) arg).getCacheFilterEnum(); - - excludeSysCaches = !(filter == CacheFilterEnum.SYSTEM); - } else - excludeSysCaches = true; - - cachesToFilter.removeIf(grp -> (grp.systemCache() && excludeSysCaches) || grp.isLocal()); + cachesToFilter.removeIf(grp -> !doesGrpMatchFilter(grp)); } /** @@ -460,7 +447,7 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe * @param cachesToFilter cache groups to filter */ private void filterByCacheNames(Set<CacheGroupContext> cachesToFilter) { - if (arg.getCaches() != null && !arg.getCaches().isEmpty()) { + if (!F.isEmpty(arg.getCaches())) { Set<Pattern> cacheNamesPatterns = new HashSet<>(); for (String cacheNameRegexp : arg.getCaches()) @@ -509,19 +496,6 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe } /** - * @return True if validates only specific caches, else false. - */ - private boolean onlySpecificCaches() { - if (arg instanceof VisorIdleVerifyDumpTaskArg) { - VisorIdleVerifyDumpTaskArg vdta = (VisorIdleVerifyDumpTaskArg)arg; - - return vdta.getCacheFilterEnum() != CacheFilterEnum.ALL; - } - - return false; - } - - /** * @param desc Cache descriptor. */ private boolean isCacheMatchFilter(DynamicCacheDescriptor desc) { @@ -533,6 +507,12 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe VisorIdleVerifyDumpTaskArg vdta = (VisorIdleVerifyDumpTaskArg)arg; switch (vdta.getCacheFilterEnum()) { + case DEFAULT: + return desc.cacheType().userCache() || !F.isEmpty(arg.getCaches()); + + case USER: + return desc.cacheType().userCache(); + case SYSTEM: return !desc.cacheType().userCache(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/CacheFilterEnum.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/CacheFilterEnum.java index ed221d0..52e2df1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/CacheFilterEnum.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/CacheFilterEnum.java @@ -24,8 +24,8 @@ import org.jetbrains.annotations.Nullable; * @see org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTaskV2 */ public enum CacheFilterEnum { - /** All. */ - ALL, + /** Default - user only, or all caches specified by name. */ + DEFAULT, /** System. */ SYSTEM, @@ -34,7 +34,13 @@ public enum CacheFilterEnum { PERSISTENT, /** Not persistent. */ - NOT_PERSISTENT; + NOT_PERSISTENT, + + /** User. */ + USER, + + /** All. */ + ALL; /** Enumerated values. */ private static final CacheFilterEnum[] VALS = values(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java index a0f99e9..e2ae6bb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java @@ -115,7 +115,7 @@ public class VisorIdleVerifyDumpTaskArg extends VisorIdleVerifyTaskArg { if (protoVer >= V2) cacheFilterEnum = CacheFilterEnum.fromOrdinal(in.readByte()); else - cacheFilterEnum = CacheFilterEnum.ALL; + cacheFilterEnum = CacheFilterEnum.DEFAULT; if (protoVer >= V2) excludeCaches(U.readSet(in)); diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 1d309d2..59bf8fc 100644 --- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -1347,6 +1347,34 @@ public class GridCommandHandlerTest extends GridCommonAbstractTest { } /** + * @throws Exception If failed. + */ + @Test + public void testCacheIdleVerifyNodeFilter() throws Exception { + IgniteEx ignite = startGrids(3); + + ignite.cluster().active(true); + + UUID lastNodeId = ignite.localNode().id(); + + ignite.createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME) + .setAffinity(new RendezvousAffinityFunction(false, 32)) + .setNodeFilter(node -> !node.id().equals(lastNodeId)) + .setBackups(1)); + + try (IgniteDataStreamer streamer = ignite.dataStreamer(DEFAULT_CACHE_NAME)) { + for (int i = 0; i < 100; i++) + streamer.addData(i, i); + } + + injectTestSystemOut(); + + assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", DEFAULT_CACHE_NAME)); + + assertContains(testOut.toString(), "no conflicts have been found"); + } + + /** * Tests that both update counter and hash conflicts are detected. * * @throws Exception If failed. @@ -1529,13 +1557,13 @@ public class GridCommandHandlerTest extends GridCommonAbstractTest { testCacheIdleVerifyMultipleCacheFilterOptionsCommon( true, - "idle_verify check has finished", + "idle_verify check has finished, found 100 partitions", "idle_verify task was executed with the following args: --cache-filter SYSTEM --exclude-caches wrong.* ", "--cache", "idle_verify", "--dump", "--cache-filter", "SYSTEM", "--exclude-caches", "wrong.*" ); testCacheIdleVerifyMultipleCacheFilterOptionsCommon( true, - "idle_verify check has finished, found 96 partitions", + "idle_verify check has finished, found 196 partitions", null, "--cache", "idle_verify", "--dump", ".*", "--exclude-caches", "wrong.*" ); @@ -1555,13 +1583,13 @@ public class GridCommandHandlerTest extends GridCommonAbstractTest { true, "idle_verify check has finished, found 160 partitions", null, - "--cache", "idle_verify", "--dump", "shared.*,wrong.*", "--cache-filter", "ALL" + "--cache", "idle_verify", "--dump", "shared.*,wrong.*", "--cache-filter", "USER" ); testCacheIdleVerifyMultipleCacheFilterOptionsCommon( true, - "idle_verify check has finished, found 160 partitions", + "There are no caches matching given filter options.", null, - "--cache", "idle_verify", "--dump", "shared.*,wrong.*", "--cache-filter", "ALL" + "--cache", "idle_verify", "--dump", "shared.*,wrong.*", "--cache-filter", "SYSTEM" ); testCacheIdleVerifyMultipleCacheFilterOptionsCommon( true, @@ -1579,7 +1607,7 @@ public class GridCommandHandlerTest extends GridCommonAbstractTest { true, "idle_verify check has finished, no conflicts have been found.", null, - "--cache", "idle_verify", ".*", "--exclude-caches", "wrong-.*" + "--cache", "idle_verify", ".*", "--exclude-caches", "wrong-.*", "--cache-filter", "DEFAULT" ); testCacheIdleVerifyMultipleCacheFilterOptionsCommon( true, @@ -2088,7 +2116,7 @@ public class GridCommandHandlerTest extends GridCommonAbstractTest { if (fileNameMatcher.find()) { String dumpWithConflicts = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1)))); - assertTrue(dumpWithConflicts.contains("idle_verify check has finished, found 0 partitions")); + assertTrue(dumpWithConflicts.contains("There are no caches matching given filter options.")); } else fail("Should be found dump with conflicts");
