Repository: ignite Updated Branches: refs/heads/master 86c18998d -> 76e1fe754
IGNITE-8560 Update index validation utility to use statistical check approach - Fixes #4051. Signed-off-by: Ivan Rakov <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/76e1fe75 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/76e1fe75 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/76e1fe75 Branch: refs/heads/master Commit: 76e1fe754d7a8bd059d7ab64d17cbefa4913a702 Parents: 86c1899 Author: Sergey Chugunov <[email protected]> Authored: Thu May 24 18:11:47 2018 +0300 Committer: Ivan Rakov <[email protected]> Committed: Thu May 24 18:11:47 2018 +0300 ---------------------------------------------------------------------- .../internal/commandline/CommandHandler.java | 92 +++++++++++++++----- .../commandline/cache/CacheArguments.java | 34 ++++++++ .../verify/VisorValidateIndexesTaskArg.java | 40 ++++++++- .../commandline/CommandHandlerParsingTest.java | 87 ++++++++++++++++++ .../visor/verify/ValidateIndexesClosure.java | 66 +++++++++++++- .../visor/verify/VisorValidateIndexesTask.java | 2 +- .../util/GridCommandHandlerIndexingTest.java | 8 +- 7 files changed, 305 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/76e1fe75/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java index 47cc233..c59e348 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java @@ -181,6 +181,12 @@ public class CommandHandler { /** */ private static final String BASELINE_SET_VERSION = "version"; + /** Parameter name for validate_indexes command. */ + static final String VI_CHECK_FIRST = "checkFirst"; + + /** Parameter name for validate_indexes command. */ + static final String VI_CHECK_THROUGH = "checkThrough"; + /** */ static final String WAL_PRINT = "print"; @@ -583,10 +589,12 @@ public class CommandHandler { usage(" Show information about caches, groups or sequences that match a regex:", CACHE, " list regexPattern [groups|seq] [nodeId]"); usage(" Show hot keys that are point of contention for multiple transactions:", CACHE, " contention minQueueSize [nodeId] [maxPrint]"); usage(" Verify partition counters and hashes between primary and backups on idle cluster:", CACHE, " idle_verify [cache1,...,cacheN]"); - usage(" Validate custom indexes on idle cluster:", CACHE, " validate_indexes [cache1,...,cacheN] [nodeId]"); + usage(" Validate custom indexes on idle cluster:", CACHE, " validate_indexes [cache1,...,cacheN] [nodeId] [checkFirst|checkThrough]"); - log(" If [nodeId] is not specified, cont and validate_indexes commands will be broadcasted to all server nodes."); + log(" If [nodeId] is not specified, contention and validate_indexes commands will be broadcasted to all server nodes."); log(" Another commands where [nodeId] is optional will run on a random server node."); + log(" checkFirst numeric parameter for validate_indexes specifies number of first K keys to be validated."); + log(" checkThrough numeric parameter for validate_indexes allows to check each Kth key."); nl(); } @@ -624,7 +632,11 @@ public class CommandHandler { * @param cacheArgs Cache args. */ private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) throws GridClientException { - VisorValidateIndexesTaskArg taskArg = new VisorValidateIndexesTaskArg(cacheArgs.caches()); + VisorValidateIndexesTaskArg taskArg = new VisorValidateIndexesTaskArg( + cacheArgs.caches(), + cacheArgs.checkFirst(), + cacheArgs.checkThrough() + ); UUID nodeId = cacheArgs.nodeId() == null ? BROADCAST_UUID : cacheArgs.nodeId(); @@ -1407,7 +1419,8 @@ public class CommandHandler { break; case IDLE_VERIFY: - parseCacheNamesIfPresent(cacheArgs); + if (hasNextCacheArg()) + parseCacheNames(nextArg(""), cacheArgs); break; @@ -1425,10 +1438,53 @@ public class CommandHandler { break; case VALIDATE_INDEXES: - parseCacheNamesIfPresent(cacheArgs); + int argsCnt = 0; - if (hasNextCacheArg()) - cacheArgs.nodeId(UUID.fromString(nextArg(""))); + while (hasNextCacheArg() && argsCnt++ < 4) { + String arg = nextArg(""); + + if (VI_CHECK_FIRST.equals(arg) || VI_CHECK_THROUGH.equals(arg)) { + if (!hasNextCacheArg()) + throw new IllegalArgumentException("Numeric value for '" + arg + "' parameter expected."); + + int numVal; + + String numStr = nextArg(""); + + try { + numVal = Integer.parseInt(numStr); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + "Not numeric value was passed for '" + + arg + + "' parameter: " + + numStr + ); + } + + if (numVal <= 0) + throw new IllegalArgumentException("Value for '" + arg + "' property should be positive."); + + if (VI_CHECK_FIRST.equals(arg)) + cacheArgs.checkFirst(numVal); + else + cacheArgs.checkThrough(numVal); + + continue; + } + + try { + cacheArgs.nodeId(UUID.fromString(arg)); + + continue; + } + catch (IllegalArgumentException ignored) { + //No-op. + } + + parseCacheNames(arg, cacheArgs); + } break; @@ -1473,22 +1529,18 @@ public class CommandHandler { /** * @param cacheArgs Cache args. */ - private void parseCacheNamesIfPresent(CacheArguments cacheArgs) { - if (hasNextCacheArg()) { - String cacheNames = nextArg(""); - - String[] cacheNamesArr = cacheNames.split(","); - Set<String> cacheNamesSet = new HashSet<>(); + private void parseCacheNames(String cacheNames, CacheArguments cacheArgs) { + String[] cacheNamesArr = cacheNames.split(","); + Set<String> cacheNamesSet = new HashSet<>(); - for (String cacheName : cacheNamesArr) { - if (F.isEmpty(cacheName)) - throw new IllegalArgumentException("Non-empty cache names expected."); + for (String cacheName : cacheNamesArr) { + if (F.isEmpty(cacheName)) + throw new IllegalArgumentException("Non-empty cache names expected."); - cacheNamesSet.add(cacheName.trim()); - } - - cacheArgs.caches(cacheNamesSet); + cacheNamesSet.add(cacheName.trim()); } + + cacheArgs.caches(cacheNamesSet); } /** http://git-wip-us.apache.org/repos/asf/ignite/blob/76e1fe75/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java index 6f315ef..1411b2a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java @@ -46,6 +46,12 @@ public class CacheArguments { /** Max print. */ private int maxPrint; + /** validate_indexes 'checkFirst' argument */ + private int checkFirst = -1; + + /** validate_indexes 'checkThrough' argument */ + private int checkThrough = -1; + /** Cache view command. */ private @Nullable VisorViewCacheCmd cacheCmd; @@ -160,4 +166,32 @@ public class CacheArguments { public void maxPrint(int maxPrint) { this.maxPrint = maxPrint; } + + /** + * @return Max number of entries to be checked. + */ + public int checkFirst() { + return checkFirst; + } + + /** + * @param checkFirst Max number of entries to be checked. + */ + public void checkFirst(int checkFirst) { + this.checkFirst = checkFirst; + } + + /** + * @return Number of entries to check through. + */ + public int checkThrough() { + return checkThrough; + } + + /** + * @param checkThrough Number of entries to check through. + */ + public void checkThrough(int checkThrough) { + this.checkThrough = checkThrough; + } } http://git-wip-us.apache.org/repos/asf/ignite/blob/76e1fe75/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java index cf9aff5..aa49977 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java @@ -35,6 +35,12 @@ public class VisorValidateIndexesTaskArg extends VisorDataTransferObject { /** Caches. */ private Set<String> caches; + /** Check first K elements. */ + private int checkFirst; + + /** Check through K element (skip K-1, check Kth). */ + private int checkThrough; + /** * Default constructor. */ @@ -45,8 +51,10 @@ public class VisorValidateIndexesTaskArg extends VisorDataTransferObject { /** * @param caches Caches. */ - public VisorValidateIndexesTaskArg(Set<String> caches) { + public VisorValidateIndexesTaskArg(Set<String> caches, int checkFirst, int checkThrough) { this.caches = caches; + this.checkFirst = checkFirst; + this.checkThrough = checkThrough; } @@ -57,14 +65,44 @@ public class VisorValidateIndexesTaskArg extends VisorDataTransferObject { return caches; } + /** + * @return checkFirst. + */ + public int getCheckFirst() { + return checkFirst; + } + + /** + * @return checkThrough. + */ + public int getCheckThrough() { + return checkThrough; + } + /** {@inheritDoc} */ @Override protected void writeExternalData(ObjectOutput out) throws IOException { U.writeCollection(out, caches); + out.writeInt(checkFirst); + out.writeInt(checkThrough); } /** {@inheritDoc} */ @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException { caches = U.readSet(in); + + if (protoVer > V1) { + checkFirst = in.readInt(); + checkThrough = in.readInt(); + } + else { + checkFirst = -1; + checkThrough = -1; + } + } + + /** {@inheritDoc} */ + @Override public byte getProtocolVersion() { + return V2; } /** {@inheritDoc} */ http://git-wip-us.apache.org/repos/asf/ignite/blob/76e1fe75/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java index 2fc40ca..737c0c7 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java @@ -21,15 +21,20 @@ import java.util.Arrays; import java.util.Collections; import java.util.UUID; import junit.framework.TestCase; +import org.apache.ignite.internal.commandline.cache.CacheArguments; +import org.apache.ignite.internal.commandline.cache.CacheCommand; import org.apache.ignite.internal.visor.tx.VisorTxProjection; import org.apache.ignite.internal.visor.tx.VisorTxSortOrder; import org.apache.ignite.internal.visor.tx.VisorTxTaskArg; import static java.util.Arrays.asList; import static org.apache.ignite.IgniteSystemProperties.IGNITE_ENABLE_EXPERIMENTAL_COMMAND; +import static org.apache.ignite.internal.commandline.Command.CACHE; import static org.apache.ignite.internal.commandline.Command.WAL; import static org.apache.ignite.internal.commandline.CommandHandler.DFLT_HOST; import static org.apache.ignite.internal.commandline.CommandHandler.DFLT_PORT; +import static org.apache.ignite.internal.commandline.CommandHandler.VI_CHECK_FIRST; +import static org.apache.ignite.internal.commandline.CommandHandler.VI_CHECK_THROUGH; import static org.apache.ignite.internal.commandline.CommandHandler.WAL_DELETE; import static org.apache.ignite.internal.commandline.CommandHandler.WAL_PRINT; @@ -52,6 +57,88 @@ public class CommandHandlerParsingTest extends TestCase { } /** + * validate_indexes command arguments parsing and validation + */ + public void testValidateIndexArguments() { + CommandHandler hnd = new CommandHandler(); + + //happy case for all parameters + try { + int expectedCheckFirst = 10; + int expectedCheckThrough = 11; + UUID nodeId = UUID.randomUUID(); + + CacheArguments args = hnd.parseAndValidate( + Arrays.asList( + CACHE.text(), + CacheCommand.VALIDATE_INDEXES.text(), + "cache1, cache2", + nodeId.toString(), + VI_CHECK_FIRST, + Integer.toString(expectedCheckFirst), + VI_CHECK_THROUGH, + Integer.toString(expectedCheckThrough) + ) + ).cacheArgs(); + + assertEquals("nodeId parameter unexpected value", nodeId, args.nodeId()); + assertEquals("checkFirst parameter unexpected value", expectedCheckFirst, args.checkFirst()); + assertEquals("checkThrough parameter unexpected value", expectedCheckThrough, args.checkThrough()); + } + catch (IllegalArgumentException e) { + fail("Unexpected exception: " + e); + } + + try { + int expectedParam = 11; + UUID nodeId = UUID.randomUUID(); + + CacheArguments args = hnd.parseAndValidate( + Arrays.asList( + CACHE.text(), + CacheCommand.VALIDATE_INDEXES.text(), + nodeId.toString(), + VI_CHECK_THROUGH, + Integer.toString(expectedParam) + ) + ).cacheArgs(); + + assertNull("caches weren't specified, null value expected", args.caches()); + assertEquals("nodeId parameter unexpected value", nodeId, args.nodeId()); + assertEquals("checkFirst parameter unexpected value", -1, args.checkFirst()); + assertEquals("checkThrough parameter unexpected value", expectedParam, args.checkThrough()); + } + catch (IllegalArgumentException e) { + e.printStackTrace(); + } + + try { + hnd.parseAndValidate( + Arrays.asList( + CACHE.text(), + CacheCommand.VALIDATE_INDEXES.text(), + VI_CHECK_FIRST, + "0" + ) + ); + + fail("Expected exception hasn't been thrown"); + } + catch (IllegalArgumentException e) { + e.printStackTrace(); + } + + try { + hnd.parseAndValidate(Arrays.asList(CACHE.text(), CacheCommand.VALIDATE_INDEXES.text(), VI_CHECK_THROUGH)); + + fail("Expected exception hasn't been thrown"); + } + catch (IllegalArgumentException e) { + e.printStackTrace(); + } + } + + /** * Test that experimental command (i.e. WAL command) is disabled by default. */ public void testExperimentalCommandIsDisabled() { http://git-wip-us.apache.org/repos/asf/ignite/blob/76e1fe75/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java index e0eff61..e01dca2 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java @@ -89,6 +89,12 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex /** Cache names. */ private Set<String> cacheNames; + /** If provided only first K elements will be validated. */ + private final int checkFirst; + + /** If provided only each Kth element will be validated. */ + private final int checkThrough; + /** Counter of processed partitions. */ private final AtomicInteger processedPartitions = new AtomicInteger(0); @@ -109,9 +115,13 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex /** * @param cacheNames Cache names. + * @param checkFirst If positive only first K elements will be validated. + * @param checkThrough If positive only each Kth element will be validated. */ - public ValidateIndexesClosure(Set<String> cacheNames) { + public ValidateIndexesClosure(Set<String> cacheNames, int checkFirst, int checkThrough) { this.cacheNames = cacheNames; + this.checkFirst = checkFirst; + this.checkThrough = checkThrough; } /** {@inheritDoc} */ @@ -320,12 +330,39 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex m.setAccessible(true); + final boolean skipConditions = checkFirst > 0 || checkThrough > 0; + final boolean bothSkipConditions = checkFirst > 0 && checkThrough > 0; + + long current = 0; + long processedNumber = 0; + while (it.hasNextX()) { if (enoughIssues) break; CacheDataRow row = it.nextX(); + if (skipConditions) { + if (bothSkipConditions) { + if (processedNumber > checkFirst) + break; + else if (current++ % checkThrough > 0) + continue; + else + processedNumber++; + } + else { + if (checkFirst > 0) { + if (current++ > checkFirst) + break; + } + else { + if (current++ % checkThrough > 0) + continue; + } + } + } + int cacheId = row.cacheId() == 0 ? grpCtx.groupId() : row.cacheId(); GridCacheContext cacheCtx = row.cacheId() == 0 ? @@ -462,6 +499,12 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex enoughIssues = true; } + final boolean skipConditions = checkFirst > 0 || checkThrough > 0; + final boolean bothSkipConditions = checkFirst > 0 && checkThrough > 0; + + long current = 0; + long processedNumber = 0; + while (!enoughIssues) { KeyCacheObject h2key = null; @@ -471,6 +514,27 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex GridH2Row h2Row = (GridH2Row)cursor.get(); + if (skipConditions) { + if (bothSkipConditions) { + if (processedNumber > checkFirst) + break; + else if (current++ % checkThrough > 0) + continue; + else + processedNumber++; + } + else { + if (checkFirst > 0) { + if (current++ > checkFirst) + break; + } + else { + if (current++ % checkThrough > 0) + continue; + } + } + } + h2key = h2Row.key(); CacheDataRow cacheDataStoreRow = ctx.group().offheap().read(ctx, h2key); http://git-wip-us.apache.org/repos/asf/ignite/blob/76e1fe75/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java index 52b48a5..abb7f7e 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java @@ -76,7 +76,7 @@ public class VisorValidateIndexesTask extends VisorMultiNodeTask<VisorValidateIn /** {@inheritDoc} */ @Override protected VisorValidateIndexesJobResult run(@Nullable VisorValidateIndexesTaskArg arg) throws IgniteException { try { - ValidateIndexesClosure clo = new ValidateIndexesClosure(arg.getCaches()); + ValidateIndexesClosure clo = new ValidateIndexesClosure(arg.getCaches(), arg.getCheckFirst(), arg.getCheckThrough()); ignite.context().resource().injectGeneric(clo); http://git-wip-us.apache.org/repos/asf/ignite/blob/76e1fe75/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java index 62d3fc0..ca9aa53 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java @@ -101,7 +101,13 @@ public class GridCommandHandlerIndexingTest extends GridCommandHandlerTest { injectTestSystemOut(); - assertEquals(EXIT_CODE_OK, execute("--cache", "validate_indexes", cacheName)); + assertEquals(EXIT_CODE_OK, + execute( + "--cache", + "validate_indexes", + cacheName, + "checkFirst", "10000", + "checkThrough", "10")); assertTrue(testOut.toString().contains("validate_indexes has finished with errors")); }
