Repository: ignite Updated Branches: refs/heads/master 4348915d6 -> 9adc29593
IGNITE-10037 Cache 2 tests optimization - Fixes #5207 Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/9adc2959 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/9adc2959 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/9adc2959 Branch: refs/heads/master Commit: 9adc2959356c817e1f018e2d0fe5dda963196c15 Parents: 4348915 Author: Alexey Platonov <aplaton...@gmail.com> Authored: Sat Nov 3 23:51:28 2018 +0300 Committer: Dmitriy Pavlov <dpav...@apache.org> Committed: Sat Nov 3 23:51:28 2018 +0300 ---------------------------------------------------------------------- .../IgniteCacheEntryProcessorNodeJoinTest.java | 20 ++++--- ...CacheLoadingConcurrentGridStartSelfTest.java | 61 ++++++++++++-------- ...ridCachePartitionNotLoadedEventSelfTest.java | 11 ++-- .../ignite/testframework/GridTestUtils.java | 52 ++++++++++++++++- .../testframework/junits/GridAbstractTest.java | 43 +++++++------- 5 files changed, 131 insertions(+), 56 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/9adc2959/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryProcessorNodeJoinTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryProcessorNodeJoinTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryProcessorNodeJoinTest.java index ec98294..d6afb49 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryProcessorNodeJoinTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryProcessorNodeJoinTest.java @@ -42,6 +42,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.GridTestUtils.SF; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.jetbrains.annotations.Nullable; @@ -61,10 +62,13 @@ public class IgniteCacheEntryProcessorNodeJoinTest extends GridCommonAbstractTes private static final int GRID_CNT = 2; /** Number of increment iterations. */ - private static final int INCREMENTS = 100; + private final int INCREMENTS = SF.apply(100); + + /** Number of test iterations. */ + private final int ITERATIONS = SF.applyLB(10, 2); /** */ - private static final int KEYS = 50; + private final int KEYS = SF.apply(50); /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { @@ -160,7 +164,7 @@ public class IgniteCacheEntryProcessorNodeJoinTest extends GridCommonAbstractTes final int RESTART_IDX = GRID_CNT + 1; - for (int iter = 0; iter < 10; iter++) { + for (int iter = 0; iter < ITERATIONS; iter++) { log.info("Iteration: " + iter); startGrid(RESTART_IDX); @@ -179,7 +183,7 @@ public class IgniteCacheEntryProcessorNodeJoinTest extends GridCommonAbstractTes } }, "stop-thread"); - int increments = checkIncrement(cacheName, iter % 2 == 2, fut, latch); + int increments = checkIncrement(cacheName, iter % 2 == 1, fut, latch); assert increments >= INCREMENTS; @@ -257,8 +261,8 @@ public class IgniteCacheEntryProcessorNodeJoinTest extends GridCommonAbstractTes * @param invokeAll If {@code true} tests invokeAll operation. * @param fut If not null then executes updates while future is not done. * @param latch Latch to count down when first update is done. - * @throws Exception If failed. * @return Number of increments. + * @throws Exception If failed. */ private int checkIncrement( String cacheName, @@ -289,7 +293,7 @@ public class IgniteCacheEntryProcessorNodeJoinTest extends GridCommonAbstractTes EntryProcessorResult<Integer> res = resMap.get(key); assertNotNull(res); - assertEquals(k + 1, (Object) res.get()); + assertEquals(k + 1, (Object)res.get()); } } else { @@ -362,7 +366,7 @@ public class IgniteCacheEntryProcessorNodeJoinTest extends GridCommonAbstractTes info("Will put: " + (updVal + 1)); for (int i = 0; i < keys; i++) - assertTrue("Failed [key=" + i + ", oldVal=" + updVal+ ']', + assertTrue("Failed [key=" + i + ", oldVal=" + updVal + ']', ignite(0).cache(DEFAULT_CACHE_NAME).replace(i, updVal, updVal + 1)); updVal++; @@ -417,4 +421,4 @@ public class IgniteCacheEntryProcessorNodeJoinTest extends GridCommonAbstractTes return vals.size(); } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/ignite/blob/9adc2959/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLoadingConcurrentGridStartSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLoadingConcurrentGridStartSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLoadingConcurrentGridStartSelfTest.java index cbd0124..5cb206f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLoadingConcurrentGridStartSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLoadingConcurrentGridStartSelfTest.java @@ -22,6 +22,7 @@ import java.util.HashSet; import java.util.LinkedHashSet; import java.util.Set; import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; import javax.cache.Cache; import javax.cache.configuration.FactoryBuilder; import javax.cache.integration.CacheLoaderException; @@ -261,8 +262,11 @@ public class CacheLoadingConcurrentGridStartSelfTest extends GridCommonAbstractT } }); + CountDownLatch startNodesLatch = new CountDownLatch(1); IgniteInternalFuture<Object> fut = runAsync(new Callable<Object>() { @Override public Object call() throws Exception { + startNodesLatch.await(); + for (int i = 2; i < GRIDS_CNT; i++) startGrid(i); @@ -272,24 +276,35 @@ public class CacheLoadingConcurrentGridStartSelfTest extends GridCommonAbstractT final HashSet<IgniteFuture> set = new HashSet<>(); - IgniteInClosure<Ignite> f = new IgniteInClosure<Ignite>() { - @Override public void apply(Ignite grid) { - try (IgniteDataStreamer<Integer, String> dataStreamer = grid.dataStreamer(DEFAULT_CACHE_NAME)) { - dataStreamer.allowOverwrite(allowOverwrite); + boolean stop = false; + int insertedKeys = 0; - ((DataStreamerImpl)dataStreamer).maxRemapCount(Integer.MAX_VALUE); + startNodesLatch.countDown(); - for (int i = 0; i < KEYS_CNT; i++) { - set.add(dataStreamer.addData(i, "Data")); + try (IgniteDataStreamer<Integer, String> dataStreamer = g0.dataStreamer(DEFAULT_CACHE_NAME)) { + dataStreamer.allowOverwrite(allowOverwrite); + ((DataStreamerImpl)dataStreamer).maxRemapCount(Integer.MAX_VALUE); - if (i % 100000 == 0) - log.info("Streaming " + i + "'th entry."); - } - } - } - }; + long startingEndTs = -1L; + + while (!stop) { + set.add(dataStreamer.addData(insertedKeys, "Data")); + insertedKeys = insertedKeys + 1; - f.apply(g0); + if (insertedKeys % 100000 == 0) + log.info("Streaming " + insertedKeys + "'th entry."); + + //When all nodes started we continue restart nodes during 1 second and stop it after this timeout. + if (fut.isDone() && startingEndTs == -1) + startingEndTs = System.currentTimeMillis(); + + if (startingEndTs != -1) //Nodes starting was ended and we check restarts duration after it. + restarts = (System.currentTimeMillis() - startingEndTs) < 1000; + + //Stop test when all keys were inserted or restarts timeout was exceeded. + stop = insertedKeys >= KEYS_CNT || (fut.isDone() && !restarts); + } + } log.info("Data loaded."); @@ -305,10 +320,10 @@ public class CacheLoadingConcurrentGridStartSelfTest extends GridCommonAbstractT long size = cache.size(CachePeekMode.PRIMARY); - if (size != KEYS_CNT) { + if (size != insertedKeys) { Set<Integer> failedKeys = new LinkedHashSet<>(); - for (int i = 0; i < KEYS_CNT; i++) + for (int i = 0; i < insertedKeys; i++) if (!cache.containsKey(i)) { log.info("Actual cache size: " + size); @@ -336,7 +351,7 @@ public class CacheLoadingConcurrentGridStartSelfTest extends GridCommonAbstractT assert failedKeys.isEmpty() : "Some failed keys: " + failedKeys.toString(); } - assertCacheSize(); + assertCacheSize(insertedKeys); } /** @@ -361,20 +376,20 @@ public class CacheLoadingConcurrentGridStartSelfTest extends GridCommonAbstractT fut.get(); } - assertCacheSize(); + assertCacheSize(KEYS_CNT); } /** * @throws Exception If failed. */ - private void assertCacheSize() throws Exception { + private void assertCacheSize(int expectedCacheSize) throws Exception { final IgniteCache<Integer, String> cache = grid(0).cache(DEFAULT_CACHE_NAME); boolean consistentCache = GridTestUtils.waitForCondition(new GridAbsPredicate() { @Override public boolean apply() { int size = cache.size(CachePeekMode.PRIMARY); - if (size != KEYS_CNT) + if (size != expectedCacheSize) log.info("Cache size: " + size); int total = 0; @@ -382,10 +397,10 @@ public class CacheLoadingConcurrentGridStartSelfTest extends GridCommonAbstractT for (int i = 0; i < GRIDS_CNT; i++) total += grid(i).cache(DEFAULT_CACHE_NAME).localSize(CachePeekMode.PRIMARY); - if (total != KEYS_CNT) + if (total != expectedCacheSize) log.info("Total size: " + size); - return size == KEYS_CNT && KEYS_CNT == total; + return size == expectedCacheSize && expectedCacheSize == total; } }, 2 * 60_000); @@ -418,4 +433,4 @@ public class CacheLoadingConcurrentGridStartSelfTest extends GridCommonAbstractT // No-op. } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/ignite/blob/9adc2959/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionNotLoadedEventSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionNotLoadedEventSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionNotLoadedEventSelfTest.java index d25304b..92783b0 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionNotLoadedEventSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionNotLoadedEventSelfTest.java @@ -38,6 +38,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.GridTestUtils.SF; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.util.TestTcpCommunicationSpi; import org.eclipse.jetty.util.ConcurrentHashSet; @@ -140,17 +141,19 @@ public class GridCachePartitionNotLoadedEventSelfTest extends GridCommonAbstract assert !cache.containsKey(key); - GridTestUtils.waitForCondition(new GridAbsPredicate() { + final long awaitingTimeoutMs = SF.apply(5 * 60 * 1000); + + assertTrue(GridTestUtils.waitForCondition(new GridAbsPredicate() { @Override public boolean apply() { return !lsnr1.lostParts.isEmpty(); } - }, getTestTimeout()); + }, awaitingTimeoutMs)); - GridTestUtils.waitForCondition(new GridAbsPredicate() { + assertTrue(GridTestUtils.waitForCondition(new GridAbsPredicate() { @Override public boolean apply() { return !lsnr2.lostParts.isEmpty(); } - }, getTestTimeout()); + }, awaitingTimeoutMs)); } /** http://git-wip-us.apache.org/repos/asf/ignite/blob/9adc2959/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java index 295e40a..12ac432 100644 --- a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java @@ -180,6 +180,11 @@ public final class GridTestUtils { } } + /** Test parameters scale factor util. */ + public static final class SF extends ScaleFactorUtil { + + } + /** */ private static final Map<Class<?>, String> addrs = new HashMap<>(); @@ -1555,7 +1560,7 @@ public final class GridTestUtils { @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter") @Nullable public static <T> T invoke(Object obj, String mtd, Object... params) throws Exception { Class<?> cls = obj.getClass(); - + do { // We cannot resolve method by parameter classes due to some of parameters can be null. // Search correct method among all methods collection. @@ -2046,4 +2051,49 @@ public final class GridTestUtils { ((IgniteEx)node).context().cache().context().exchange().mergeExchangesTestWaitVersion( new AffinityTopologyVersion(topVer, 0), mergedEvts); } + + /** Test parameters scale factor util. */ + private static class ScaleFactorUtil { + /** Test speed scale factor property name. */ + private static final String TEST_SCALE_FACTOR_PROPERTY = "TEST_SCALE_FACTOR"; + + /** Min test scale factor value. */ + private static final double MIN_TEST_SCALE_FACTOR_VALUE = 0.1; + + /** Max test scale factor value. */ + private static final double MAX_TEST_SCALE_FACTOR_VALUE = 1.0; + + /** Test speed scale factor. */ + private static final double TEST_SCALE_FACTOR_VALUE = readScaleFactor(); + + /** */ + private static double readScaleFactor() { + double scaleFactor = Double.parseDouble(System.getProperty(TEST_SCALE_FACTOR_PROPERTY, "1.0")); + + scaleFactor = Math.max(scaleFactor, MIN_TEST_SCALE_FACTOR_VALUE); + scaleFactor = Math.min(scaleFactor, MAX_TEST_SCALE_FACTOR_VALUE); + + return scaleFactor; + } + + /** */ + public static int apply(int val) { + return (int) (TEST_SCALE_FACTOR_VALUE * val); + } + + /** */ + public static int apply(int val, int lowerBound, int upperBound) { + return applyUB(applyLB(val, lowerBound), upperBound); + } + + /** Apply scale factor with lower bound */ + public static int applyLB(int val, int lowerBound) { + return Math.max((int) (TEST_SCALE_FACTOR_VALUE * val), lowerBound); + } + + /** Apply scale factor with upper bound */ + public static int applyUB(int val, int upperBound) { + return Math.min((int) (TEST_SCALE_FACTOR_VALUE * val), upperBound); + } + } } http://git-wip-us.apache.org/repos/asf/ignite/blob/9adc2959/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java index af573fe..25ed470 100755 --- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java @@ -203,7 +203,7 @@ public abstract class GridAbstractTest extends TestCase { * */ private static final boolean PERSISTENCE_ALLOWED = - IgniteSystemProperties.getBoolean(PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, true); + IgniteSystemProperties.getBoolean(PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, true); /** * @@ -400,8 +400,8 @@ public abstract class GridAbstractTest extends TestCase { } /** - * Runs given code in multiple threads and synchronously waits for all threads to complete. - * If any thread failed, exception will be thrown out of this method. + * Runs given code in multiple threads and synchronously waits for all threads to complete. If any thread failed, + * exception will be thrown out of this method. * * @param r Runnable. * @param threadNum Thread number. @@ -502,7 +502,8 @@ public abstract class GridAbstractTest extends TestCase { * @throws Exception If failed. * @return Future. */ - protected IgniteInternalFuture<?> multithreadedAsync(Callable<?> c, int threadNum, String threadName) throws Exception { + protected IgniteInternalFuture<?> multithreadedAsync(Callable<?> c, int threadNum, + String threadName) throws Exception { return GridTestUtils.runMultiThreadedAsync(c, threadNum, threadName); } @@ -541,8 +542,8 @@ public abstract class GridAbstractTest extends TestCase { } /** - * Called after execution of every test method in class or - * if {@link #beforeTest()} failed without test method execution. + * Called after execution of every test method in class or if {@link #beforeTest()} failed without test method + * execution. * * @throws Exception If failed. */ @@ -604,8 +605,8 @@ public abstract class GridAbstractTest extends TestCase { if (isFirstTest()) { info(">>> Starting test class: " + testClassDescription() + " <<<"); - if(isSafeTopology()) - assert G.allGrids().isEmpty() : "Not all Ignite instances stopped before tests execution:" + G.allGrids(); + if (isSafeTopology()) + assert G.allGrids().isEmpty() : "Not all Ignite instances stopped before tests execution:" + G.allGrids(); if (startGrid) { IgniteConfiguration cfg = optimize(getConfiguration()); @@ -1012,7 +1013,7 @@ public abstract class GridAbstractTest extends TestCase { */ protected Ignite startGridWithSpringCtx(String gridName, boolean client, String cfgUrl) throws Exception { IgniteBiTuple<Collection<IgniteConfiguration>, ? extends GridSpringResourceContext> cfgMap = - IgnitionEx.loadConfigurations(cfgUrl); + IgnitionEx.loadConfigurations(cfgUrl); IgniteConfiguration cfg = F.first(cfgMap.get1()); @@ -1095,7 +1096,7 @@ public abstract class GridAbstractTest extends TestCase { m.setAccessible(true); - cfg.setDiscoverySpi((DiscoverySpi) m.invoke(discoverySpi)); + cfg.setDiscoverySpi((DiscoverySpi)m.invoke(discoverySpi)); resetDiscovery = false; } @@ -1566,6 +1567,7 @@ public abstract class GridAbstractTest extends TestCase { /** * Create instance of {@link BinaryMarshaller} suitable for use * without starting a grid upon an empty {@link IgniteConfiguration}. + * * @return Binary marshaller. * @throws IgniteCheckedException if failed. */ @@ -1576,6 +1578,7 @@ public abstract class GridAbstractTest extends TestCase { /** * Create instance of {@link BinaryMarshaller} suitable for use * without starting a grid upon given {@link IgniteConfiguration}. + * * @return Binary marshaller. * @throws IgniteCheckedException if failed. */ @@ -1663,10 +1666,10 @@ public abstract class GridAbstractTest extends TestCase { /** * This method should be overridden by subclasses to change configuration parameters. * - * @return Grid configuration used for starting of grid. * @param igniteInstanceName Ignite instance name. * @param rsrcs Resources. * @throws Exception If failed. + * @return Grid configuration used for starting of grid. */ @SuppressWarnings("deprecation") protected IgniteConfiguration getConfiguration(String igniteInstanceName, IgniteTestResources rsrcs) @@ -2023,7 +2026,7 @@ public abstract class GridAbstractTest extends TestCase { * @param cache Cache. * @param job Job. */ - public static <K,V,R> R executeOnLocalOrRemoteJvm(IgniteCache<K,V> cache, TestCacheCallable<K,V,R> job) { + public static <K, V, R> R executeOnLocalOrRemoteJvm(IgniteCache<K, V> cache, TestCacheCallable<K, V, R> job) { Ignite ignite = cache.unwrap(Ignite.class); if (!isMultiJvmObject(ignite)) @@ -2082,7 +2085,7 @@ public abstract class GridAbstractTest extends TestCase { @Override public R call() throws Exception { Ignite ignite = Ignition.ignite(id); - IgniteCache<K,V> cache = ignite.cache(cacheName); + IgniteCache<K, V> cache = ignite.cache(cacheName); return job.call(ignite, cache); } @@ -2135,7 +2138,7 @@ public abstract class GridAbstractTest extends TestCase { if (runner.isAlive()) { U.error(log, "Test has been timed out and will be interrupted (threads dump will be taken before interruption) [" + - "test=" + getName() + ", timeout=" + getTestTimeout() + ']'); + "test=" + getName() + ", timeout=" + getTestTimeout() + ']'); List<Ignite> nodes = IgnitionEx.allGridsx(); @@ -2153,7 +2156,7 @@ public abstract class GridAbstractTest extends TestCase { U.join(runner, log); throw new TimeoutException("Test has been timed out [test=" + getName() + ", timeout=" + - getTestTimeout() + ']' ); + getTestTimeout() + ']'); } Throwable t = ex.get(); @@ -2168,8 +2171,7 @@ public abstract class GridAbstractTest extends TestCase { } /** - * @return Error handler to process all uncaught exceptions of the test run - * ({@code null} by default). + * @return Error handler to process all uncaught exceptions of the test run ({@code null} by default). */ protected IgniteClosure<Throwable, Throwable> errorHandler() { return null; @@ -2254,7 +2256,7 @@ public abstract class GridAbstractTest extends TestCase { * @param itfClses Interfaces that should be implemented by proxy (vararg parameter) * @return Created proxy. */ - protected <T> T notSerializableProxy(final T obj, Class<? super T> itfCls, Class<? super T> ... itfClses) { + protected <T> T notSerializableProxy(final T obj, Class<? super T> itfCls, Class<? super T>... itfClses) { Class<?>[] itfs = Arrays.copyOf(itfClses, itfClses.length + 3); itfs[itfClses.length] = itfCls; @@ -2315,7 +2317,7 @@ public abstract class GridAbstractTest extends TestCase { AffinityTopologyVersion exchVer = ctx.cache().context().exchange().readyAffinityVersion(); if (!topVer.equals(exchVer)) { - info("Topology version mismatch [node=" + g.name() + + info("Topology version mismatch [node=" + g.name() + ", exchVer=" + exchVer + ", topVer=" + topVer + ']'); @@ -2330,6 +2332,7 @@ public abstract class GridAbstractTest extends TestCase { } } } + /** * @param expSize Expected nodes number. * @throws Exception If failed. @@ -2345,7 +2348,7 @@ public abstract class GridAbstractTest extends TestCase { return false; } - for (Ignite node: nodes) { + for (Ignite node : nodes) { try { IgniteFuture<?> reconnectFut = node.cluster().clientReconnectFuture();
