Repository: ignite Updated Branches: refs/heads/ignite-971 07b476aaa -> c05bf538d
ignite-971 Fix offheap to swap eviction. Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c05bf538 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c05bf538 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c05bf538 Branch: refs/heads/ignite-971 Commit: c05bf538dd36e74569cfa6908d58a7bffd9a92af Parents: 07b476a Author: sboikov <[email protected]> Authored: Fri Sep 11 12:59:14 2015 +0300 Committer: sboikov <[email protected]> Committed: Fri Sep 11 13:04:27 2015 +0300 ---------------------------------------------------------------------- .../processors/cache/GridCacheSwapManager.java | 6 +- .../cache/GridCachePutAllFailoverSelfTest.java | 60 ++++++++++ .../ignite/testframework/GridTestUtils.java | 118 +++++++++++++++++++ ...lientQueryReplicatedNodeRestartSelfTest.java | 8 +- .../IgniteCacheQueryNodeRestartSelfTest.java | 4 +- .../IgniteCacheQueryNodeRestartSelfTest2.java | 8 +- 6 files changed, 197 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/c05bf538/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSwapManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSwapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSwapManager.java index 92fb065..b5f754a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSwapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSwapManager.java @@ -34,6 +34,7 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.internal.managers.swapspace.GridSwapSpaceManager; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException; import org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; import org.apache.ignite.internal.processors.cache.version.GridCacheVersionAware; @@ -167,6 +168,9 @@ public class GridCacheSwapManager extends GridCacheManagerAdapter { } } } + catch (GridDhtInvalidPartitionException e) { + // Skip entry. + } catch (IgniteCheckedException e) { U.error(log, "Failed to unmarshal off-heap entry", e); } @@ -238,7 +242,7 @@ public class GridCacheSwapManager extends GridCacheManagerAdapter { if (cctx.config().isStatisticsEnabled()) cctx.cache().metrics0().onOffHeapEvict(); - if (!firstEvictWarn) + if (firstEvictWarn) return; synchronized (this) { http://git-wip-us.apache.org/repos/asf/ignite/blob/c05bf538/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePutAllFailoverSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePutAllFailoverSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePutAllFailoverSelfTest.java index 26911a3..66377a9 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePutAllFailoverSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePutAllFailoverSelfTest.java @@ -60,6 +60,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.spi.failover.FailoverContext; import org.apache.ignite.spi.failover.always.AlwaysFailoverSpi; +import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -101,6 +102,9 @@ public class GridCachePutAllFailoverSelfTest extends GridCommonAbstractTest { /** Backups count. */ private int backups; + /** */ + private GridTestUtils.TestMemoryMode memMode = GridTestUtils.TestMemoryMode.HEAP; + /** Filter to include only worker nodes. */ private static final IgnitePredicate<ClusterNode> workerNodesFilter = new PN() { @SuppressWarnings("unchecked") @@ -202,6 +206,60 @@ public class GridCachePutAllFailoverSelfTest extends GridCommonAbstractTest { checkPutAllFailoverColocated(false, 5, 2); } + /** + * @throws Exception If failed. + */ + public void testPutAllFailoverColocatedNearEnabledTwoBackupsSwap() throws Exception { + memMode = GridTestUtils.TestMemoryMode.SWAP; + + checkPutAllFailoverColocated(true, 5, 2); + } + + /** + * @throws Exception If failed. + */ + public void testPutAllFailoverColocatedTwoBackupsSwap() throws Exception { + memMode = GridTestUtils.TestMemoryMode.SWAP; + + checkPutAllFailoverColocated(false, 5, 2); + } + + /** + * @throws Exception If failed. + */ + public void testPutAllFailoverColocatedNearEnabledTwoBackupsOffheapTiered() throws Exception { + memMode = GridTestUtils.TestMemoryMode.OFFHEAP_TIERED; + + checkPutAllFailoverColocated(true, 5, 2); + } + + /** + * @throws Exception If failed. + */ + public void testPutAllFailoverColocatedNearEnabledTwoBackupsOffheapTieredSwap() throws Exception { + memMode = GridTestUtils.TestMemoryMode.OFFHEAP_TIERED_SWAP; + + checkPutAllFailoverColocated(true, 5, 2); + } + + /** + * @throws Exception If failed. + */ + public void testPutAllFailoverColocatedNearEnabledTwoBackupsOffheapEvict() throws Exception { + memMode = GridTestUtils.TestMemoryMode.OFFHEAP_EVICT; + + checkPutAllFailoverColocated(true, 5, 2); + } + + /** + * @throws Exception If failed. + */ + public void testPutAllFailoverColocatedNearEnabledTwoBackupsOffheapEvictSwap() throws Exception { + memMode = GridTestUtils.TestMemoryMode.OFFHEAP_EVICT_SWAP; + + checkPutAllFailoverColocated(true, 5, 2); + } + /** {@inheritDoc} */ @Override protected long getTestTimeout() { return super.getTestTimeout() * 5; @@ -670,6 +728,8 @@ public class GridCachePutAllFailoverSelfTest extends GridCommonAbstractTest { cacheCfg.setWriteSynchronizationMode(FULL_SYNC); + GridTestUtils.setMemoryMode(cfg, cacheCfg, memMode, 1000, 10 * 1024); + cfg.setCacheConfiguration(cacheCfg); } else http://git-wip-us.apache.org/repos/asf/ignite/blob/c05bf538/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java index b585a8f..66aab7b 100644 --- a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java @@ -59,7 +59,11 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.Ignition; +import org.apache.ignite.cache.CacheMemoryMode; +import org.apache.ignite.cache.eviction.lru.LruEvictionPolicy; import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.IgniteInterruptedCheckedException; @@ -83,6 +87,7 @@ import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.LT; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgnitePredicate; +import org.apache.ignite.spi.swapspace.file.FileSwapSpaceSpi; import org.apache.ignite.ssl.SslContextFactory; import org.apache.ignite.testframework.config.GridTestProperties; import org.jetbrains.annotations.NotNull; @@ -1582,4 +1587,117 @@ public final class GridTestUtils { suite.addTestSuite(test); } + + /** + * Sets cache configuration parameters according to test memory mode. + * + * @param cfg Ignite configuration. + * @param ccfg Cache configuration. + * @param testMode Test memory mode. + * @param maxHeapCnt Maximum number of entries in heap (used if test mode involves eviction from heap). + * @param maxOffheapSize Maximum offheap memory size (used if test mode involves eviction from offheap to swap). + */ + public static void setMemoryMode(IgniteConfiguration cfg, CacheConfiguration ccfg, + TestMemoryMode testMode, + int maxHeapCnt, + long maxOffheapSize) { + assert testMode != null; + assert cfg != null; + assert ccfg != null; + + CacheMemoryMode memMode; + boolean swap = false; + boolean evictionPlc = false; + long offheapMaxMem = -1L; + + switch (testMode) { + case HEAP: { + memMode = CacheMemoryMode.ONHEAP_TIERED; + swap = false; + + break; + } + + case SWAP: { + memMode = CacheMemoryMode.ONHEAP_TIERED; + evictionPlc = true; + swap = true; + + break; + } + + case OFFHEAP_TIERED: { + memMode = CacheMemoryMode.OFFHEAP_TIERED; + offheapMaxMem = 0; + + break; + } + + case OFFHEAP_TIERED_SWAP: { + assert maxOffheapSize > 0 : maxOffheapSize; + + memMode = CacheMemoryMode.OFFHEAP_TIERED; + offheapMaxMem = maxOffheapSize; + swap = true; + + break; + } + + case OFFHEAP_EVICT: { + memMode = CacheMemoryMode.ONHEAP_TIERED; + evictionPlc = true; + offheapMaxMem = 0; + + break; + } + + case OFFHEAP_EVICT_SWAP: { + assert maxOffheapSize > 0 : maxOffheapSize; + + memMode = CacheMemoryMode.ONHEAP_TIERED; + swap = true; + evictionPlc = true; + offheapMaxMem = maxOffheapSize; + + break; + } + + default: + throw new IllegalArgumentException("Invalid mode: " + testMode); + } + + ccfg.setMemoryMode(memMode); + ccfg.setSwapEnabled(swap); + + if (swap) + cfg.setSwapSpaceSpi(new FileSwapSpaceSpi()); + + if (evictionPlc) { + LruEvictionPolicy plc = new LruEvictionPolicy(); + + plc.setMaxSize(maxHeapCnt); + + ccfg.setEvictionPolicy(plc); + } + + ccfg.setOffHeapMaxMemory(offheapMaxMem); + } + + /** + * + */ + public enum TestMemoryMode { + /** Heap only. */ + HEAP, + /** Evict from heap to swap with eviction policy. */ + SWAP, + /** Always evict to offheap, no swap. */ + OFFHEAP_TIERED, + /** Always evict to offheap + evict from offheap to swap when max offheap memory limit is reached. */ + OFFHEAP_TIERED_SWAP, + /** Evict to offheap with eviction policy, no swap. */ + OFFHEAP_EVICT, + /** Evict to offheap with eviction policy + evict from offheap to swap when max offheap memory limit is reached. */ + OFFHEAP_EVICT_SWAP, + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/c05bf538/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheClientQueryReplicatedNodeRestartSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheClientQueryReplicatedNodeRestartSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheClientQueryReplicatedNodeRestartSelfTest.java index 7e72292..5dbb12c 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheClientQueryReplicatedNodeRestartSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheClientQueryReplicatedNodeRestartSelfTest.java @@ -315,7 +315,7 @@ public class IgniteCacheClientQueryReplicatedNodeRestartSelfTest extends GridCom info("Executed queries: " + c); } } - }, qryThreadNum); + }, qryThreadNum, "query-thread"); final AtomicInteger restartCnt = new AtomicInteger(); @@ -334,10 +334,14 @@ public class IgniteCacheClientQueryReplicatedNodeRestartSelfTest extends GridCom } while (!locks.compareAndSet(g, 0, -1)); + log.info("Stop node: " + g); + stopGrid(g); Thread.sleep(rnd.nextInt(nodeLifeTime)); + log.info("Start node: " + g); + startGrid(g); Thread.sleep(rnd.nextInt(nodeLifeTime)); @@ -352,7 +356,7 @@ public class IgniteCacheClientQueryReplicatedNodeRestartSelfTest extends GridCom return true; } - }, restartThreadsNum); + }, restartThreadsNum, "restart-thread"); Thread.sleep(duration); http://git-wip-us.apache.org/repos/asf/ignite/blob/c05bf538/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartSelfTest.java index 32da55a..45a249d 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartSelfTest.java @@ -149,7 +149,7 @@ public class IgniteCacheQueryNodeRestartSelfTest extends GridCacheAbstractSelfTe info("Executed queries: " + c); } } - }, qryThreadNum); + }, qryThreadNum, "query-thread"); final AtomicInteger restartCnt = new AtomicInteger(); @@ -178,7 +178,7 @@ public class IgniteCacheQueryNodeRestartSelfTest extends GridCacheAbstractSelfTe return true; } - }, 1); + }, 1, "restart-thread"); Thread.sleep(duration); http://git-wip-us.apache.org/repos/asf/ignite/blob/c05bf538/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartSelfTest2.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartSelfTest2.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartSelfTest2.java index 71f764b..e00611b 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartSelfTest2.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartSelfTest2.java @@ -275,7 +275,7 @@ public class IgniteCacheQueryNodeRestartSelfTest2 extends GridCommonAbstractTest info("Executed queries: " + c); } } - }, qryThreadNum); + }, qryThreadNum, "query-thread"); final AtomicInteger restartCnt = new AtomicInteger(); @@ -294,10 +294,14 @@ public class IgniteCacheQueryNodeRestartSelfTest2 extends GridCommonAbstractTest } while (!locks.compareAndSet(g, 0, -1)); + log.info("Stop node: " + g); + stopGrid(g); Thread.sleep(rnd.nextInt(nodeLifeTime)); + log.info("Start node: " + g); + startGrid(g); Thread.sleep(rnd.nextInt(nodeLifeTime)); @@ -312,7 +316,7 @@ public class IgniteCacheQueryNodeRestartSelfTest2 extends GridCommonAbstractTest return true; } - }, restartThreadsNum); + }, restartThreadsNum, "restart-thread"); Thread.sleep(duration);
