IGNITE-9658 Add an option to reuse memory after deactivation - Fixes #4874.
Signed-off-by: Alexey Goncharuk <alexey.goncha...@gmail.com> Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/2f9faf1e Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/2f9faf1e Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/2f9faf1e Branch: refs/heads/ignite-5797 Commit: 2f9faf1e0d885185348653d6e3b817d32b992261 Parents: 9aef609 Author: ascherbakoff <alexey.scherbak...@gmail.com> Authored: Mon Oct 1 14:54:41 2018 +0300 Committer: Alexey Goncharuk <alexey.goncha...@gmail.com> Committed: Mon Oct 1 16:05:27 2018 +0300 ---------------------------------------------------------------------- .../benchmarks/jmh/tree/BPlusTreeBenchmark.java | 2 +- .../apache/ignite/IgniteSystemProperties.java | 5 ++ .../internal/mem/DirectMemoryProvider.java | 6 +- .../mem/file/MappedFileMemoryProvider.java | 6 +- .../mem/unsafe/UnsafeMemoryProvider.java | 29 +++++-- .../ignite/internal/pagemem/PageMemory.java | 16 +++- .../pagemem/impl/PageMemoryNoStoreImpl.java | 4 +- .../GridCacheDatabaseSharedManager.java | 6 +- .../IgniteCacheDatabaseSharedManager.java | 81 ++++++++++++++++---- .../persistence/pagemem/PageMemoryImpl.java | 4 +- .../pagemem/impl/PageMemoryNoLoadSelfTest.java | 10 +-- .../IgniteClusterActivateDeactivateTest.java | 5 +- ...tivateTestWithPersistenceAndMemoryReuse.java | 40 ++++++++++ .../pagemem/FullPageIdTableTest.java | 4 +- ...gnitePageMemReplaceDelayedWriteUnitTest.java | 4 +- .../pagemem/PageIdDistributionTest.java | 4 +- .../pagemem/PageMemoryNoStoreLeakTest.java | 2 +- .../wal/memtracker/PageMemoryTracker.java | 2 +- .../processors/database/BPlusTreeSelfTest.java | 2 +- .../database/CacheFreeListImplSelfTest.java | 2 +- .../database/IndexStorageSelfTest.java | 2 +- .../ignite/testsuites/IgnitePdsTestSuite.java | 9 +-- .../ignite/testsuites/IgnitePdsTestSuite4.java | 3 + .../h2/database/InlineIndexHelperTest.java | 10 +-- 24 files changed, 191 insertions(+), 67 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/2f9faf1e/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/tree/BPlusTreeBenchmark.java ---------------------------------------------------------------------- diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/tree/BPlusTreeBenchmark.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/tree/BPlusTreeBenchmark.java index e80e13d..15c4710 100644 --- a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/tree/BPlusTreeBenchmark.java +++ b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/tree/BPlusTreeBenchmark.java @@ -137,7 +137,7 @@ public class BPlusTreeBenchmark extends JmhAbstractBenchmark { public void tearDown() throws Exception { tree.destroy(); - pageMem.stop(); + pageMem.stop(true); } /** http://git-wip-us.apache.org/repos/asf/ignite/blob/2f9faf1e/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java index 01fb02a..892689c 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java @@ -994,6 +994,11 @@ public final class IgniteSystemProperties { public static final String IGNITE_ZOOKEEPER_DISCOVERY_MAX_RETRY_COUNT = "IGNITE_ZOOKEEPER_DISCOVERY_MAX_RETRY_COUNT"; /** + * Try reuse memory on deactivation. Useful in case of huge page memory region size. + */ + public static final String IGNITE_REUSE_MEMORY_ON_DEACTIVATE = "IGNITE_REUSE_MEMORY_ON_DEACTIVATE"; + + /** * Enforces singleton. */ private IgniteSystemProperties() { http://git-wip-us.apache.org/repos/asf/ignite/blob/2f9faf1e/modules/core/src/main/java/org/apache/ignite/internal/mem/DirectMemoryProvider.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/mem/DirectMemoryProvider.java b/modules/core/src/main/java/org/apache/ignite/internal/mem/DirectMemoryProvider.java index a90c6b8..03d386b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/mem/DirectMemoryProvider.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/mem/DirectMemoryProvider.java @@ -27,9 +27,11 @@ public interface DirectMemoryProvider { public void initialize(long[] chunkSizes); /** - * Shuts down the provider. Will deallocate all previously allocated regions. + * Shuts down the provider. + * + * @param deallocate {@code True} to deallocate memory, {@code false} to allow memory reuse. */ - public void shutdown(); + public void shutdown(boolean deallocate); /** * Attempts to allocate next memory region. Will return {@code null} if no more regions are available. http://git-wip-us.apache.org/repos/asf/ignite/blob/2f9faf1e/modules/core/src/main/java/org/apache/ignite/internal/mem/file/MappedFileMemoryProvider.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/mem/file/MappedFileMemoryProvider.java b/modules/core/src/main/java/org/apache/ignite/internal/mem/file/MappedFileMemoryProvider.java index 54b4af4..67e86f5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/mem/file/MappedFileMemoryProvider.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/mem/file/MappedFileMemoryProvider.java @@ -30,7 +30,9 @@ import org.apache.ignite.internal.mem.DirectMemoryRegion; import org.apache.ignite.internal.util.typedef.internal.U; /** - * + * Memory provider implementation based on memory mapped file. + * <p> + * Doesn't support memory reuse semantics. */ public class MappedFileMemoryProvider implements DirectMemoryProvider { /** */ @@ -101,7 +103,7 @@ public class MappedFileMemoryProvider implements DirectMemoryProvider { } /** {@inheritDoc} */ - @Override public void shutdown() { + @Override public void shutdown(boolean deallocate) { if (mappedFiles != null) { for (MappedFile file : mappedFiles) { try { http://git-wip-us.apache.org/repos/asf/ignite/blob/2f9faf1e/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java b/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java index d964648..8cb8119 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java @@ -29,7 +29,9 @@ import org.apache.ignite.internal.util.GridUnsafe; import org.apache.ignite.internal.util.typedef.internal.U; /** - * + * Memory provider implementation based on unsafe memory access. + * <p> + * Supports memory reuse semantics. */ public class UnsafeMemoryProvider implements DirectMemoryProvider { /** */ @@ -44,6 +46,9 @@ public class UnsafeMemoryProvider implements DirectMemoryProvider { /** Flag shows if current memory provider have been already initialized. */ private boolean isInit; + /** */ + private int used = 0; + /** * @param log Ignite logger to use. */ @@ -54,7 +59,7 @@ public class UnsafeMemoryProvider implements DirectMemoryProvider { /** {@inheritDoc} */ @Override public void initialize(long[] sizes) { if (isInit) - throw new IgniteException("Second initialization does not allowed for current provider"); + return; this.sizes = sizes; @@ -64,24 +69,32 @@ public class UnsafeMemoryProvider implements DirectMemoryProvider { } /** {@inheritDoc} */ - @Override public void shutdown() { + @Override public void shutdown(boolean deallocate) { if (regions != null) { for (Iterator<DirectMemoryRegion> it = regions.iterator(); it.hasNext(); ) { DirectMemoryRegion chunk = it.next(); - GridUnsafe.freeMemory(chunk.address()); + if (deallocate) { + GridUnsafe.freeMemory(chunk.address()); - // Safety. - it.remove(); + // Safety. + it.remove(); + } } + + if (!deallocate) + used = 0; } } /** {@inheritDoc} */ @Override public DirectMemoryRegion nextRegion() { - if (regions.size() == sizes.length) + if (used == sizes.length) return null; + if (used < regions.size()) + return regions.get(used++); + long chunkSize = sizes[regions.size()]; long ptr; @@ -111,6 +124,8 @@ public class UnsafeMemoryProvider implements DirectMemoryProvider { regions.add(region); + used++; + return region; } } http://git-wip-us.apache.org/repos/asf/ignite/blob/2f9faf1e/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageMemory.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageMemory.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageMemory.java index 6f2e2c9..f7391d2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageMemory.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageMemory.java @@ -18,11 +18,25 @@ package org.apache.ignite.internal.pagemem; import java.nio.ByteBuffer; +import org.apache.ignite.IgniteException; import org.apache.ignite.lifecycle.LifecycleAware; /** */ -public interface PageMemory extends LifecycleAware, PageIdAllocator, PageSupport { +public interface PageMemory extends PageIdAllocator, PageSupport { + /** + * Start page memory. + */ + public void start() throws IgniteException; + + /** + * Stop page memory. + * + * @param deallocate {@code True} to deallocate memory, {@code false} to allow memory reuse on subsequent {@link #start()} + * @throws IgniteException + */ + public void stop(boolean deallocate) throws IgniteException; + /** * @return Page size in bytes. */ http://git-wip-us.apache.org/repos/asf/ignite/blob/2f9faf1e/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java index c5eba60..02afac8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java @@ -239,11 +239,11 @@ public class PageMemoryNoStoreImpl implements PageMemory { /** {@inheritDoc} */ @SuppressWarnings("OverlyStrongTypeCast") - @Override public void stop() throws IgniteException { + @Override public void stop(boolean deallocate) throws IgniteException { if (log.isDebugEnabled()) log.debug("Stopping page memory."); - directMemoryProvider.shutdown(); + directMemoryProvider.shutdown(deallocate); if (directMemoryProvider instanceof Closeable) { try { http://git-wip-us.apache.org/repos/asf/ignite/blob/2f9faf1e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index 5e0b7cb..aff2b42 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -655,7 +655,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan metaStorage = null; - storePageMem.stop(); + storePageMem.stop(true); } catch (StorageException e) { cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e)); @@ -1122,8 +1122,8 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan checkPointBufferIdxCnt.set(chunkSizes.length); } - @Override public void shutdown() { - memProvider.shutdown(); + @Override public void shutdown(boolean deallocate) { + memProvider.shutdown(deallocate); } @Override public DirectMemoryRegion nextRegion() { http://git-wip-us.apache.org/repos/asf/ignite/blob/2f9faf1e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index 52430c0..f35d15a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -30,6 +30,7 @@ import org.apache.ignite.DataRegionMetrics; import org.apache.ignite.DataStorageMetrics; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteLogger; +import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.configuration.DataPageEvictionMode; import org.apache.ignite.configuration.DataRegionConfiguration; import org.apache.ignite.configuration.DataStorageConfiguration; @@ -69,6 +70,7 @@ import org.apache.ignite.lang.IgniteOutClosure; import org.apache.ignite.mxbean.DataRegionMetricsMXBean; import org.jetbrains.annotations.Nullable; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_REUSE_MEMORY_ON_DEACTIVATE; import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_DATA_REG_DEFAULT_NAME; import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_PAGE_SIZE; import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_WAL_ARCHIVE_MAX_SIZE; @@ -88,6 +90,9 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap /** Maximum initial size on 32-bit JVM */ private static final long MAX_PAGE_MEMORY_INIT_SIZE_32_BIT = 2L * 1024 * 1024 * 1024; + /** {@code True} to reuse memory on deactive. */ + private final boolean reuseMemory = IgniteSystemProperties.getBoolean(IGNITE_REUSE_MEMORY_ON_DEACTIVATE); + /** */ protected volatile Map<String, DataRegion> dataRegionMap; @@ -112,6 +117,9 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap /** First eviction was warned flag. */ private volatile boolean firstEvictWarn; + /** Stores memory providers eligible for reuse. */ + private Map<String, DirectMemoryProvider> memProviderMap; + /** {@inheritDoc} */ @Override protected void start0() throws IgniteCheckedException { if (cctx.kernalContext().clientNode() && cctx.kernalContext().config().getDataStorageConfiguration() == null) @@ -243,6 +251,7 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap dataRegionMap = U.newHashMap(3 + dataRegions); memMetricsMap = U.newHashMap(3 + dataRegions); + memProviderMap = reuseMemory ? U.newHashMap(3 + dataRegions) : null; if (dataRegionCfgs != null) { for (DataRegionConfiguration dataRegionCfg : dataRegionCfgs) @@ -265,9 +274,8 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap CU.isPersistenceEnabled(memCfg) ); - for (DatabaseLifecycleListener lsnr : getDatabaseListeners(cctx.kernalContext())) { + for (DatabaseLifecycleListener lsnr : getDatabaseListeners(cctx.kernalContext())) lsnr.onInitDataRegions(this); - } } /** @@ -706,7 +714,7 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap /** {@inheritDoc} */ @Override protected void stop0(boolean cancel) { - onDeActivate(cctx.kernalContext()); + onDeActivate(true); } /** @@ -935,17 +943,49 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap DataRegionMetricsImpl memMetrics, boolean trackable ) throws IgniteCheckedException { + PageMemory pageMem = createPageMemory(createOrReuseMemoryProvider(plcCfg), memCfg, plcCfg, memMetrics, trackable); + + return new DataRegion(pageMem, plcCfg, memMetrics, createPageEvictionTracker(plcCfg, pageMem)); + } + + /** + * @param plcCfg Policy config. + * @return DirectMemoryProvider provider. + */ + private DirectMemoryProvider createOrReuseMemoryProvider(DataRegionConfiguration plcCfg) + throws IgniteCheckedException { + if (!supportsMemoryReuse(plcCfg)) + return createMemoryProvider(plcCfg); + + DirectMemoryProvider memProvider = memProviderMap.get(plcCfg.getName()); + + if (memProvider == null) + memProviderMap.put(plcCfg.getName(), (memProvider = createMemoryProvider(plcCfg))); + + return memProvider; + } + + /** + * @param plcCfg Policy config. + * + * @return {@code True} if policy supports memory reuse. + */ + private boolean supportsMemoryReuse(DataRegionConfiguration plcCfg) { + return reuseMemory && plcCfg.getSwapPath() == null; + } + + /** + * @param plcCfg Policy config. + * @return DirectMemoryProvider provider. + */ + private DirectMemoryProvider createMemoryProvider(DataRegionConfiguration plcCfg) throws IgniteCheckedException { File allocPath = buildAllocPath(plcCfg); - DirectMemoryProvider memProvider = allocPath == null ? + return allocPath == null ? new UnsafeMemoryProvider(log) : new MappedFileMemoryProvider( log, allocPath); - - PageMemory pageMem = createPageMemory(memProvider, memCfg, plcCfg, memMetrics, trackable); - - return new DataRegion(pageMem, plcCfg, memMetrics, createPageEvictionTracker(plcCfg, pageMem)); } /** @@ -1045,8 +1085,8 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap memProvider.initialize(chunkSizes); } - @Override public void shutdown() { - memProvider.shutdown(); + @Override public void shutdown(boolean deallocate) { + memProvider.shutdown(deallocate); } @Override public DirectMemoryRegion nextRegion() { @@ -1095,20 +1135,25 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap initPageMemoryDataStructures(memCfg); - for (DatabaseLifecycleListener lsnr : getDatabaseListeners(kctx)) { + for (DatabaseLifecycleListener lsnr : getDatabaseListeners(kctx)) lsnr.afterInitialise(this); - } } /** {@inheritDoc} */ @Override public void onDeActivate(GridKernalContext kctx) { - for (DatabaseLifecycleListener lsnr : getDatabaseListeners(cctx.kernalContext())) { + onDeActivate(!reuseMemory); + } + + /** + * @param shutdown Shutdown. + */ + private void onDeActivate(boolean shutdown) { + for (DatabaseLifecycleListener lsnr : getDatabaseListeners(cctx.kernalContext())) lsnr.beforeStop(this); - } if (dataRegionMap != null) { for (DataRegion memPlc : dataRegionMap.values()) { - memPlc.pageMemory().stop(); + memPlc.pageMemory().stop(shutdown); memPlc.evictionTracker().stop(); @@ -1119,6 +1164,12 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap dataRegionMap = null; + if (shutdown && memProviderMap != null) { + memProviderMap.clear(); + + memProviderMap = null; + } + dataRegionsInitialized = false; } } http://git-wip-us.apache.org/repos/asf/ignite/blob/2f9faf1e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java index ba565c9..e49b7e2 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java @@ -387,7 +387,7 @@ public class PageMemoryImpl implements PageMemoryEx { /** {@inheritDoc} */ @SuppressWarnings("OverlyStrongTypeCast") - @Override public void stop() throws IgniteException { + @Override public void stop(boolean deallocate) throws IgniteException { if (log.isDebugEnabled()) log.debug("Stopping page memory."); @@ -398,7 +398,7 @@ public class PageMemoryImpl implements PageMemoryEx { seg.close(); } - directMemoryProvider.shutdown(); + directMemoryProvider.shutdown(deallocate); } /** {@inheritDoc} */ http://git-wip-us.apache.org/repos/asf/ignite/blob/2f9faf1e/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java index 6ad977f..c5988e3 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java @@ -96,7 +96,7 @@ public class PageMemoryNoLoadSelfTest extends GridCommonAbstractTest { } } finally { - mem.stop(); + mem.stop(true); } } @@ -121,7 +121,7 @@ public class PageMemoryNoLoadSelfTest extends GridCommonAbstractTest { assertEquals(mem.loadedPages(), expPages); } finally { - mem.stop(); + mem.stop(true); } } @@ -173,7 +173,7 @@ public class PageMemoryNoLoadSelfTest extends GridCommonAbstractTest { } } finally { - mem.stop(); + mem.stop(true); } } @@ -200,7 +200,7 @@ public class PageMemoryNoLoadSelfTest extends GridCommonAbstractTest { assertFalse(handles.add(allocatePage(mem))); } finally { - mem.stop(); + mem.stop(true); } } @@ -304,7 +304,7 @@ public class PageMemoryNoLoadSelfTest extends GridCommonAbstractTest { } } finally { - mem.stop(); + mem.stop(true); } } http://git-wip-us.apache.org/repos/asf/ignite/blob/2f9faf1e/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java index 6ed0ef7..f6b4ddd 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java @@ -120,11 +120,11 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest DataStorageConfiguration memCfg = new DataStorageConfiguration(); memCfg.setPageSize(4 * 1024); memCfg.setDefaultDataRegionConfiguration(new DataRegionConfiguration() - .setMaxSize(300L * 1024 * 1024) + .setMaxSize(150L * 1024 * 1024) .setPersistenceEnabled(persistenceEnabled())); memCfg.setDataRegionConfigurations(new DataRegionConfiguration() - .setMaxSize(300L * 1024 * 1024) + .setMaxSize(150L * 1024 * 1024) .setName(NO_PERSISTENCE_REGION) .setPersistenceEnabled(false)); @@ -132,6 +132,7 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest memCfg.setWalMode(WALMode.LOG_ONLY); cfg.setDataStorageConfiguration(memCfg); + cfg.setFailureDetectionTimeout(60_000); if (testSpi) { TestRecordingCommunicationSpi spi = new TestRecordingCommunicationSpi(); http://git-wip-us.apache.org/repos/asf/ignite/blob/2f9faf1e/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistenceAndMemoryReuse.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistenceAndMemoryReuse.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistenceAndMemoryReuse.java new file mode 100644 index 0000000..48d1754 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistenceAndMemoryReuse.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import org.apache.ignite.IgniteSystemProperties; + +/** + * + */ +public class IgniteClusterActivateDeactivateTestWithPersistenceAndMemoryReuse extends + IgniteClusterActivateDeactivateTestWithPersistence { + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + System.setProperty(IgniteSystemProperties.IGNITE_REUSE_MEMORY_ON_DEACTIVATE, "true"); + + super.beforeTest(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + System.clearProperty(IgniteSystemProperties.IGNITE_REUSE_MEMORY_ON_DEACTIVATE); + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/2f9faf1e/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTableTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTableTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTableTest.java index e337bb1..fd23ce5 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTableTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTableTest.java @@ -108,7 +108,7 @@ public class FullPageIdTableTest { } } finally { - prov.shutdown(); + prov.shutdown(true); } } @@ -225,7 +225,7 @@ public class FullPageIdTableTest { finally { long msPassed = U.currentTimeMillis() - seed; System.err.println("Seed used [" + seed + "] duration ["+ msPassed+ "] ms"); - prov.shutdown(); + prov.shutdown(true); } } http://git-wip-us.apache.org/repos/asf/ignite/blob/2f9faf1e/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgnitePageMemReplaceDelayedWriteUnitTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgnitePageMemReplaceDelayedWriteUnitTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgnitePageMemReplaceDelayedWriteUnitTest.java index aa1e37d..5c965b1 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgnitePageMemReplaceDelayedWriteUnitTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgnitePageMemReplaceDelayedWriteUnitTest.java @@ -127,7 +127,7 @@ public class IgnitePageMemReplaceDelayedWriteUnitTest { assert totalEvicted.get() > 0; - memory.stop(); + memory.stop(true); } /** @@ -182,7 +182,7 @@ public class IgnitePageMemReplaceDelayedWriteUnitTest { assert totalEvicted.get() > 0; - memory.stop(); + memory.stop(true); } /** http://git-wip-us.apache.org/repos/asf/ignite/blob/2f9faf1e/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageIdDistributionTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageIdDistributionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageIdDistributionTest.java index 626e794..2008b22 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageIdDistributionTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageIdDistributionTest.java @@ -29,8 +29,6 @@ import org.apache.ignite.internal.mem.DirectMemoryRegion; import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider; import org.apache.ignite.internal.pagemem.FullPageId; import org.apache.ignite.internal.pagemem.PageIdUtils; -import org.apache.ignite.internal.processors.cache.persistence.pagemem.FullPageIdTable; -import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl; import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.U; @@ -228,7 +226,7 @@ public class PageIdDistributionTest extends GridCommonAbstractTest { } } finally { - prov.shutdown(); + prov.shutdown(true); } } } http://git-wip-us.apache.org/repos/asf/ignite/blob/2f9faf1e/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryNoStoreLeakTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryNoStoreLeakTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryNoStoreLeakTest.java index 65e8c36..8505a35 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryNoStoreLeakTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryNoStoreLeakTest.java @@ -71,7 +71,7 @@ public class PageMemoryNoStoreLeakTest extends GridCommonAbstractTest { mem.start(); } finally { - mem.stop(); + mem.stop(true); } long committedVMSize = D.getCommittedVirtualMemorySize(); http://git-wip-us.apache.org/repos/asf/ignite/blob/2f9faf1e/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/memtracker/PageMemoryTracker.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/memtracker/PageMemoryTracker.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/memtracker/PageMemoryTracker.java index 6649993..661b6c7 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/memtracker/PageMemoryTracker.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/memtracker/PageMemoryTracker.java @@ -283,7 +283,7 @@ public class PageMemoryTracker implements IgnitePlugin { stats.clear(); - memoryProvider.shutdown(); + memoryProvider.shutdown(true); if (checkpointLsnr != null) { ((GridCacheDatabaseSharedManager)gridCtx.cache().context().database()) http://git-wip-us.apache.org/repos/asf/ignite/blob/2f9faf1e/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java index 487cdbe..6b694c9 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java @@ -204,7 +204,7 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest { } finally { if (pageMem != null) - pageMem.stop(); + pageMem.stop(true); MAX_PER_PAGE = 0; PUT_INC = 1; http://git-wip-us.apache.org/repos/asf/ignite/blob/2f9faf1e/modules/core/src/test/java/org/apache/ignite/internal/processors/database/CacheFreeListImplSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/CacheFreeListImplSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/CacheFreeListImplSelfTest.java index 7422815..d9804bf 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/CacheFreeListImplSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/CacheFreeListImplSelfTest.java @@ -71,7 +71,7 @@ public class CacheFreeListImplSelfTest extends GridCommonAbstractTest { super.afterTest(); if (pageMem != null) - pageMem.stop(); + pageMem.stop(true); pageMem = null; } http://git-wip-us.apache.org/repos/asf/ignite/blob/2f9faf1e/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IndexStorageSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IndexStorageSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IndexStorageSelfTest.java index 69a86b4..bbdcd38 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IndexStorageSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IndexStorageSelfTest.java @@ -129,7 +129,7 @@ public class IndexStorageSelfTest extends GridCommonAbstractTest { } } finally { - mem.stop(); + mem.stop(true); } } http://git-wip-us.apache.org/repos/asf/ignite/blob/2f9faf1e/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java index 2b57223..4b5327b 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java @@ -18,21 +18,17 @@ package org.apache.ignite.testsuites; import junit.framework.TestSuite; -import org.apache.ignite.internal.pagemem.impl.PageMemoryNoLoadSelfTest; import org.apache.ignite.internal.processors.cache.IgniteClusterActivateDeactivateTestWithPersistence; +import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCacheConfigurationFileConsistencyCheckTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCacheObjectBinaryProcessorOnDiscoveryTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDestroyCacheTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDestroyCacheWithoutCheckpointsTest; -import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCacheConfigurationFileConsistencyCheckTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDynamicCacheTest; -import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsRemoveDuringRebalancingTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsSingleNodePutGetPersistenceTest; -import org.apache.ignite.internal.processors.cache.persistence.IgnitePersistenceSequentialCheckpointTest; import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsCacheRestoreTest; import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsDataRegionMetricsTest; import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsWithTtlTest; import org.apache.ignite.internal.processors.cache.persistence.db.file.DefaultPageSizeBackwardsCompatibilityTest; -import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsCheckpointSimpleTest; import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsCheckpointSimulationWithRealCpDisabledTest; import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsPageReplacementTest; import org.apache.ignite.internal.processors.cache.persistence.metastorage.IgniteMetaStorageBasicTest; @@ -40,17 +36,14 @@ import org.apache.ignite.internal.processors.cache.persistence.pagemem.BPlusTree import org.apache.ignite.internal.processors.cache.persistence.pagemem.BPlusTreeReuseListPageMemoryImplTest; import org.apache.ignite.internal.processors.cache.persistence.pagemem.FillFactorMetricTest; import org.apache.ignite.internal.processors.cache.persistence.pagemem.IndexStoragePageMemoryImplTest; -import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageIdDistributionTest; import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImplNoLoadTest; import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImplTest; import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryNoStoreLeakTest; import org.apache.ignite.internal.processors.cache.persistence.pagemem.PagesWriteThrottleSmokeTest; -import org.apache.ignite.internal.processors.cache.persistence.tree.io.TrackingPageIOTest; import org.apache.ignite.internal.processors.cache.persistence.wal.CpTriggeredWalDeltaConsistencyTest; import org.apache.ignite.internal.processors.cache.persistence.wal.ExplicitWalDeltaConsistencyTest; import org.apache.ignite.internal.processors.cache.persistence.wal.SegmentedRingByteBufferTest; import org.apache.ignite.internal.processors.cache.persistence.wal.SysPropWalDeltaConsistencyTest; -import org.apache.ignite.internal.processors.cache.persistence.wal.aware.SegmentAware; import org.apache.ignite.internal.processors.cache.persistence.wal.aware.SegmentAwareTest; import org.apache.ignite.internal.processors.database.IgniteDbDynamicCacheSelfTest; import org.apache.ignite.internal.processors.database.IgniteDbMultiNodePutGetTest; http://git-wip-us.apache.org/repos/asf/ignite/blob/2f9faf1e/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java index 2e6a439..c164635 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java @@ -18,6 +18,7 @@ package org.apache.ignite.testsuites; import junit.framework.TestSuite; +import org.apache.ignite.internal.processors.cache.IgniteClusterActivateDeactivateTestWithPersistenceAndMemoryReuse; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTestWithSharedGroupAndIndexes; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsRecoveryAfterFileCorruptionTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsTaskCancelingTest; @@ -41,6 +42,8 @@ public class IgnitePdsTestSuite4 extends TestSuite { suite.addTestSuite(IgnitePdsTaskCancelingTest.class); + suite.addTestSuite(IgniteClusterActivateDeactivateTestWithPersistenceAndMemoryReuse.class); + return suite; } http://git-wip-us.apache.org/repos/asf/ignite/blob/2f9faf1e/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelperTest.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelperTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelperTest.java index 4c64264..fe77da3 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelperTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelperTest.java @@ -218,7 +218,7 @@ public class InlineIndexHelperTest extends GridCommonAbstractTest { if (page != 0L) pageMem.releasePage(CACHE_ID, pageId, page); - pageMem.stop(); + pageMem.stop(true); } } @@ -345,7 +345,7 @@ public class InlineIndexHelperTest extends GridCommonAbstractTest { finally { if (page != 0L) pageMem.releasePage(CACHE_ID, pageId, page); - pageMem.stop(); + pageMem.stop(true); } } @@ -403,7 +403,7 @@ public class InlineIndexHelperTest extends GridCommonAbstractTest { finally { if (page != 0L) pageMem.releasePage(CACHE_ID, pageId, page); - pageMem.stop(); + pageMem.stop(true); } } @@ -461,7 +461,7 @@ public class InlineIndexHelperTest extends GridCommonAbstractTest { finally { if (page != 0L) pageMem.releasePage(CACHE_ID, pageId, page); - pageMem.stop(); + pageMem.stop(true); } } @@ -582,7 +582,7 @@ public class InlineIndexHelperTest extends GridCommonAbstractTest { if (page != 0L) pageMem.releasePage(CACHE_ID, pageId, page); - pageMem.stop(); + pageMem.stop(true); } }