Repository: geode Updated Branches: refs/heads/develop 1aec1f835 -> e606f3e6e
GEODE-1672: Disabled recovering values for LRU region during startup. When recovering persistent files, system stores the values in temp maps (for regions) using a background thread, as these maps are not actual regions, the temp-regions are not considered/included for LRU eviction, which causes the system to run OOM. The problem is fixed by skipping recovering values for LRU regions. A new system property ""disk.recoverLruValues" is added to support reading values for lru regions. Project: http://git-wip-us.apache.org/repos/asf/geode/repo Commit: http://git-wip-us.apache.org/repos/asf/geode/commit/e606f3e6 Tree: http://git-wip-us.apache.org/repos/asf/geode/tree/e606f3e6 Diff: http://git-wip-us.apache.org/repos/asf/geode/diff/e606f3e6 Branch: refs/heads/develop Commit: e606f3e6ec0828f5fc30e20a9dbdf3aa8c3c8620 Parents: 1aec1f8 Author: Anil <aging...@pivotal.io> Authored: Wed Feb 1 11:23:25 2017 -0800 Committer: Anil <aging...@pivotal.io> Committed: Wed Feb 1 11:23:25 2017 -0800 ---------------------------------------------------------------------- .../geode/internal/cache/DiskStoreImpl.java | 10 + .../org/apache/geode/internal/cache/Oplog.java | 40 ++- .../cache/DiskLruRegRecoveryJUnitTest.java | 290 +++++++++++++++++++ .../cache/DiskRegRecoveryJUnitTest.java | 8 + .../internal/cache/DiskRegionHelperFactory.java | 26 ++ .../internal/cache/DiskRegionProperties.java | 12 +- .../PersistentPartitionedRegionJUnitTest.java | 137 +++++++++ 7 files changed, 508 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/geode/blob/e606f3e6/geode-core/src/main/java/org/apache/geode/internal/cache/DiskStoreImpl.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/DiskStoreImpl.java b/geode-core/src/main/java/org/apache/geode/internal/cache/DiskStoreImpl.java index 5affdb5..e53aa5d 100644 --- a/geode-core/src/main/java/org/apache/geode/internal/cache/DiskStoreImpl.java +++ b/geode-core/src/main/java/org/apache/geode/internal/cache/DiskStoreImpl.java @@ -124,10 +124,20 @@ public class DiskStoreImpl implements DiskStore { DistributionConfig.GEMFIRE_PREFIX + "disk.recoverValues"; public static final String RECOVER_VALUES_SYNC_PROPERTY_NAME = DistributionConfig.GEMFIRE_PREFIX + "disk.recoverValuesSync"; + + /** + * Allows recovering values for LRU regions. By default values are not recovered for LRU regions + * during recovery. + */ + public static final String RECOVER_LRU_VALUES_PROPERTY_NAME = + DistributionConfig.GEMFIRE_PREFIX + "disk.recoverLruValues"; + boolean RECOVER_VALUES = getBoolean(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME, true); boolean RECOVER_VALUES_SYNC = getBoolean(DiskStoreImpl.RECOVER_VALUES_SYNC_PROPERTY_NAME, false); boolean FORCE_KRF_RECOVERY = getBoolean(DistributionConfig.GEMFIRE_PREFIX + "disk.FORCE_KRF_RECOVERY", false); + final boolean RECOVER_LRU_VALUES = + getBoolean(DiskStoreImpl.RECOVER_LRU_VALUES_PROPERTY_NAME, false); public static boolean getBoolean(String sysProp, boolean def) { return Boolean.valueOf(System.getProperty(sysProp, Boolean.valueOf(def).toString())); http://git-wip-us.apache.org/repos/asf/geode/blob/e606f3e6/geode-core/src/main/java/org/apache/geode/internal/cache/Oplog.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/Oplog.java b/geode-core/src/main/java/org/apache/geode/internal/cache/Oplog.java index 32d6e30..0b98364 100755 --- a/geode-core/src/main/java/org/apache/geode/internal/cache/Oplog.java +++ b/geode-core/src/main/java/org/apache/geode/internal/cache/Oplog.java @@ -47,11 +47,13 @@ import org.apache.geode.internal.util.BlobHelper; import org.apache.geode.internal.util.IOUtils; import org.apache.geode.internal.util.TransformUtils; import org.apache.geode.pdx.internal.PdxWriterImpl; + import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; import it.unimi.dsi.fastutil.longs.Long2ObjectMap; import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; import it.unimi.dsi.fastutil.objects.ObjectIterator; + import org.apache.logging.log4j.Logger; import java.io.*; @@ -2421,6 +2423,24 @@ public final class Oplog implements CompactableOplog, Flushable { } /** + * Returns true if the values for the given disk recovery store should be recovered. + */ + private boolean recoverLruValue(DiskRecoveryStore drs) { + if (isLruValueRecoveryDisabled(drs)) { + return false; + } else if (drs.lruLimitExceeded()) { + this.stats.incRecoveredValuesSkippedDueToLRU(); + return false; + } + return true; + } + + private boolean isLruValueRecoveryDisabled(DiskRecoveryStore store) { + return !store.getDiskStore().isOffline() && !getParent().RECOVER_LRU_VALUES + && !store.getEvictionAttributes().getAlgorithm().isNone(); + } + + /** * Reads an oplog entry of type Create * * @param dis DataInputStream from which the oplog is being read @@ -2465,10 +2485,10 @@ public final class Oplog implements CompactableOplog, Flushable { this.stats.incRecoveryRecordsSkipped(); incSkipped(); } - } else if (recoverValue && drs.lruLimitExceeded() && !getParent().isOfflineCompacting()) { - this.stats.incRecoveredValuesSkippedDueToLRU(); - recoverValue = false; + } else if (recoverValue && !getParent().isOfflineCompacting()) { + recoverValue = recoverLruValue(drs); } + CompactionRecord p2cr = null; long crOffset; if (EntryBits.isAnyInvalid(userBits) || EntryBits.isTombstone(userBits)) { @@ -2659,9 +2679,8 @@ public final class Oplog implements CompactableOplog, Flushable { incSkipped(); this.stats.incRecoveryRecordsSkipped(); } - } else if (recoverValue && drs.lruLimitExceeded() && !getParent().isOfflineCompacting()) { - this.stats.incRecoveredValuesSkippedDueToLRU(); - recoverValue = false; + } else if (recoverValue && !getParent().isOfflineCompacting()) { + recoverValue = recoverLruValue(drs); } byte[] objValue = null; @@ -2884,9 +2903,8 @@ public final class Oplog implements CompactableOplog, Flushable { incSkipped(); this.stats.incRecoveryRecordsSkipped(); } - } else if (recoverValue && drs.lruLimitExceeded() && !getParent().isOfflineCompacting()) { - this.stats.incRecoveredValuesSkippedDueToLRU(); - recoverValue = false; + } else if (recoverValue && !getParent().isOfflineCompacting()) { + recoverValue = recoverLruValue(drs); } byte[] objValue = null; @@ -6176,15 +6194,13 @@ public final class Oplog implements CompactableOplog, Flushable { HashMap<Long, DiskRegionInfo> targetRegions = new HashMap<Long, DiskRegionInfo>(this.regionMap); synchronized (diskRecoveryStores) { - // Don't bother to include any stores that have reached the lru limit Iterator<DiskRecoveryStore> itr = diskRecoveryStores.values().iterator(); while (itr.hasNext()) { DiskRecoveryStore store = itr.next(); - if (store.lruLimitExceeded()) { + if (isLruValueRecoveryDisabled(store) || store.lruLimitExceeded()) { itr.remove(); } } - // Get the a sorted list of live entries from the target regions targetRegions.keySet().retainAll(diskRecoveryStores.keySet()); } http://git-wip-us.apache.org/repos/asf/geode/blob/e606f3e6/geode-core/src/test/java/org/apache/geode/internal/cache/DiskLruRegRecoveryJUnitTest.java ---------------------------------------------------------------------- diff --git a/geode-core/src/test/java/org/apache/geode/internal/cache/DiskLruRegRecoveryJUnitTest.java b/geode-core/src/test/java/org/apache/geode/internal/cache/DiskLruRegRecoveryJUnitTest.java new file mode 100644 index 0000000..0bf3b38 --- /dev/null +++ b/geode-core/src/test/java/org/apache/geode/internal/cache/DiskLruRegRecoveryJUnitTest.java @@ -0,0 +1,290 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.geode.internal.cache; + +import static org.junit.Assert.*; + +import java.util.concurrent.CountDownLatch; + +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.apache.geode.cache.EntryNotFoundException; +import org.apache.geode.cache.Region; +import org.apache.geode.cache.Scope; +import org.apache.geode.test.junit.categories.IntegrationTest; + +/** + * Disk LRU region recovery tests. + */ +@Category(IntegrationTest.class) +public class DiskLruRegRecoveryJUnitTest extends DiskRegionTestingBase { + + @Override + protected final void postTearDown() throws Exception { + DiskStoreObserver.setInstance(null); + } + + private int getValuesInVM(Region region, int size) { + int valuesInVm = 0; + for (int i = 0; i < size; i++) { + try { + Object value = ((LocalRegion) region).getValueInVM(new Integer(i)); + if (value != null) { + valuesInVm++; + } + } catch (EntryNotFoundException e) { + fail("Entry not found not expected but occured "); + } + } + return valuesInVm; + } + + private Region createNonLruRegion() { + DiskRegionProperties nonLruDiskProps = new DiskRegionProperties(); + nonLruDiskProps.setDiskDirs(dirs); + nonLruDiskProps.setPersistBackup(true); + nonLruDiskProps.setRegionName("RecoveryTestNonLruRegion"); + return DiskRegionHelperFactory.getSyncPersistOnlyRegion(cache, nonLruDiskProps, Scope.LOCAL); + } + + private void createRegionAndIntiateRecovery(boolean lruEntryEviction, + boolean recoveryByCacheClose, boolean addNonLruRegion, int load, int expectedValues) { + + DiskRegionProperties diskProps = new DiskRegionProperties(); + Region nonLruRegion = null; + + diskProps.setDiskDirs(dirs); + diskProps.setPersistBackup(true); + diskProps.setRegionName("RecoveryTestRegion"); + + if (lruEntryEviction) { + int overflowCapacity = 5; + diskProps.setOverFlowCapacity(overflowCapacity); + region = DiskRegionHelperFactory.getSyncOverFlowAndPersistRegion(cache, diskProps); + } else { + region = DiskRegionHelperFactory.getSyncHeapLruAndPersistRegion(cache, diskProps); + } + + if (addNonLruRegion) { + nonLruRegion = createNonLruRegion(); + } + + for (int i = 0; i < load; i++) { + region.put(new Integer(i), new Integer(i)); + if (nonLruRegion != null) { + nonLruRegion.put(new Integer(i), new Integer(i)); + } + } + + if (recoveryByCacheClose) { + cache.close(); + cache = createCache(); + } else { + region.close(); + if (nonLruRegion != null) { + nonLruRegion.close(); + } + } + + final CountDownLatch recoveryDone = new CountDownLatch(1); + DiskStoreObserver.setInstance(new DiskStoreObserver() { + @Override + public void afterAsyncValueRecovery(DiskStoreImpl store) { + recoveryDone.countDown(); + } + }); + + if (lruEntryEviction) { + region = DiskRegionHelperFactory.getSyncOverFlowAndPersistRegion(cache, diskProps); + } else { + region = DiskRegionHelperFactory.getSyncHeapLruAndPersistRegion(cache, diskProps); + } + + if (addNonLruRegion) { + nonLruRegion = createNonLruRegion(); + } + + // Wait for recovery to finish. + try { + recoveryDone.await(); + } catch (InterruptedException ie) { + fail("Found interrupted exception while waiting for recovery."); + } + + int valuesInVm = getValuesInVM(region, load); + assertEquals("Values for lru regions should not be recovered from Disk.", expectedValues, valuesInVm); + + if (nonLruRegion != null) { + valuesInVm = getValuesInVM(nonLruRegion, load); + // The values should be recovered for non LRU regions. + assertEquals("Values for non lru regions are not recovered from Disk.", load, valuesInVm); + } + } + + @Test + public void testValuesAreNotRecoveredForLruRegionWithCacheClose() { + createRegionAndIntiateRecovery(true, true, false, 10, 0); + } + + @Test + public void testValuesAreNotRecoveredForLruRegionWithRegionClose() { + createRegionAndIntiateRecovery(true, false, false, 10, 0); + } + + @Test + public void testValuesAreNotRecoveredForLruAndRecoveredForNonLru() { + createRegionAndIntiateRecovery(true, true, true, 10, 0); + } + + @Test + public void testValuesAreNotRecoveredForLruAndRecoveredForNonLruWithRegionClose() { + createRegionAndIntiateRecovery(true, false, true, 10, 0); + } + + @Test + public void testValuesAreNotRecoveredForHeapLruRegion() { + createRegionAndIntiateRecovery(false, true, true, 10, 0); + } + + @Test + public void testValuesAreNotRecoveredForHeapLruRegionWithRegionClose() { + createRegionAndIntiateRecovery(false, false, true, 10, 0); + } + + @Test + public void testValuesAreNotRecoveredForLruRegionWithReocoverValuePropertySet() { + String oldValue = System.getProperty(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME); + System.setProperty(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME, "true"); + + try { + createRegionAndIntiateRecovery(true, true, true, 10, 0); + } finally { + if (oldValue != null) { + System.setProperty(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME, oldValue); + } else { + System.clearProperty(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME); + } + } + } + + @Test + public void testValuesAreNotRecoveredForLruRegionWithReocoverValuePropertySetWithRegionClose() { + String oldValue = System.getProperty(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME); + System.setProperty(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME, "true"); + + try { + createRegionAndIntiateRecovery(true, false, true, 10, 0); + } finally { + if (oldValue != null) { + System.setProperty(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME, oldValue); + } else { + System.clearProperty(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME); + } + } + } + + @Test + public void testValuesAreRecoveredForLruRegionWithReocoverValueAndRecoverLruPropertySet() { + String oldValue = System.getProperty(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME); + System.setProperty(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME, "true"); + + String lruOldValue = System.getProperty(DiskStoreImpl.RECOVER_LRU_VALUES_PROPERTY_NAME); + System.setProperty(DiskStoreImpl.RECOVER_LRU_VALUES_PROPERTY_NAME, "true"); + + try { + createRegionAndIntiateRecovery(false, true, true, 10, 10); + } finally { + if (oldValue != null) { + System.setProperty(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME, oldValue); + } else { + System.clearProperty(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME); + } + + if (lruOldValue != null) { + System.setProperty(DiskStoreImpl.RECOVER_LRU_VALUES_PROPERTY_NAME, lruOldValue); + } else { + System.clearProperty(DiskStoreImpl.RECOVER_LRU_VALUES_PROPERTY_NAME); + } + } + } + + @Test + public void testValuesAreRecoveredForLruRegionWithReocoverValueAndRecoverLruPropertySetWithRegionClose() { + String oldValue = System.getProperty(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME); + System.setProperty(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME, "true"); + + String lruOldValue = System.getProperty(DiskStoreImpl.RECOVER_LRU_VALUES_PROPERTY_NAME); + System.setProperty(DiskStoreImpl.RECOVER_LRU_VALUES_PROPERTY_NAME, "true"); + + try { + createRegionAndIntiateRecovery(false, false, true, 10, 10); + } finally { + if (oldValue != null) { + System.setProperty(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME, oldValue); + } else { + System.clearProperty(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME); + } + + if (lruOldValue != null) { + System.setProperty(DiskStoreImpl.RECOVER_LRU_VALUES_PROPERTY_NAME, lruOldValue); + } else { + System.clearProperty(DiskStoreImpl.RECOVER_LRU_VALUES_PROPERTY_NAME); + } + } + } + + @Test + public void testBasicVerifyStats() { + DiskRegionProperties diskProps = new DiskRegionProperties(); + final Integer key = 1; + diskProps.setDiskDirs(dirs); + diskProps.setPersistBackup(true); + diskProps.setRegionName("basicVerifyStats"); + region = DiskRegionHelperFactory.getSyncOverFlowAndPersistRegion(cache, diskProps); + region.put(key, new Integer(1)); + region.put(key, new Integer(2)); + region.close(); + + final CountDownLatch recoveryDone = new CountDownLatch(1); + DiskStoreObserver.setInstance(new DiskStoreObserver() { + @Override + public void afterAsyncValueRecovery(DiskStoreImpl store) { + recoveryDone.countDown(); + } + }); + + region = DiskRegionHelperFactory.getSyncOverFlowAndPersistRegion(cache, diskProps); + // Wait for recovery to finish. + try { + recoveryDone.await(); + } catch (InterruptedException ie) { + fail("Found interrupted exception while waiting for recovery."); + } + + DiskRegion dr = ((LocalRegion) region).getDiskRegion(); + assertEquals(0, dr.getNumEntriesInVM()); + assertEquals(1, dr.getNumOverflowOnDisk()); + + region.get(key); + assertEquals(1, dr.getNumEntriesInVM()); + assertEquals(0, dr.getNumOverflowOnDisk()); + + region.clear(); + assertEquals(0, dr.getNumEntriesInVM()); + assertEquals(0, dr.getNumOverflowOnDisk()); + } + +} + http://git-wip-us.apache.org/repos/asf/geode/blob/e606f3e6/geode-core/src/test/java/org/apache/geode/internal/cache/DiskRegRecoveryJUnitTest.java ---------------------------------------------------------------------- diff --git a/geode-core/src/test/java/org/apache/geode/internal/cache/DiskRegRecoveryJUnitTest.java b/geode-core/src/test/java/org/apache/geode/internal/cache/DiskRegRecoveryJUnitTest.java index 6a5c338..d6ee1ea 100644 --- a/geode-core/src/test/java/org/apache/geode/internal/cache/DiskRegRecoveryJUnitTest.java +++ b/geode-core/src/test/java/org/apache/geode/internal/cache/DiskRegRecoveryJUnitTest.java @@ -515,7 +515,9 @@ public class DiskRegRecoveryJUnitTest extends DiskRegionTestingBase { @Test public void testNoEvictionDuringRecoveryIfNoGIIRecoverValuesTrue() { String oldValue = System.getProperty(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME); + String oldLruValue = System.getProperty(DiskStoreImpl.RECOVER_LRU_VALUES_PROPERTY_NAME); System.setProperty(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME, "true"); + System.setProperty(DiskStoreImpl.RECOVER_LRU_VALUES_PROPERTY_NAME, "true"); try { diskProps.setDiskDirs(dirs); diskProps.setPersistBackup(true); @@ -560,6 +562,12 @@ public class DiskRegRecoveryJUnitTest extends DiskRegionTestingBase { } else { System.clearProperty(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME); } + + if (oldLruValue != null) { + System.setProperty(DiskStoreImpl.RECOVER_LRU_VALUES_PROPERTY_NAME, oldLruValue); + } else { + System.clearProperty(DiskStoreImpl.RECOVER_LRU_VALUES_PROPERTY_NAME); + } } } http://git-wip-us.apache.org/repos/asf/geode/blob/e606f3e6/geode-core/src/test/java/org/apache/geode/internal/cache/DiskRegionHelperFactory.java ---------------------------------------------------------------------- diff --git a/geode-core/src/test/java/org/apache/geode/internal/cache/DiskRegionHelperFactory.java b/geode-core/src/test/java/org/apache/geode/internal/cache/DiskRegionHelperFactory.java index fa68044..4ad246f 100755 --- a/geode-core/src/test/java/org/apache/geode/internal/cache/DiskRegionHelperFactory.java +++ b/geode-core/src/test/java/org/apache/geode/internal/cache/DiskRegionHelperFactory.java @@ -17,6 +17,7 @@ package org.apache.geode.internal.cache; import java.io.File; import org.apache.geode.cache.*; +import org.apache.geode.cache.util.ObjectSizer; /** * @@ -76,6 +77,9 @@ public class DiskRegionHelperFactory { factory.setEvictionAttributes( EvictionAttributes.createLRUEntryAttributes(capacity, EvictionAction.OVERFLOW_TO_DISK)); + } else if (diskProps.isHeapEviction()) { + factory.setEvictionAttributes(EvictionAttributes.createLRUHeapAttributes(ObjectSizer.DEFAULT, + EvictionAction.OVERFLOW_TO_DISK)); } factory.setConcurrencyLevel(diskProps.getConcurrencyLevel()); @@ -158,6 +162,28 @@ public class DiskRegionHelperFactory { return getRegion(cache, diskRegionProperties, Scope.LOCAL); } + public static Region getSyncHeapLruAndPersistRegion(Cache cache, + DiskRegionProperties diskRegionProperties) { + if (diskRegionProperties == null) { + diskRegionProperties = new DiskRegionProperties(); + } + diskRegionProperties.setPersistBackup(true); + diskRegionProperties.setSynchronous(true); + diskRegionProperties.setHeapEviction(true); + return getRegion(cache, diskRegionProperties, Scope.LOCAL); + } + + public static Region getAsyncHeapLruAndPersistRegion(Cache cache, + DiskRegionProperties diskRegionProperties) { + if (diskRegionProperties == null) { + diskRegionProperties = new DiskRegionProperties(); + } + diskRegionProperties.setPersistBackup(true); + diskRegionProperties.setSynchronous(false); + diskRegionProperties.setHeapEviction(true); + return getRegion(cache, diskRegionProperties, Scope.LOCAL); + } + public static Region getSyncPersistOnlyRegionInfiniteOplog(Cache cache, DiskRegionProperties diskRegionProperties, String regionName) { if (diskRegionProperties == null) { http://git-wip-us.apache.org/repos/asf/geode/blob/e606f3e6/geode-core/src/test/java/org/apache/geode/internal/cache/DiskRegionProperties.java ---------------------------------------------------------------------- diff --git a/geode-core/src/test/java/org/apache/geode/internal/cache/DiskRegionProperties.java b/geode-core/src/test/java/org/apache/geode/internal/cache/DiskRegionProperties.java index 19662bf..e7299c0 100755 --- a/geode-core/src/test/java/org/apache/geode/internal/cache/DiskRegionProperties.java +++ b/geode-core/src/test/java/org/apache/geode/internal/cache/DiskRegionProperties.java @@ -50,6 +50,7 @@ public class DiskRegionProperties { private int initialCapacity = 16; private float loadFactor = 0.75f; private boolean statisticsEnabled = false; + private boolean isHeapEviction = false; public DiskRegionProperties() {} @@ -66,6 +67,10 @@ public class DiskRegionProperties { return isOverflow; } + public boolean isHeapEviction() { + return isHeapEviction; + } + public boolean isPersistBackup() { return isPersistBackup; } @@ -128,6 +133,10 @@ public class DiskRegionProperties { this.isOverflow = isOverflow; } + public void setHeapEviction(boolean isHeapEviction) { + this.isHeapEviction = isHeapEviction; + } + public void setPersistBackup(boolean isPersistBackup) { this.isPersistBackup = isPersistBackup; } @@ -160,17 +169,14 @@ public class DiskRegionProperties { this.timeInterval = timeInterval; } - public String getRegionName() { return regionName; } - public void setRegionName(String regionName) { this.regionName = regionName; } - public void setStatisticsEnabled(boolean v) { this.statisticsEnabled = v; } http://git-wip-us.apache.org/repos/asf/geode/blob/e606f3e6/geode-core/src/test/java/org/apache/geode/internal/cache/PersistentPartitionedRegionJUnitTest.java ---------------------------------------------------------------------- diff --git a/geode-core/src/test/java/org/apache/geode/internal/cache/PersistentPartitionedRegionJUnitTest.java b/geode-core/src/test/java/org/apache/geode/internal/cache/PersistentPartitionedRegionJUnitTest.java index b0170e1..a72b069 100644 --- a/geode-core/src/test/java/org/apache/geode/internal/cache/PersistentPartitionedRegionJUnitTest.java +++ b/geode-core/src/test/java/org/apache/geode/internal/cache/PersistentPartitionedRegionJUnitTest.java @@ -15,6 +15,7 @@ package org.apache.geode.internal.cache; import org.apache.geode.cache.*; +import org.apache.geode.cache.util.ObjectSizer; import org.apache.geode.internal.FileUtil; import org.apache.geode.test.junit.categories.IntegrationTest; import org.junit.After; @@ -24,9 +25,11 @@ import org.junit.experimental.categories.Category; import java.io.File; import java.util.Properties; +import java.util.concurrent.CountDownLatch; import static org.apache.geode.distributed.ConfigurationProperties.*; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; @Category(IntegrationTest.class) public class PersistentPartitionedRegionJUnitTest { @@ -190,7 +193,132 @@ public class PersistentPartitionedRegionJUnitTest { assertEquals(1000, bucket.getDiskRegion().getStats().getNumEntriesInVM()); } + @Test + public void testValuesAreNotRecoveredForHeapLruRegions() { + createLRURegionAndValidateRecovery(false, true, 10, 0); + } + + @Test + public void testValuesAreNotRecoveredForHeapLruRegionsWithRegionClose() { + createLRURegionAndValidateRecovery(true, true, 10, 0); + } + + @Test + public void testValuesAreNotRecoveredForHeapLruRegionsWithRecoverPropertySet() { + String oldValue = System.getProperty(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME); + System.setProperty(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME, "true"); + + try { + createLRURegionAndValidateRecovery(false, true, 10, 0); + } finally { + if (oldValue != null) { + System.setProperty(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME, oldValue); + } else { + System.clearProperty(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME); + } + } + } + + @Test + public void testValuesAreRecoveredForHeapLruRegionsWithRecoverValueAndRecoverLruPropertySet() { + String oldValue = System.getProperty(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME); + System.setProperty(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME, "true"); + + String lruOldValue = System.getProperty(DiskStoreImpl.RECOVER_LRU_VALUES_PROPERTY_NAME); + System.setProperty(DiskStoreImpl.RECOVER_LRU_VALUES_PROPERTY_NAME, "true"); + + try { + createLRURegionAndValidateRecovery(false, true, 10, 10); + } finally { + if (oldValue != null) { + System.setProperty(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME, oldValue); + } else { + System.clearProperty(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME); + } + + if (lruOldValue != null) { + System.setProperty(DiskStoreImpl.RECOVER_LRU_VALUES_PROPERTY_NAME, lruOldValue); + } else { + System.clearProperty(DiskStoreImpl.RECOVER_LRU_VALUES_PROPERTY_NAME); + } + } + } + + @Test + public void testValuesAreNotRecoveredForEntryLruRegion() { + createLRURegionAndValidateRecovery(false, false, 10, 0); + } + + @Test + public void testValuesAreNotRecoveredForEntryLruRegionWithRegionClose() { + createLRURegionAndValidateRecovery(true, false, 10, 0); + } + + private void createLRURegionAndValidateRecovery(boolean isRegionClose, boolean heapLru, int size, int expectedInMemory) { + PartitionedRegion region; + boolean entryLru = !heapLru; + region = (PartitionedRegion) createRegion(-1, heapLru, entryLru); + + for (int i = 0; i < size; i++) { + region.put(new Integer(i), new Integer(i)); + } + + if (isRegionClose) { + region.close(); + } else { + cache.close(); + } + + final CountDownLatch recoveryDone = new CountDownLatch(1); + DiskStoreObserver.setInstance(new DiskStoreObserver() { + @Override + public void afterAsyncValueRecovery(DiskStoreImpl store) { + recoveryDone.countDown(); + } + }); + region = (PartitionedRegion) createRegion(-1, heapLru, entryLru); + + // Wait for recovery to finish. + try { + recoveryDone.await(); + } catch (InterruptedException ie) { + fail("Found interrupted exception while waiting for recovery."); + } + + int valuesInVm = getValuesInVM(region, size); + assertEquals("Values for lru regions should not be recovered from Disk.", expectedInMemory, valuesInVm); + + BucketRegion bucket = region.getBucketRegion("A"); + assertEquals((size - expectedInMemory), bucket.getDiskRegion().getStats().getNumOverflowOnDisk()); + assertEquals(expectedInMemory, bucket.getDiskRegion().getStats().getNumEntriesInVM()); + + // Load values into memory using get. + for (int i = 0; i < size; i++) { + region.get(new Integer(i)); + } + assertEquals(size, bucket.getDiskRegion().getStats().getNumEntriesInVM()); + } + + private int getValuesInVM(Region region, int size) { + int valuesInVm = 0; + for (int i = 0; i < size; i++) { + try { + Object value = ((LocalRegion) region).getValueInVM(new Integer(i)); + if (value != null) { + valuesInVm++; + } + } catch (EntryNotFoundException e) { + fail("Entry not found not expected but occured "); + } + } + return valuesInVm; + } + private Region createRegion(int ttl) { + return createRegion(ttl, false, false); + } + + private Region createRegion(int ttl, boolean isHeapEviction, boolean isEntryEviction) { Properties props = new Properties(); props.setProperty(MCAST_PORT, "0"); props.setProperty(LOG_LEVEL, "info"); @@ -205,6 +333,15 @@ public class PersistentPartitionedRegionJUnitTest { if (ttl > 0) { rf.setEntryTimeToLive(new ExpirationAttributes(ttl, ExpirationAction.DESTROY)); } + + if (isEntryEviction) { + rf.setEvictionAttributes( + EvictionAttributes.createLRUEntryAttributes(10, EvictionAction.OVERFLOW_TO_DISK)); + } else if (isHeapEviction) { + rf.setEvictionAttributes(EvictionAttributes.createLRUHeapAttributes(ObjectSizer.DEFAULT, + EvictionAction.OVERFLOW_TO_DISK)); + } + Region region = rf.create("region"); return region; }
