GEODE-3198: Allow bucket creation when over local-max-memory The bucket will not be created on a member that is over local max memory, unless all members have failed to create the bucket. In that case the implementation sets a "force" flag that will now cause the bucket to be created even on a member that is over local max memory.
This closes #636 Project: http://git-wip-us.apache.org/repos/asf/geode/repo Commit: http://git-wip-us.apache.org/repos/asf/geode/commit/43d98e03 Tree: http://git-wip-us.apache.org/repos/asf/geode/tree/43d98e03 Diff: http://git-wip-us.apache.org/repos/asf/geode/diff/43d98e03 Branch: refs/heads/feature/GEM-1483 Commit: 43d98e03f0cb1bdefcc7d18ba4a12272056475e7 Parents: 9f6bfaf Author: Nick Reich <nre...@pivotal.io> Authored: Fri Jul 14 11:09:31 2017 -0700 Committer: Darrel Schneider <dschnei...@pivotal.io> Committed: Fri Jul 14 11:54:19 2017 -0700 ---------------------------------------------------------------------- .../cache/PartitionedRegionDataStore.java | 2 +- .../PartitionedRegionDataStoreJUnitTest.java | 26 +++ ...artitionedRegionLocalMaxMemoryDUnitTest.java | 229 +++++++------------ 3 files changed, 105 insertions(+), 152 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/geode/blob/43d98e03/geode-core/src/main/java/org/apache/geode/internal/cache/PartitionedRegionDataStore.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/PartitionedRegionDataStore.java b/geode-core/src/main/java/org/apache/geode/internal/cache/PartitionedRegionDataStore.java index 2589626..0318c75 100644 --- a/geode-core/src/main/java/org/apache/geode/internal/cache/PartitionedRegionDataStore.java +++ b/geode-core/src/main/java/org/apache/geode/internal/cache/PartitionedRegionDataStore.java @@ -934,7 +934,7 @@ public class PartitionedRegionDataStore implements HasCachePerfStats { } return false; } - if (!canAccommodateMoreBytesSafely(size)) { + if (!forceCreation && !canAccommodateMoreBytesSafely(size)) { if (logger.isDebugEnabled()) { logger.debug( "Partitioned Region {} has exceeded local maximum memory configuration {} Mb, current size is {} Mb", http://git-wip-us.apache.org/repos/asf/geode/blob/43d98e03/geode-core/src/test/java/org/apache/geode/internal/cache/PartitionedRegionDataStoreJUnitTest.java ---------------------------------------------------------------------- diff --git a/geode-core/src/test/java/org/apache/geode/internal/cache/PartitionedRegionDataStoreJUnitTest.java b/geode-core/src/test/java/org/apache/geode/internal/cache/PartitionedRegionDataStoreJUnitTest.java index 66792f2..cd0917e 100644 --- a/geode-core/src/test/java/org/apache/geode/internal/cache/PartitionedRegionDataStoreJUnitTest.java +++ b/geode-core/src/test/java/org/apache/geode/internal/cache/PartitionedRegionDataStoreJUnitTest.java @@ -224,4 +224,30 @@ public class PartitionedRegionDataStoreJUnitTest { regionAck.put(new Integer(key), "foo"); } } + + @Test + public void doesNotCreateBucketIfOverMemoryLimit() { + final int numMBytes = 5; + final PartitionedRegion regionAck = (PartitionedRegion) new RegionFactory() + .setPartitionAttributes(new PartitionAttributesFactory().setRedundantCopies(0) + .setLocalMaxMemory(numMBytes).create()) + .create(this.regionName); + + boolean createdBucket = + regionAck.getDataStore().handleManageBucketRequest(1, Integer.MAX_VALUE, null, false); + assertFalse(createdBucket); + } + + @Test + public void createsBucketWhenForcedIfOverMemoryLimit() { + final int numMBytes = 5; + final PartitionedRegion regionAck = (PartitionedRegion) new RegionFactory() + .setPartitionAttributes(new PartitionAttributesFactory().setRedundantCopies(0) + .setLocalMaxMemory(numMBytes).create()) + .create(this.regionName); + + boolean createdBucket = + regionAck.getDataStore().handleManageBucketRequest(1, Integer.MAX_VALUE, null, true); + assertTrue(createdBucket); + } } http://git-wip-us.apache.org/repos/asf/geode/blob/43d98e03/geode-core/src/test/java/org/apache/geode/internal/cache/PartitionedRegionLocalMaxMemoryDUnitTest.java ---------------------------------------------------------------------- diff --git a/geode-core/src/test/java/org/apache/geode/internal/cache/PartitionedRegionLocalMaxMemoryDUnitTest.java b/geode-core/src/test/java/org/apache/geode/internal/cache/PartitionedRegionLocalMaxMemoryDUnitTest.java index 89daed0..92699e5 100755 --- a/geode-core/src/test/java/org/apache/geode/internal/cache/PartitionedRegionLocalMaxMemoryDUnitTest.java +++ b/geode-core/src/test/java/org/apache/geode/internal/cache/PartitionedRegionLocalMaxMemoryDUnitTest.java @@ -21,16 +21,16 @@ import java.io.DataOutput; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Iterator; import java.util.List; +import org.junit.After; +import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.apache.geode.DataSerializable; import org.apache.geode.DataSerializer; import org.apache.geode.cache.Cache; -import org.apache.geode.cache.PartitionedRegionStorageException; import org.apache.geode.cache.Region; import org.apache.geode.cache.util.ObjectSizer; import org.apache.geode.cache30.CacheSerializableRunnable; @@ -42,202 +42,128 @@ import org.apache.geode.test.dunit.VM; import org.apache.geode.test.junit.categories.DistributedTest; /** - * This class is to test localMaxMemory property of partition region while creation of bucket. + * This class is to test LOCAL_MAX_MEMORY property of partition region while creation of bucket. */ @Category(DistributedTest.class) public class PartitionedRegionLocalMaxMemoryDUnitTest extends PartitionedRegionDUnitTestCase { - /** Prefix is used in name of Partition Region */ - private static String prPrefix = null; + private static final int LOCAL_MAX_MEMORY = 1; + private static int MAX_REGIONS = 1; + private static final int REDUNDANCY = 1; - /** Maximum number of regions * */ - static int MAX_REGIONS = 1; + private String regionNamePrefix = null; + private List<VM> vms = new ArrayList<>(2); - /** local maxmemory used for the creation of the partition region */ - int localMaxMemory = 1; + @Before + public void setup() { + Host host = Host.getHost(0); + vms.add(host.getVM(0)); + vms.add(host.getVM(1)); + } - /** to store references of 4 vms */ - VM vm[] = new VM[4]; + @After + public void cleanup() { + destroyRegion(vms.get(0)); + } /** * This test performs following operations <br> - * 1.Create Partition region with localMaxMemory = 1MB on all the VMs </br> + * 1.Create Partition region with LOCAL_MAX_MEMORY = 1MB on all the VMs </br> * <br> * 2.Put objects in partition region so that only one bucket gets created and size of that bucket - * exceeds localMaxMemory <br> + * exceeds LOCAL_MAX_MEMORY <br> * 3.Put object such that new bucket gets formed</br> * <br> - * 4.Test should throw PartitionedRegionStorageException when it tries to create new bucket</br> + * 4.Test should create a new bucket</br> */ @Test public void testLocalMaxMemoryInPartitionedRegion() { - Host host = Host.getHost(0); - /** creating 4 VMs */ - this.vm[0] = host.getVM(0); - this.vm[1] = host.getVM(1); - this.vm[2] = null; - this.vm[3] = null; - /** Prefix will be used for naming the partititon Region */ - prPrefix = "testLocalMaxMemoryInPartitionedRegion"; - /** these indices represents range of partition regions present in each VM */ - int startIndexForRegion = 0; - int endIndexForRegion = MAX_REGIONS; - int startIndexForNode = 0; - int endIndexForNode = 2; - // creating partition region on 4 nodes with - // localMaxMemory=1MB redundancy = 3 - List vmList = addNodeToList(startIndexForNode, endIndexForNode); - localMaxMemory = 1; - final int redundancy = 1; - System.setProperty(PartitionedRegion.RETRY_TIMEOUT_PROPERTY, "20000"); - createPartitionRegion(vmList, startIndexForRegion, endIndexForRegion, localMaxMemory, - redundancy, false); - System.setProperty(PartitionedRegion.RETRY_TIMEOUT_PROPERTY, - Integer.toString(PartitionedRegionHelper.DEFAULT_TOTAL_WAIT_RETRY_ITERATION)); - putFromOneVm(vm[0], true); - putFromOneVm(vm[0], false); - destroyRegion(vm[0]); + regionNamePrefix = "maxMemoryTest"; + createPartitionRegionOnAllVMs(false); + VM vm = vms.get(0); + putFromOneVm(vm, 10, true); + putFromOneVm(vm, 21, false); } /** - * This test makes sure that we don't enforce the localMaxMemory setting when eviction is enabled. + * This test makes sure that we don't enforce the LOCAL_MAX_MEMORY setting when eviction is + * enabled. */ @Test public void testLocalMaxMemoryInPartitionedRegionWithEviction() { - Host host = Host.getHost(0); - /** creating 4 VMs */ - this.vm[0] = host.getVM(0); - this.vm[1] = host.getVM(1); - this.vm[2] = null; - this.vm[3] = null; - /** Prefix will be used for naming the partititon Region */ - prPrefix = "testLocalMaxMemoryInPartitionedRegion"; - /** these indices represents range of partition regions present in each VM */ - int startIndexForRegion = 0; - int endIndexForRegion = MAX_REGIONS; - int startIndexForNode = 0; - int endIndexForNode = 2; - // creating partition region on 4 nodes with - // localMaxMemory=1MB redundancy = 3 - List vmList = addNodeToList(startIndexForNode, endIndexForNode); - localMaxMemory = 1; - final int redundancy = 1; - System.setProperty(PartitionedRegion.RETRY_TIMEOUT_PROPERTY, "20000"); - createPartitionRegion(vmList, startIndexForRegion, endIndexForRegion, localMaxMemory, - redundancy, true); - System.setProperty(PartitionedRegion.RETRY_TIMEOUT_PROPERTY, - Integer.toString(PartitionedRegionHelper.DEFAULT_TOTAL_WAIT_RETRY_ITERATION)); - putFromOneVm(vm[0], true); - putFromOneVm(vm[0], true); - destroyRegion(vm[0]); + regionNamePrefix = "maxMemoryWithEvictionTest"; + createPartitionRegionOnAllVMs(true); + VM vm = vms.get(0); + putFromOneVm(vm, 10, true); + putFromOneVm(vm, 10, true); } - /** - * function is used perform put() operation from one VM - * - * @param vm - * @param objectFlg - */ - private void putFromOneVm(VM vm, boolean objectFlg) { - vm.invoke(putObjectInPartitionRegion(objectFlg)); + private void putFromOneVm(VM vm, int objectId, boolean fillMemory) { + if (fillMemory) { + vm.invoke(fillRegion(objectId)); + } else { + vm.invoke(putObjectInPartitionRegion(objectId)); + } } - /** - * This function is used to put objects of different hashcode depending upon value of objectFlag - * - * @param objectFlg - * @return - */ - private CacheSerializableRunnable putObjectInPartitionRegion(final boolean objectFlg) { + private CacheSerializableRunnable putObjectInPartitionRegion(int objectId) { + CacheSerializableRunnable putObject = new CacheSerializableRunnable("putObject") { + public void run2() { + PartitionedRegion pr = getRegion(); + TestObject1 kv = new TestObject1("testObject1" + 0, objectId); + pr.put(kv, kv); + LogWriterUtils.getLogWriter().info( + "putObjectInPartitionRegion() - Put operation with different identifier done successfully"); + } + }; + return putObject; + } + private CacheSerializableRunnable fillRegion(int objectId) { CacheSerializableRunnable putObject = new CacheSerializableRunnable("putObject") { public void run2() { Cache cache = getCache(); - PartitionedRegion pr = (PartitionedRegion) cache - .getRegion(Region.SEPARATOR + "testLocalMaxMemoryInPartitionedRegion0"); - assertNotNull("Name of region : " + pr.getName(), pr); - int i = 0; + PartitionedRegion pr = getRegion(); - if (objectFlg == true) { - long size = 0; - while ((size = - pr.getDataStore().currentAllocatedMemory()) < PartitionedRegionHelper.BYTES_PER_MB) { - cache.getLogger().info("size: " + size); - Object obj = new TestObject1("testObject1" + i, 10); - pr.put(obj, obj); - i++; - } - assertEquals(1, pr.getDataStore().localBucket2RegionMap.size()); - LogWriterUtils.getLogWriter() - .info("putObjectInPartitionRegion() - Put operation done successfully"); - } else { - final String expectedExceptions = PartitionedRegionStorageException.class.getName(); - getCache().getLogger() - .info("<ExpectedException action=add>" + expectedExceptions + "</ExpectedException>"); - try { - TestObject1 kv = new TestObject1("testObject1" + i, 21); - pr.put(kv, kv); - fail("Bucket gets created even if no memory is available"); - } catch (PartitionedRegionStorageException e) { - LogWriterUtils.getLogWriter().info( - "putObjectInPartitionRegion()- got correct PartitionedRegionStorageException while creating bucket when no memory is available"); - } - getCache().getLogger().info( - "<ExpectedException action=remove>" + expectedExceptions + "</ExpectedException>"); - } + fillAllMemoryWithPuts(cache, pr, objectId); + assertEquals(1, pr.getDataStore().localBucket2RegionMap.size()); + LogWriterUtils.getLogWriter() + .info("putObjectInPartitionRegion() - Put operation done successfully"); } }; return putObject; } - /** - * This function createas multiple partition regions on nodes specified in the vmList - * - * @param evict - */ - private void createPartitionRegion(List vmList, int startIndexForRegion, int endIndexForRegion, - int localMaxMemory, int redundancy, boolean evict) { - Iterator nodeIterator = vmList.iterator(); - while (nodeIterator.hasNext()) { - VM vm = (VM) nodeIterator.next(); - vm.invoke(createMultiplePartitionRegion(prPrefix, startIndexForRegion, endIndexForRegion, - redundancy, localMaxMemory, evict)); - } + private PartitionedRegion getRegion() { + Cache cache = getCache(); + return (PartitionedRegion) cache.getRegion(Region.SEPARATOR + regionNamePrefix + "0"); } - /** - * This function adds nodes to node list - * - * @param startIndexForNode - * @param endIndexForNode - * @return - */ - private List addNodeToList(int startIndexForNode, int endIndexForNode) { - List localvmList = new ArrayList(); - for (int i = startIndexForNode; i < endIndexForNode; i++) { - localvmList.add(vm[i]); + private void fillAllMemoryWithPuts(Cache cache, PartitionedRegion pr, int objectId) { + int i = 0; + long allocatedMemory; + while ((allocatedMemory = + pr.getDataStore().currentAllocatedMemory()) < PartitionedRegionHelper.BYTES_PER_MB) { + cache.getLogger().info("size: " + allocatedMemory); + Object obj = new TestObject1("testObject1" + i, objectId); + pr.put(obj, obj); + i++; } - return localvmList; + assertEquals(1, pr.getDataStore().localBucket2RegionMap.size()); } - /** - * this function creates vms in given host - * - * @param host - */ - private void createVMs(Host host) { - for (int i = 0; i < 4; i++) { - vm[i] = host.getVM(i); + private void createPartitionRegionOnAllVMs(boolean evict) { + for (VM vm : vms) { + vm.invoke(createMultiplePartitionRegion(regionNamePrefix, 0, MAX_REGIONS, REDUNDANCY, + LOCAL_MAX_MEMORY, evict)); } } private void destroyRegion(VM vm) { SerializableRunnable destroyObj = new CacheSerializableRunnable("destroyObj") { public void run2() { - Cache cache = getCache(); - PartitionedRegion pr = (PartitionedRegion) cache - .getRegion(Region.SEPARATOR + "testLocalMaxMemoryInPartitionedRegion0"); - assertNotNull("Name of region : " + pr.getName(), pr); + PartitionedRegion pr = getRegion(); + assertNotNull(pr); pr.destroyRegion(); } }; @@ -245,7 +171,8 @@ public class PartitionedRegionLocalMaxMemoryDUnitTest extends PartitionedRegionD } /** - * Object used for the put() operation as key and object + * Object used for the put() operation as key and object. The objectIdentifier is used to provide + * a predetermined hashcode for the object. */ static public class TestObject1 implements DataSerializable, Sizeable { String name;
