Merge remote-tracking branch 'origin/develop' into feature/GEODE-77
Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/e0d1c4f9 Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/e0d1c4f9 Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/e0d1c4f9 Branch: refs/heads/feature/GEODE-77 Commit: e0d1c4f908b30bdf239041403bf9c6369f2be22b Parents: ab68f4e eb7e7b7 Author: Bruce Schuchardt <bschucha...@pivotal.io> Authored: Fri Aug 14 13:39:20 2015 -0700 Committer: Bruce Schuchardt <bschucha...@pivotal.io> Committed: Fri Aug 14 13:39:20 2015 -0700 ---------------------------------------------------------------------- build.gradle | 1 + docker/Dockerfile | 56 ++ docker/README.md | 67 ++ gemfire-assembly/build.gradle | 1 + gemfire-core/build.gradle | 30 +- .../cache/operations/GetOperationContext.java | 7 +- .../operations/KeyValueOperationContext.java | 70 +- .../internal/GetOperationContextImpl.java | 16 +- .../cache/query/internal/RuntimeIterator.java | 3 - .../query/internal/index/AbstractMapIndex.java | 6 +- .../internal/index/CompactMapRangeIndex.java | 7 +- .../internal/DistributionConfigImpl.java | 1 - .../gemfire/internal/SharedLibrary.java | 20 +- .../gemfire/internal/cache/CacheConfig.java | 4 +- .../DistTXStateProxyImplOnCoordinator.java | 2 + .../gemfire/internal/cache/ExpiryTask.java | 2 +- .../internal/cache/GemFireCacheImpl.java | 3 +- .../gemfire/internal/cache/LocalRegion.java | 9 +- .../cache/MinimumSystemRequirements.java | 4 +- .../internal/cache/RegionIdleExpiryTask.java | 3 +- .../gemfire/internal/cache/TXManagerImpl.java | 11 +- .../cache/control/HeapMemoryMonitor.java | 14 +- .../internal/cache/control/MemoryEvent.java | 9 +- .../cache/control/OffHeapMemoryMonitor.java | 88 +- .../cache/control/ResourceManagerStats.java | 13 + .../internal/cache/partitioned/GetMessage.java | 6 + .../AbstractGatewaySenderEventProcessor.java | 1 + .../parallel/ParallelGatewaySenderQueue.java | 41 +- .../internal/cache/xmlcache/CacheXmlParser.java | 26 +- .../gemfire/internal/lang/SystemUtils.java | 35 +- .../gemfire/internal/logging/LogService.java | 58 +- .../internal/logging/log4j/Configurator.java | 65 +- .../internal/logging/log4j/FastLogger.java | 50 +- .../offheap/SimpleMemoryAllocatorImpl.java | 2 +- .../internal/redis/ByteArrayWrapper.java | 11 +- .../internal/redis/ByteToCommandDecoder.java | 57 +- .../gemstone/gemfire/internal/redis/Coder.java | 24 +- .../gemfire/internal/redis/Command.java | 25 +- .../gemfire/internal/redis/DoubleWrapper.java | 14 +- .../internal/redis/ExecutionHandlerContext.java | 139 +++- .../gemfire/internal/redis/Executor.java | 2 +- .../gemfire/internal/redis/Extendable.java | 6 +- .../gemfire/internal/redis/RedisConstants.java | 2 +- .../gemfire/internal/redis/RedisDataType.java | 9 - .../gemfire/internal/redis/RegionCache.java | 410 ---------- .../internal/redis/RegionCreationException.java | 9 +- .../gemfire/internal/redis/RegionProvider.java | 531 ++++++++++++ .../redis/executor/AbstractExecutor.java | 12 +- .../redis/executor/AbstractScanExecutor.java | 2 +- .../internal/redis/executor/DBSizeExecutor.java | 2 +- .../internal/redis/executor/DelExecutor.java | 2 +- .../internal/redis/executor/EchoExecutor.java | 2 +- .../internal/redis/executor/ExistsExecutor.java | 2 +- .../redis/executor/ExpirationExecutor.java | 6 +- .../redis/executor/ExpireAtExecutor.java | 4 +- .../internal/redis/executor/ExpireExecutor.java | 4 +- .../redis/executor/FlushAllExecutor.java | 15 +- .../internal/redis/executor/KeysExecutor.java | 7 +- .../internal/redis/executor/ListQuery.java | 12 +- .../redis/executor/PersistExecutor.java | 2 +- .../internal/redis/executor/PingExecutor.java | 2 +- .../internal/redis/executor/QuitExecutor.java | 2 +- .../internal/redis/executor/ScanExecutor.java | 4 +- .../internal/redis/executor/SortedSetQuery.java | 36 +- .../internal/redis/executor/TTLExecutor.java | 4 +- .../internal/redis/executor/TypeExecutor.java | 2 +- .../internal/redis/executor/UnkownExecutor.java | 2 +- .../redis/executor/hash/HDelExecutor.java | 2 +- .../redis/executor/hash/HGetAllExecutor.java | 7 +- .../redis/executor/hash/HKeysExecutor.java | 7 +- .../redis/executor/hash/HScanExecutor.java | 5 +- .../redis/executor/hash/HValsExecutor.java | 9 +- .../redis/executor/hash/HashExecutor.java | 4 +- .../redis/executor/hll/HllExecutor.java | 2 +- .../redis/executor/hll/PFAddExecutor.java | 2 +- .../redis/executor/hll/PFCountExecutor.java | 2 +- .../redis/executor/hll/PFMergeExecutor.java | 4 +- .../redis/executor/list/LIndexExecutor.java | 4 +- .../redis/executor/list/LLenExecutor.java | 2 +- .../redis/executor/list/LRangeExecutor.java | 10 +- .../redis/executor/list/LRemExecutor.java | 4 +- .../redis/executor/list/LSetExecutor.java | 4 +- .../redis/executor/list/LTrimExecutor.java | 26 +- .../redis/executor/list/ListExecutor.java | 26 +- .../redis/executor/list/PopExecutor.java | 23 +- .../redis/executor/list/PushExecutor.java | 2 +- .../redis/executor/list/PushXExecutor.java | 2 +- .../org/apache/hadoop/fs/GlobPattern.java | 164 ---- .../redis/executor/set/SAddExecutor.java | 2 +- .../redis/executor/set/SCardExecutor.java | 2 +- .../redis/executor/set/SIsMemberExecutor.java | 2 +- .../redis/executor/set/SMembersExecutor.java | 7 +- .../redis/executor/set/SMoveExecutor.java | 2 +- .../redis/executor/set/SPopExecutor.java | 4 +- .../redis/executor/set/SRandMemberExecutor.java | 4 +- .../redis/executor/set/SRemExecutor.java | 2 +- .../redis/executor/set/SScanExecutor.java | 4 +- .../redis/executor/set/SetOpExecutor.java | 9 +- .../executor/sortedset/SortedSetExecutor.java | 4 +- .../executor/sortedset/ZRangeByLexExecutor.java | 2 - .../sortedset/ZRangeByScoreExecutor.java | 3 +- .../redis/executor/sortedset/ZRemExecutor.java | 2 +- .../sortedset/ZRemRangeByLexExecutor.java | 3 +- .../sortedset/ZRemRangeByRankExecutor.java | 4 +- .../sortedset/ZRemRangeByScoreExecutor.java | 4 +- .../redis/executor/sortedset/ZScanExecutor.java | 5 +- .../redis/executor/string/AppendExecutor.java | 2 +- .../redis/executor/string/BitCountExecutor.java | 2 +- .../redis/executor/string/BitOpExecutor.java | 2 +- .../redis/executor/string/BitPosExecutor.java | 2 +- .../redis/executor/string/DecrByExecutor.java | 2 +- .../redis/executor/string/DecrExecutor.java | 4 +- .../redis/executor/string/GetBitExecutor.java | 2 +- .../redis/executor/string/GetExecutor.java | 2 +- .../redis/executor/string/GetRangeExecutor.java | 2 +- .../redis/executor/string/GetSetExecutor.java | 2 +- .../redis/executor/string/IncrByExecutor.java | 2 +- .../executor/string/IncrByFloatExecutor.java | 2 +- .../redis/executor/string/IncrExecutor.java | 2 +- .../redis/executor/string/MGetExecutor.java | 2 +- .../redis/executor/string/MSetExecutor.java | 2 +- .../redis/executor/string/MSetNXExecutor.java | 2 +- .../redis/executor/string/SetBitExecutor.java | 2 +- .../redis/executor/string/SetEXExecutor.java | 4 +- .../redis/executor/string/SetExecutor.java | 4 +- .../redis/executor/string/SetNXExecutor.java | 2 +- .../redis/executor/string/SetRangeExecutor.java | 4 +- .../redis/executor/string/StringExecutor.java | 4 +- .../redis/executor/string/StrlenExecutor.java | 2 +- .../redis/org/apache/hadoop/fs/GlobPattern.java | 164 ++++ .../size/ReflectionSingleObjectSizer.java | 4 +- .../internal/cli/commands/QueueCommands.java | 4 +- .../CreateAsyncEventQueueFunction.java | 8 +- .../internal/cli/i18n/CliStrings.java | 4 +- .../controllers/QueueCommandsController.java | 6 +- .../pdx/ReflectionBasedAutoSerializer.java | 9 + .../pdx/internal/AutoSerializableManager.java | 8 - .../gemfire/pdx/internal/TypeRegistry.java | 4 +- .../gemfire/redis/GemFireRedisServer.java | 130 +-- .../internal/RegionWithHDFSBasicDUnitTest.java | 22 +- .../RegionWithHDFSOffHeapBasicDUnitTest.java | 10 +- .../hdfs/internal/RegionWithHDFSTestBase.java | 6 +- .../MemoryThresholdsOffHeapDUnitTest.java | 42 +- .../PutOperationContextJUnitTest.java | 248 ++++++ .../GetOperationContextImplJUnitTest.java | 275 +++++++ .../query/dunit/CloseCacheAuthorization.java | 2 - .../index/CopyOnReadIndexDUnitTest.java | 68 +- .../index/CopyOnReadIndexJUnitTest.java | 5 +- .../MapRangeIndexMaintenanceJUnitTest.java | 50 ++ .../gemfire/cache30/MultiVMRegionTestCase.java | 94 ++- .../gemfire/cache30/RegionTestCase.java | 137 ++-- .../LocatorLauncherRemoteFileJUnitTest.java | 8 +- .../LocatorLauncherRemoteJUnitTest.java | 57 +- .../ServerLauncherLocalJUnitTest.java | 47 +- .../ServerLauncherRemoteFileJUnitTest.java | 2 - .../ServerLauncherRemoteJUnitTest.java | 2 +- .../ServerLauncherWithSpringJUnitTest.java | 2 + .../gemfire/disttx/DistTXDebugDUnitTest.java | 3 + .../disttx/DistributedTransactionDUnitTest.java | 19 +- .../internal/cache/Bug34011JUnitTest.java | 199 ----- .../cache/ClientServerTransactionDUnitTest.java | 38 +- .../ConcurrentRegionOperationsJUnitTest.java | 7 +- .../PartitionedRegionSingleHopDUnitTest.java | 4 - .../cache/RemoteTransactionDUnitTest.java | 113 ++- .../control/RebalanceOperationDUnitTest.java | 2 + .../cache/ha/Bug36853EventsExpiryDUnitTest.java | 1 + .../DistributedSystemLogFileJUnitTest.java | 20 +- .../logging/LoggingIntegrationTestSuite.java | 14 + .../internal/logging/LoggingUnitTestSuite.java | 12 + .../log4j/FastLoggerIntegrationJUnitTest.java | 557 +++++++++++++ .../logging/log4j/FastLoggerJUnitTest.java | 274 +++---- .../FastLoggerWithDefaultConfigJUnitTest.java | 74 ++ .../log4j/Log4jIntegrationTestSuite.java | 12 + .../logging/log4j/Log4jUnitTestSuite.java | 16 + .../internal/size/ObjectSizerJUnitTest.java | 14 +- .../gemfire/pdx/AutoSerializableJUnitTest.java | 2 +- .../gemfire/redis/ConcurrentStartTest.java | 58 ++ .../gemstone/gemfire/redis/HashesJUnitTest.java | 175 ++++ .../gemstone/gemfire/redis/ListsJUnitTest.java | 238 ++++++ .../gemfire/redis/RedisDistDUnitTest.java | 231 ++++++ .../gemstone/gemfire/redis/SetsJUnitTest.java | 242 ++++++ .../gemfire/redis/SortedSetsJUnitTest.java | 414 ++++++++++ .../gemfire/redis/StringsJunitTest.java | 296 +++++++ .../gemfire/test/golden/ExecutableProcess.java | 8 + .../gemfire/test/golden/FailOutputTestCase.java | 22 +- .../golden/FailWithErrorInOutputJUnitTest.java | 18 +- .../FailWithExtraLineInOutputJUnitTest.java | 41 +- ...WithLineMissingFromEndOfOutputJUnitTest.java | 39 +- ...hLineMissingFromMiddleOfOutputJUnitTest.java | 39 +- .../FailWithLoggerErrorInOutputJUnitTest.java | 18 +- .../FailWithLoggerFatalInOutputJUnitTest.java | 18 +- .../FailWithLoggerWarnInOutputJUnitTest.java | 18 +- .../golden/FailWithProblemInOutputTestCase.java | 30 +- .../golden/FailWithSevereInOutputJUnitTest.java | 18 +- ...hTimeoutOfWaitForOutputToMatchJUnitTest.java | 42 +- .../FailWithWarningInOutputJUnitTest.java | 18 +- .../gemfire/test/golden/GoldenComparator.java | 64 +- .../test/golden/GoldenStringComparator.java | 5 +- .../gemfire/test/golden/GoldenTestCase.java | 108 +-- .../golden/GoldenTestFrameworkTestSuite.java | 27 + .../gemfire/test/golden/PassJUnitTest.java | 50 +- .../golden/PassWithExpectedErrorJUnitTest.java | 18 +- .../golden/PassWithExpectedProblemTestCase.java | 58 +- .../golden/PassWithExpectedSevereJUnitTest.java | 18 +- .../PassWithExpectedWarningJUnitTest.java | 18 +- .../test/golden/RegexGoldenComparator.java | 10 +- .../test/golden/StringGoldenComparator.java | 4 +- .../process/ProcessTestFrameworkTestSuite.java | 12 + .../gemfire/test/process/ProcessWrapper.java | 251 +++--- .../test/process/ProcessWrapperJUnitTest.java | 19 +- .../test/java/dunit/DistributedTestCase.java | 19 +- .../gemfire/test/golden/log4j2-test.xml | 18 + gemfire-rebalancer/build.gradle | 12 + .../gemfire/cache/util/AutoBalancer.java | 512 ++++++++++++ .../cache/util/AutoBalancerJUnitTest.java | 803 +++++++++++++++++++ .../connector/GemFirePairRDDFunctions.scala | 5 +- .../spark/connector/GemFireRDDFunctions.scala | 5 +- .../internal/DefaultGemFireConnection.scala | 30 +- .../connector/internal/LocatorHelper.scala | 91 ++- .../internal/rdd/GemFireRegionRDD.scala | 2 +- .../gemfire/spark/connector/package.scala | 7 + .../spark/connector/LocatorHelperTest.scala | 77 ++ settings.gradle | 1 + 223 files changed, 7167 insertions(+), 2385 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/e0d1c4f9/gemfire-assembly/build.gradle ---------------------------------------------------------------------- diff --cc gemfire-assembly/build.gradle index a50d3ff,f65930d..7e7dcb3 --- a/gemfire-assembly/build.gradle +++ b/gemfire-assembly/build.gradle @@@ -112,7 -113,7 +112,8 @@@ def cp = it.contains('spring-shell') || it.contains('snappy-java') || it.contains('hbase') || + it.contains('jgroups') + it.contains('netty') }.join(' ') } http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/e0d1c4f9/gemfire-core/build.gradle ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/e0d1c4f9/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/DistributionConfigImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/e0d1c4f9/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/LocalRegion.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/e0d1c4f9/gemfire-core/src/main/java/com/gemstone/gemfire/internal/logging/LogService.java ---------------------------------------------------------------------- diff --cc gemfire-core/src/main/java/com/gemstone/gemfire/internal/logging/LogService.java index e12f83f,a4a399d..877fdb1 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/logging/LogService.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/logging/LogService.java @@@ -117,12 -119,13 +119,13 @@@ public class LogService extends LogMana /** * Check to see if the user has specified a Log4j configuration file. If not, attempt * to find a GemFire Log4j configuration file in various locations. + * + * @return true if log4j.configurationFile property was set; false if it was unchanged */ - private static final void setLog4jConfigFileProperty() { + private static final boolean setLog4jConfigFileProperty() { // fix bug #52175 final URL configInClasspath = ConfigLocator.findConfigInClasspath(); - if (configInClasspath != null ) { + if (configInClasspath != null) { - // System.out.println("log config is " + configInClasspath); // Log4J 2 will find the configuration file in classpath so do nothing configFileInformation = "Using log4j configuration found in classpath: '" + configInClasspath.toString() + "'"; StatusLogger.getLogger().info(configFileInformation); http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/e0d1c4f9/gemfire-core/src/main/java/com/gemstone/gemfire/internal/redis/RegionProvider.java ---------------------------------------------------------------------- diff --cc gemfire-core/src/main/java/com/gemstone/gemfire/internal/redis/RegionProvider.java index 0000000,a95f853..18971eb mode 000000,100644..100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/redis/RegionProvider.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/redis/RegionProvider.java @@@ -1,0 -1,531 +1,531 @@@ + package com.gemstone.gemfire.internal.redis; + + import java.io.Closeable; + import java.util.HashMap; + import java.util.Map; + import java.util.Map.Entry; + import java.util.Set; + import java.util.concurrent.ConcurrentHashMap; + import java.util.concurrent.ConcurrentMap; + import java.util.concurrent.ScheduledExecutorService; + import java.util.concurrent.ScheduledFuture; + import java.util.concurrent.TimeUnit; + import java.util.concurrent.locks.Lock; + import java.util.concurrent.locks.ReentrantLock; + + import com.gemstone.gemfire.LogWriter; + import com.gemstone.gemfire.cache.Cache; + import com.gemstone.gemfire.cache.CacheTransactionManager; + import com.gemstone.gemfire.cache.Region; + import com.gemstone.gemfire.cache.RegionShortcut; + import com.gemstone.gemfire.cache.TransactionId; + import com.gemstone.gemfire.cache.query.IndexNameConflictException; + import com.gemstone.gemfire.cache.query.Query; + import com.gemstone.gemfire.cache.query.QueryInvalidException; + import com.gemstone.gemfire.cache.query.QueryService; + import com.gemstone.gemfire.cache.query.RegionNotFoundException; + import com.gemstone.gemfire.internal.cache.GemFireCacheImpl; + import com.gemstone.gemfire.internal.redis.executor.ExpirationExecutor; + import com.gemstone.gemfire.internal.redis.executor.ListQuery; + import com.gemstone.gemfire.internal.redis.executor.SortedSetQuery; + import com.gemstone.gemfire.internal.redis.executor.hll.HyperLogLogPlus; + import com.gemstone.gemfire.management.cli.Result; + import com.gemstone.gemfire.management.cli.Result.Status; + import com.gemstone.gemfire.management.internal.cli.commands.CreateAlterDestroyRegionCommands; + import com.gemstone.gemfire.redis.GemFireRedisServer; + + /** + * This class stands between {@link Executor} and {@link Cache#getRegion(String)}. + * This is needed because some keys for Redis represented as a {@link Region} in + * {@link GemFireRedisServer} come with additional state. Therefore getting, creating, + * or destroying a {@link Region} needs to be synchronized, which is done away with + * and abstracted by this class. + * + * @author Vitaly Gavrilov + * + */ + public class RegionProvider implements Closeable { + + private final ConcurrentHashMap<ByteArrayWrapper, Region<?, ?>> regions; + + /** + * This is the Redis meta data {@link Region} that holds the {@link RedisDataType} + * information for all Regions created. The mapping is a {@link String} key which is the name + * of the {@link Region} created to hold the data to the RedisDataType it contains. + */ + private final Region<String, RedisDataType> redisMetaRegion; + + /** + * This is the {@link RedisDataType#REDIS_STRING} {@link Region}. This is the Region + * that stores all string contents + */ + private final Region<ByteArrayWrapper, ByteArrayWrapper> stringsRegion; + + /** + * This is the {@link RedisDataType#REDIS_HLL} {@link Region}. This is the Region + * that stores all HyperLogLog contents + */ + private final Region<ByteArrayWrapper, HyperLogLogPlus> hLLRegion; + + private final Cache cache; + private final QueryService queryService; + private final ConcurrentMap<ByteArrayWrapper, Map<Enum<?>, Query>> preparedQueries = new ConcurrentHashMap<ByteArrayWrapper, Map<Enum<?>, Query>>(); + private final ConcurrentMap<ByteArrayWrapper, ScheduledFuture<?>> expirationsMap; + private final ScheduledExecutorService expirationExecutor; + private final RegionShortcut defaultRegionType; + private static final CreateAlterDestroyRegionCommands cliCmds = new CreateAlterDestroyRegionCommands(); + private final ConcurrentHashMap<String, Lock> locks; + private final LogWriter logger; + + public RegionProvider(Region<ByteArrayWrapper, ByteArrayWrapper> stringsRegion, Region<ByteArrayWrapper, HyperLogLogPlus> hLLRegion, Region<String, RedisDataType> redisMetaRegion, ConcurrentMap<ByteArrayWrapper, ScheduledFuture<?>> expirationsMap, ScheduledExecutorService expirationExecutor, RegionShortcut defaultShortcut) { + if (stringsRegion == null || hLLRegion == null || redisMetaRegion == null) + throw new NullPointerException(); + this.regions = new ConcurrentHashMap<ByteArrayWrapper, Region<?, ?>>(); + this.stringsRegion = stringsRegion; + this.hLLRegion = hLLRegion; + this.redisMetaRegion = redisMetaRegion; + this.cache = GemFireCacheImpl.getInstance(); + this.queryService = cache.getQueryService(); + this.expirationsMap = expirationsMap; + this.expirationExecutor = expirationExecutor; + this.defaultRegionType = defaultShortcut; + this.locks = new ConcurrentHashMap<String, Lock>(); + this.logger = this.cache.getLogger(); + } + + public boolean existsKey(ByteArrayWrapper key) { + return this.redisMetaRegion.containsKey(key.toString()); + } + + public Set<String> metaKeySet() { + return this.redisMetaRegion.keySet(); + } + + public Set<Map.Entry<String, RedisDataType>> metaEntrySet() { + return this.redisMetaRegion.entrySet(); + } + + public int getMetaSize() { + return this.redisMetaRegion.size() - RedisConstants.NUM_DEFAULT_KEYS; + } + + private boolean metaRemoveEntry(ByteArrayWrapper key) { + return this.redisMetaRegion.remove(key.toString()) != null; + } + + public RedisDataType metaPutIfAbsent(ByteArrayWrapper key, RedisDataType value) { + return this.redisMetaRegion.putIfAbsent(key.toString(), value); + } + + public RedisDataType metaPut(ByteArrayWrapper key, RedisDataType value) { + return this.redisMetaRegion.put(key.toString(), value); + } + + public RedisDataType metaGet(ByteArrayWrapper key) { + return this.redisMetaRegion.get(key.toString()); + } + + public Region<?, ?> getRegion(ByteArrayWrapper key) { + return this.regions.get(key); + } + + public void removeRegionReferenceLocally(ByteArrayWrapper key, RedisDataType type) { + Lock lock = this.locks.get(key.toString()); + boolean locked = false; + try { + locked = lock.tryLock(); + // If we cannot get the lock we ignore this remote event, this key has local event + // that started independently, ignore this event to prevent deadlock + if (locked) { + cancelKeyExpiration(key); + removeRegionState(key, type); + } + } finally { + if (locked) { + lock.unlock(); + } + } + } + + public boolean removeKey(ByteArrayWrapper key) { + RedisDataType type = getRedisDataType(key); + return removeKey(key, type); + } + + public boolean removeKey(ByteArrayWrapper key, RedisDataType type) { + return removeKey(key, type, true); + } + + public boolean removeKey(ByteArrayWrapper key, RedisDataType type, boolean cancelExpiration) { + if (type == null || type == RedisDataType.REDIS_PROTECTED) + return false; + Lock lock = this.locks.get(key.toString()); + try { + if (lock != null) {// Strings/hlls will not have locks + lock.lock(); + } + metaRemoveEntry(key); + try { + if (type == RedisDataType.REDIS_STRING) { + return this.stringsRegion.remove(key) != null; + } else if (type == RedisDataType.REDIS_HLL) { + return this.hLLRegion.remove(key) != null; + } else { + return destroyRegion(key, type); + } + } catch (Exception exc) { + return false; + } finally { + if (cancelExpiration) + cancelKeyExpiration(key); + else + removeKeyExpiration(key); + if (lock != null) + this.locks.remove(key.toString()); + } + } finally { + if (lock != null) { + lock.unlock(); + } + } + } + + public Region<?, ?> getOrCreateRegion(ByteArrayWrapper key, RedisDataType type, ExecutionHandlerContext context) { + return getOrCreateRegion0(key, type, context, true); + } + + public void createRemoteRegionLocally(ByteArrayWrapper key, RedisDataType type) { + if (type == null || type == RedisDataType.REDIS_STRING || type == RedisDataType.REDIS_HLL) + return; + Region<?, ?> r = this.regions.get(key); + if (r != null) + return; + if (!this.regions.contains(key)) { + String stringKey = key.toString(); + Lock lock = this.locks.get(stringKey); + if (lock == null) { + this.locks.putIfAbsent(stringKey, new ReentrantLock()); + lock = this.locks.get(stringKey); + } + boolean locked = false; + try { + locked = lock.tryLock(); + // If we cannot get the lock then this remote even may have been initialized + // independently on this machine, so if we wait on the lock it is more than + // likely we will deadlock just to do the same task, this even can be ignored + if (locked) { + r = cache.getRegion(key.toString()); + if (type == RedisDataType.REDIS_LIST) + doInitializeList(key, r); + else if (type == RedisDataType.REDIS_SORTEDSET) + doInitializeSortedSet(key, r); + this.regions.put(key, r); + } + } finally { + if (locked) { + lock.unlock(); + } + } + } + } + + private Region<?, ?> getOrCreateRegion0(ByteArrayWrapper key, RedisDataType type, ExecutionHandlerContext context, boolean addToMeta) { + checkDataType(key, type); + Region<?, ?> r = this.regions.get(key); + if (r != null && r.isDestroyed()) { + removeKey(key, type); + r = null; + } + if (r == null) { + String stringKey = key.toString(); + Lock lock = this.locks.get(stringKey); + if (lock == null) { + this.locks.putIfAbsent(stringKey, new ReentrantLock()); + lock = this.locks.get(stringKey); + } + + try { + lock.lock(); + r = regions.get(key); + if (r == null) { + boolean hasTransaction = context != null && context.hasTransaction(); // Can create without context + CacheTransactionManager txm = null; + TransactionId transactionId = null; + try { + if (hasTransaction) { + txm = cache.getCacheTransactionManager(); + transactionId = txm.suspend(); + } + Exception concurrentCreateDestroyException = null; + do { + concurrentCreateDestroyException = null; + r = createRegionGlobally(stringKey); + try { + if (type == RedisDataType.REDIS_LIST) + doInitializeList(key, r); + else if (type == RedisDataType.REDIS_SORTEDSET) + doInitializeSortedSet(key, r); + } catch (QueryInvalidException e) { + if (e.getCause() instanceof RegionNotFoundException) { + concurrentCreateDestroyException = e; + } + } + } while(concurrentCreateDestroyException != null); + this.regions.put(key, r); + if (addToMeta) { + RedisDataType existingType = metaPutIfAbsent(key, type); + if (existingType != null && existingType != type) + throw new RedisDataTypeMismatchException("The key name \"" + key + "\" is already used by a " + existingType.toString()); + } + } finally { + if (hasTransaction) + txm.resume(transactionId); + } + } + } finally { + lock.unlock(); + } + } + return r; + } + + /** + * SYNCHRONIZE EXTERNALLY OF this.locks.get(key.toString())!!!!! + * + * @param key Key of region to destroy + * @param type Type of region to destroyu + * @return Flag if destroyed + */ + private boolean destroyRegion(ByteArrayWrapper key, RedisDataType type) { + Region<?, ?> r = this.regions.get(key); + if (r != null) { + try { + r.destroyRegion(); + } catch (Exception e) { + return false; + } finally { + removeRegionState(key, type); + } + } + return true; + } + + /** + * Do not call this method if you are not synchronized on the lock associated with this key + * + * @param key Key of region to remove + * @param type Type of key to remove all state + */ + private void removeRegionState(ByteArrayWrapper key, RedisDataType type) { + this.preparedQueries.remove(key); + this.regions.remove(key); + } + + private void doInitializeSortedSet(ByteArrayWrapper key, Region<?, ?> r) { + String fullpath = r.getFullPath(); + try { + queryService.createIndex("scoreIndex", "entry.value.score", r.getFullPath() + ".entrySet entry"); + queryService.createIndex("scoreIndex2", "value.score", r.getFullPath() + ".values value"); + } catch (Exception e) { + if (!(e instanceof IndexNameConflictException)) { + if (logger.errorEnabled()) { + logger.error(e); + } + } + } + HashMap<Enum<?>, Query> queryList = new HashMap<Enum<?>, Query>(); + for (SortedSetQuery lq: SortedSetQuery.values()) { + String queryString = lq.getQueryString(fullpath); + Query query = this.queryService.newQuery(queryString); + queryList.put(lq, query); + } + this.preparedQueries.put(key, queryList); + } + + private void doInitializeList(ByteArrayWrapper key, Region r) { + r.put("head", Integer.valueOf(0)); + r.put("tail", Integer.valueOf(0)); + String fullpath = r.getFullPath(); + HashMap<Enum<?>, Query> queryList = new HashMap<Enum<?>, Query>(); + for (ListQuery lq: ListQuery.values()) { + String queryString = lq.getQueryString(fullpath); + Query query = this.queryService.newQuery(queryString); + queryList.put(lq, query); + } + this.preparedQueries.put(key, queryList); + } + + /** + * This method creates a Region globally with the given name. If + * there is an error in the creation, a runtime exception will + * be thrown. + * + * @param key Name of Region to create + * @return Region Region created globally + */ + private Region<?, ?> createRegionGlobally(String key) { + Region<?, ?> r = null; + r = cache.getRegion(key); + if (r != null) return r; + do { - Result result = cliCmds.createRegion(key, defaultRegionType, null, null, true, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); ++ Result result = cliCmds.createRegion(key, defaultRegionType, null, null, true, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); + r = cache.getRegion(key); + if (result.getStatus() == Status.ERROR && r == null) { + String err = ""; + while(result.hasNextLine()) + err += result.nextLine(); + if (this.logger.errorEnabled()) { + this.logger.error("Region creation failure- "+ err); + } + throw new RegionCreationException(err); + } + } while(r == null); // The region can be null in the case that it is concurrently destroyed by + // a remote even triggered internally by Geode + return r; + } + + public Query getQuery(ByteArrayWrapper key, Enum<?> query) { + return this.preparedQueries.get(key).get(query); + /* + if (query instanceof ListQuery) { + return this.queryService.newQuery(((ListQuery)query).getQueryString(this.regions.get(key).getFullPath())); + } else { + return this.queryService.newQuery(((SortedSetQuery)query).getQueryString(this.regions.get(key).getFullPath())); + } + */ + } + + /** + * Checks if the given key is associated with the passed data type. + * If there is a mismatch, a {@link RuntimeException} is thrown + * + * @param key Key to check + * @param type Type to check to + */ + protected void checkDataType(ByteArrayWrapper key, RedisDataType type) { + RedisDataType currentType = redisMetaRegion.get(key.toString()); + if (currentType == null) + return; + if (currentType == RedisDataType.REDIS_PROTECTED) + throw new RedisDataTypeMismatchException("The key name \"" + key + "\" is protected"); + if (currentType != type) + throw new RedisDataTypeMismatchException("The key name \"" + key + "\" is already used by a " + currentType.toString()); + } + + public boolean regionExists(ByteArrayWrapper key) { + return this.regions.containsKey(key); + } + + public Region<ByteArrayWrapper, ByteArrayWrapper> getStringsRegion() { + return this.stringsRegion; + } + + public Region<ByteArrayWrapper, HyperLogLogPlus> gethLLRegion() { + return this.hLLRegion; + } + + private RedisDataType getRedisDataType(String key) { + return this.redisMetaRegion.get(key); + } + + public RedisDataType getRedisDataType(ByteArrayWrapper key) { + return getRedisDataType(key.toString()); + } + + /** + * Sets the expiration for a key. The setting and modifying of a key expiration can only be set by a delay, + * which means that both expiring after a time and at a time can be done but + * the delay to expire at a time must be calculated before these calls. It is + * also important to note that the delay is always handled in milliseconds + * + * @param key The key to set the expiration for + * @param delay The delay in milliseconds of the expiration + * @return True is expiration set, false otherwise + */ + public final boolean setExpiration(ByteArrayWrapper key, long delay) { + RedisDataType type = getRedisDataType(key); + if (type == null) + return false; + ScheduledFuture<?> future = this.expirationExecutor.schedule(new ExpirationExecutor(key, type, this), delay, TimeUnit.MILLISECONDS); + this.expirationsMap.put(key, future); + return true; + } + + /** + * Modifies an expiration on a key + * + * @param key String key to modify expiration on + * @param delay Delay in milliseconds to reset the expiration to + * @return True if reset, false if not + */ + public final boolean modifyExpiration(ByteArrayWrapper key, long delay) { + /* + * Attempt to cancel future task + */ + boolean canceled = cancelKeyExpiration(key); + + if (!canceled) + return false; + + RedisDataType type = getRedisDataType(key); + if (type == null) + return false; + + ScheduledFuture<?> future = this.expirationExecutor.schedule(new ExpirationExecutor(key, type, this), delay, TimeUnit.MILLISECONDS); + this.expirationsMap.put(key, future); + return true; + } + + /** + * Removes an expiration from a key + * + * @param key Key + * @param context Context + * @return True is expiration cancelled on the key, false otherwise + */ + public final boolean cancelKeyExpiration(ByteArrayWrapper key) { + ScheduledFuture<?> future = expirationsMap.remove(key); + if (future == null) + return false; + return future.cancel(false); + } + + private boolean removeKeyExpiration(ByteArrayWrapper key) { + return expirationsMap.remove(key) != null; + } + + /** + * Check method if key has expiration + * + * @param key Key + * @return True if key has expiration, false otherwise + */ + public boolean hasExpiration(ByteArrayWrapper key) { + return this.expirationsMap.containsKey(key); + } + + /** + * Get remaining expiration time + * + * @param key Key + * @return Remaining time in milliseconds or 0 if no delay or key doesn't exist + */ + public final long getExpirationDelayMillis(ByteArrayWrapper key) { + ScheduledFuture<?> future = this.expirationsMap.get(key); + return future != null ? future.getDelay(TimeUnit.MILLISECONDS) : 0L; + } + + @Override + public void close() { + this.preparedQueries.clear(); + } + + public String dumpRegionsCache() { + StringBuilder builder = new StringBuilder(); + for (Entry<ByteArrayWrapper, Region<?, ?>> e : this.regions.entrySet()) { + builder.append(e.getKey() + " --> {" + e.getValue() + "}\n"); + } + return builder.toString(); + } + + } http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/e0d1c4f9/gemfire-core/src/main/java/com/gemstone/gemfire/management/internal/cli/i18n/CliStrings.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/e0d1c4f9/gemfire-core/src/test/java/com/gemstone/gemfire/cache/management/MemoryThresholdsOffHeapDUnitTest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/e0d1c4f9/gemfire-core/src/test/java/com/gemstone/gemfire/cache/query/internal/index/CopyOnReadIndexDUnitTest.java ---------------------------------------------------------------------- diff --cc gemfire-core/src/test/java/com/gemstone/gemfire/cache/query/internal/index/CopyOnReadIndexDUnitTest.java index e66c047,61bb55d..5ad8537 --- a/gemfire-core/src/test/java/com/gemstone/gemfire/cache/query/internal/index/CopyOnReadIndexDUnitTest.java +++ b/gemfire-core/src/test/java/com/gemstone/gemfire/cache/query/internal/index/CopyOnReadIndexDUnitTest.java @@@ -598,12 -601,25 +600,24 @@@ public class CopyOnReadIndexDUnitTest e return p; } - protected Properties getServerProperties(int mcastPort) { + protected Properties getServerProperties() { Properties p = new Properties(); - p.setProperty(DistributionConfig.MCAST_PORT_NAME, mcastPort+""); - p.setProperty(DistributionConfig.LOCATORS_NAME, ""); + p.setProperty(DistributionConfig.LOCATORS_NAME, "localhost["+getDUnitLocatorPort()+"]"); return p; } - + + private WaitCriterion verifyPortfolioCount(final int expected) { + return new WaitCriterion() { + private int expectedCount = expected; + public boolean done() { + return expectedCount == Portfolio.instanceCount.get(); + } + + public String description() { + return "verifying number of object instances created"; + } + }; + } } http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/e0d1c4f9/gemfire-core/src/test/java/com/gemstone/gemfire/internal/cache/ClientServerTransactionDUnitTest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/e0d1c4f9/gemfire-core/src/test/java/dunit/DistributedTestCase.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/e0d1c4f9/settings.gradle ----------------------------------------------------------------------
