Updated Branches: refs/heads/master 28470cd67 -> dc94a2ca9
Refactoring to allow for easier configuration. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/dc94a2ca Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/dc94a2ca Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/dc94a2ca Branch: refs/heads/master Commit: dc94a2ca99f4de0f40e7825556f7c1745e0f64fb Parents: 28470cd Author: Aaron McCurry <[email protected]> Authored: Thu Oct 10 14:16:15 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Thu Oct 10 14:16:32 2013 -0400 ---------------------------------------------------------------------- .../blur/thrift/ThriftBlurShardServer.java | 67 +++----------------- .../store/BlockCacheDirectoryFactoryV1.java | 58 ++++++++++++++++- .../store/BlockCacheDirectoryFactoryV2.java | 12 +++- .../org/apache/blur/utils/BlurConstants.java | 2 +- .../src/main/resources/blur-default.properties | 11 ++-- docs/cluster-setup.html | 17 ++--- 6 files changed, 88 insertions(+), 79 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dc94a2ca/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java index 82cf541..926c9e9 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java @@ -28,9 +28,7 @@ import static org.apache.blur.utils.BlurConstants.BLUR_MAX_HEAP_PER_ROW_FETCH; import static org.apache.blur.utils.BlurConstants.BLUR_MAX_RECORDS_PER_ROW_FETCH_REQUEST; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BIND_ADDRESS; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BIND_PORT; -import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCKCACHE_DIRECT_MEMORY_ALLOCATION; -import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCKCACHE_SLAB_COUNT; -import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_EXPERIMENTAL_BLOCK_CACHE; +import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_VERSION; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_FETCHCOUNT; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_FILTER_CACHE_CLASS; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_HOSTNAME; @@ -77,10 +75,6 @@ import org.apache.blur.server.TableContext; import org.apache.blur.store.BlockCacheDirectoryFactory; import org.apache.blur.store.BlockCacheDirectoryFactoryV1; import org.apache.blur.store.BlockCacheDirectoryFactoryV2; -import org.apache.blur.store.blockcache.BlockCache; -import org.apache.blur.store.blockcache.BlockDirectory; -import org.apache.blur.store.blockcache.BlockDirectoryCache; -import org.apache.blur.store.blockcache.Cache; import org.apache.blur.store.buffer.BufferStore; import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TJSONProtocol; import org.apache.blur.thirdparty.thrift_0_9_0.server.TServlet; @@ -157,48 +151,15 @@ public class ThriftBlurShardServer extends ThriftServer { BlockCacheDirectoryFactory blockCacheDirectoryFactory; // Alternate BlockCacheDirectoryFactory support currently disabled in 0.2.0, // look for it in 0.2.1 - boolean experimentalBlockCache = configuration.getBoolean(BLUR_SHARD_EXPERIMENTAL_BLOCK_CACHE, false); - experimentalBlockCache = false; - if (!experimentalBlockCache) { - // setup block cache - // 134,217,728 is the slab size, therefore there are 16,384 blocks - // in a slab when using a block size of 8,192 - int numberOfBlocksPerSlab = 16384; - int blockSize = BlockDirectory.BLOCK_SIZE; - int slabCount = configuration.getInt(BLUR_SHARD_BLOCKCACHE_SLAB_COUNT, -1); - slabCount = getSlabCount(slabCount, numberOfBlocksPerSlab, blockSize); - Cache cache; - if (slabCount >= 1) { - BlockCache blockCache; - boolean directAllocation = configuration.getBoolean(BLUR_SHARD_BLOCKCACHE_DIRECT_MEMORY_ALLOCATION, true); - - int slabSize = numberOfBlocksPerSlab * blockSize; - LOG.info("Number of slabs of block cache [{0}] with direct memory allocation set to [{1}]", slabCount, - directAllocation); - LOG.info("Block cache target memory usage, slab size of [{0}] will allocate [{1}] slabs and use ~[{2}] bytes", - slabSize, slabCount, ((long) slabCount * (long) slabSize)); - - try { - long totalMemory = (long) slabCount * (long) numberOfBlocksPerSlab * (long) blockSize; - blockCache = new BlockCache(directAllocation, totalMemory, slabSize); - } catch (OutOfMemoryError e) { - if ("Direct buffer memory".equals(e.getMessage())) { - System.err - .println("The max direct memory is too low. Either increase by setting (-XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages) or disable direct allocation by (blur.shard.blockcache.direct.memory.allocation=false) in blur-site.properties"); - System.exit(1); - } - throw e; - } - cache = new BlockDirectoryCache(blockCache); - } else { - cache = BlockDirectory.NO_CACHE; - } - blockCacheDirectoryFactory = new BlockCacheDirectoryFactoryV1(cache); - } else { - long totalNumberOfBytes = VM.maxDirectMemory() - _64MB; + String blockCacheVersion = configuration.get(BLUR_SHARD_BLOCK_CACHE_VERSION, "v2"); + long totalNumberOfBytes = VM.maxDirectMemory() - _64MB; + if (blockCacheVersion.equals("v1")) { + blockCacheDirectoryFactory = new BlockCacheDirectoryFactoryV1(configuration, totalNumberOfBytes); + } else if (blockCacheVersion.equals("v2")) { blockCacheDirectoryFactory = new BlockCacheDirectoryFactoryV2(configuration, totalNumberOfBytes); + } else { + throw new RuntimeException("Unknown block cache version [" + blockCacheVersion + "] can be [v1,v2]"); } - LOG.info("Shard Server using index [{0}] bind address [{1}] random port assignment [{2}]", serverIndex, bindAddress + ":" + bindPort, randomPort); @@ -296,18 +257,6 @@ public class ThriftBlurShardServer extends ThriftServer { return server; } - private static int getSlabCount(int slabCount, int numberOfBlocksPerSlab, int blockSize) { - if (slabCount < 0) { - long slabSize = numberOfBlocksPerSlab * blockSize; - long maxDirectMemorySize = VM.maxDirectMemory() - _64MB; - if (maxDirectMemorySize < slabSize) { - throw new RuntimeException("Auto slab setup cannot happen, JVM option -XX:MaxDirectMemorySize not set."); - } - return (int) (maxDirectMemorySize / slabSize); - } - return slabCount; - } - private static BlurFilterCache getFilterCache(BlurConfiguration configuration) { String blurFilterCacheClass = configuration.get(BLUR_SHARD_FILTER_CACHE_CLASS); if (blurFilterCacheClass != null) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dc94a2ca/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV1.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV1.java b/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV1.java index 182b494..8e9aa11 100644 --- a/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV1.java +++ b/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV1.java @@ -16,18 +16,61 @@ */ package org.apache.blur.store; +import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCKCACHE_DIRECT_MEMORY_ALLOCATION; +import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCKCACHE_SLAB_COUNT; + import java.io.IOException; import java.util.Set; +import org.apache.blur.BlurConfiguration; +import org.apache.blur.log.Log; +import org.apache.blur.log.LogFactory; +import org.apache.blur.store.blockcache.BlockCache; import org.apache.blur.store.blockcache.BlockDirectory; +import org.apache.blur.store.blockcache.BlockDirectoryCache; import org.apache.blur.store.blockcache.Cache; import org.apache.lucene.store.Directory; public class BlockCacheDirectoryFactoryV1 implements BlockCacheDirectoryFactory { + private static final Log LOG = LogFactory.getLog(BlockCacheDirectoryFactoryV1.class); + private final Cache _cache; - - public BlockCacheDirectoryFactoryV1(Cache cache) { + + public BlockCacheDirectoryFactoryV1(BlurConfiguration configuration, long totalNumberOfBytes) { + // setup block cache + // 134,217,728 is the slab size, therefore there are 16,384 blocks + // in a slab when using a block size of 8,192 + int numberOfBlocksPerSlab = 16384; + int blockSize = BlockDirectory.BLOCK_SIZE; + int slabCount = configuration.getInt(BLUR_SHARD_BLOCKCACHE_SLAB_COUNT, -1); + slabCount = getSlabCount(slabCount, numberOfBlocksPerSlab, blockSize, totalNumberOfBytes); + Cache cache; + if (slabCount >= 1) { + BlockCache blockCache; + boolean directAllocation = configuration.getBoolean(BLUR_SHARD_BLOCKCACHE_DIRECT_MEMORY_ALLOCATION, true); + + int slabSize = numberOfBlocksPerSlab * blockSize; + LOG.info("Number of slabs of block cache [{0}] with direct memory allocation set to [{1}]", slabCount, + directAllocation); + LOG.info("Block cache target memory usage, slab size of [{0}] will allocate [{1}] slabs and use ~[{2}] bytes", + slabSize, slabCount, ((long) slabCount * (long) slabSize)); + + try { + long totalMemory = (long) slabCount * (long) numberOfBlocksPerSlab * (long) blockSize; + blockCache = new BlockCache(directAllocation, totalMemory, slabSize); + } catch (OutOfMemoryError e) { + if ("Direct buffer memory".equals(e.getMessage())) { + System.err + .println("The max direct memory is too low. Either increase by setting (-XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages) or disable direct allocation by (blur.shard.blockcache.direct.memory.allocation=false) in blur-site.properties"); + System.exit(1); + } + throw e; + } + cache = new BlockDirectoryCache(blockCache); + } else { + cache = BlockDirectory.NO_CACHE; + } _cache = cache; } @@ -36,4 +79,15 @@ public class BlockCacheDirectoryFactoryV1 implements BlockCacheDirectoryFactory return new BlockDirectory(name, directory, _cache, blockCacheFileTypes); } + private static int getSlabCount(int slabCount, int numberOfBlocksPerSlab, int blockSize, long totalNumberOfBytes) { + if (slabCount < 0) { + long slabSize = numberOfBlocksPerSlab * blockSize; + if (totalNumberOfBytes < slabSize) { + throw new RuntimeException("Auto slab setup cannot happen, JVM option -XX:MaxDirectMemorySize not set."); + } + return (int) (totalNumberOfBytes / slabSize); + } + return slabCount; + } + } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dc94a2ca/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java b/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java index e3a7007..d61dd73 100644 --- a/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java +++ b/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java @@ -33,13 +33,17 @@ public class BlockCacheDirectoryFactoryV2 implements BlockCacheDirectoryFactory private Cache _cache; public BlockCacheDirectoryFactoryV2(BlurConfiguration configuration, long totalNumberOfBytes) { - int fileBufferSize = 8192; + final int fileBufferSize = 8192; + final int blockSize = 8192; + final STORE store = STORE.OFF_HEAP; + FileNameBlockSize fileNameBlockSize = new FileNameBlockSize() { @Override public int getBlockSize(String directoryName, String fileName) { - return 8192; + return blockSize; } }; + FileNameFilter readFilter = new FileNameFilter() { @Override public boolean accept(String directoryName, String fileName) { @@ -49,6 +53,7 @@ public class BlockCacheDirectoryFactoryV2 implements BlockCacheDirectoryFactory return true; } }; + FileNameFilter writeFilter = new FileNameFilter() { @Override public boolean accept(String directoryName, String fileName) { @@ -58,8 +63,9 @@ public class BlockCacheDirectoryFactoryV2 implements BlockCacheDirectoryFactory return true; } }; + _cache = new BaseCache(totalNumberOfBytes, fileBufferSize, fileNameBlockSize, readFilter, writeFilter, - STORE.OFF_HEAP); + store); } @Override http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dc94a2ca/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java ---------------------------------------------------------------------- diff --git a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java index 1373ad2..993208b 100644 --- a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java +++ b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java @@ -44,7 +44,7 @@ public class BlurConstants { public static final String BLUR_SHARD_BIND_ADDRESS = "blur.shard.bind.address"; public static final String BLUR_SHARD_BLOCKCACHE_DIRECT_MEMORY_ALLOCATION = "blur.shard.blockcache.direct.memory.allocation"; public static final String BLUR_SHARD_BLOCKCACHE_SLAB_COUNT = "blur.shard.blockcache.slab.count"; - public static final String BLUR_SHARD_EXPERIMENTAL_BLOCK_CACHE = "blur.shard.experimental.block.cache"; + public static final String BLUR_SHARD_BLOCK_CACHE_VERSION = "blur.shard.block.cache.version"; public static final String BLUR_SHARD_SAFEMODEDELAY = "blur.shard.safemodedelay"; public static final String BLUR_CONTROLLER_HOSTNAME = "blur.controller.hostname"; public static final String BLUR_CONTROLLER_BIND_PORT = "blur.controller.bind.port"; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dc94a2ca/blur-util/src/main/resources/blur-default.properties ---------------------------------------------------------------------- diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties index 48aaabf..922dd6d 100644 --- a/blur-util/src/main/resources/blur-default.properties +++ b/blur-util/src/main/resources/blur-default.properties @@ -72,14 +72,13 @@ blur.shard.index.warmup.class=org.apache.blur.manager.indexserver.DefaultBlurInd # Throttles the warmup to 30MB/s across all the warmup threads blur.shard.index.warmup.throttle=30000000 -# By default the block cache using off heap memory -blur.shard.blockcache.direct.memory.allocation=true +# By default the v2 version of the block cache is enabled +blur.shard.block.cache.version=v2 -# By default the experimental block cache is off -blur.shard.experimental.block.cache=false +# v1 version of block cache only. By default the block cache using off heap memory +blur.shard.blockcache.direct.memory.allocation=true -# The slabs in the blockcache are automatically configured by default (-1) otherwise 1 slab equals 128MB -# The auto config is detected through the MaxDirectoryMemorySize provided to the JVM +# v1 version of block cache only. The slabs in the blockcache are automatically configured by default (-1) otherwise 1 slab equals 128MB. The auto config is detected through the MaxDirectoryMemorySize provided to the JVM blur.shard.blockcache.slab.count=-1 # The number of 1K byte buffers http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dc94a2ca/docs/cluster-setup.html ---------------------------------------------------------------------- diff --git a/docs/cluster-setup.html b/docs/cluster-setup.html index 1c02ca7..ad5eb37 100644 --- a/docs/cluster-setup.html +++ b/docs/cluster-setup.html @@ -302,16 +302,17 @@ blur.shard.index.warmup.class=org.apache.blur.manager.indexserver.DefaultBlurInd # Throttles the warmup to 30MB/s across all the warmup threads blur.shard.index.warmup.throttle=30000000 -# By default the block cache using off heap memory -blur.shard.blockcache.direct.memory.allocation=true +# By default the v2 version of the block cache is enabled +blur.shard.block.cache.version=v2 -# By default the experimental block cache is off -blur.shard.experimental.block.cache=false +# v1 version of block cache only. By default the block cache using +# off heap memory +blur.shard.blockcache.direct.memory.allocation=true -# The slabs in the blockcache are automatically configured by -# default (-1) otherwise 1 slab equals 128MB. The auto config -# is detected through the MaxDirectoryMemorySize provided to -# the JVM +# v1 version of block cache only. The slabs in the blockcache are +# automatically configured by default (-1) otherwise 1 slab equals +# 128MB. The auto config is detected through the MaxDirectoryMemorySize +# provided to the JVM blur.shard.blockcache.slab.count=-1 # The number of 1K byte buffers
