Updated Branches: refs/heads/0.2-dev 6cb3402f2 -> de4fece0b
Added auto configuration of cache based on the maxdirectmemory setting. If the slab count is left to the default of -1 the maxdirectmemory value is used to the number of slabs to be allocated. So if the setting is -XX:MaxDirectMemory=1100m then the number slabs would be set automatically on startup to 8 (8 128MB slabs of block cache memory). NOTE: Whatever the value set is to, a default of 64MB is leftover to allow for any other JVM direct memory needs. This is the default amount the JVM allows when max direct memory is not set. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/de4fece0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/de4fece0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/de4fece0 Branch: refs/heads/0.2-dev Commit: de4fece0beea4496daf58d5ebf4a6a3faae8cfe7 Parents: b298c08 Author: Aaron McCurry <[email protected]> Authored: Thu Mar 7 21:47:16 2013 -0500 Committer: Aaron McCurry <[email protected]> Committed: Thu Mar 7 21:47:16 2013 -0500 ---------------------------------------------------------------------- .../indexserver/DistributedIndexServer.java | 8 ++- .../org/apache/blur/thrift/ThriftBlurServer.java | 43 ++++++++++++++- .../src/main/resources/blur-default.properties | 2 +- 3 files changed, 47 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/de4fece0/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java b/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java index a3fa7cb..aa5d7a8 100644 --- a/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java +++ b/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java @@ -474,8 +474,12 @@ public class DistributedIndexServer extends AbstractIndexServer { directory.setLockFactory(lockFactory); Directory dir = new BlockDirectory(table + "_" + shard, directory, _cache); - // Directory dir = shardContext.getDirectory(); - // dir.setLockFactory(lockFactory); +// URI uri = hdfsDirPath.toUri(); +// File dirFile = new File(uri); +// FSDirectory dir = MMapDirectory.open(dirFile,lockFactory); + +// Directory dir = shardContext.getDirectory(); +// dir.setLockFactory(lockFactory); BlurIndex index; if (tableDescriptor.isReadOnly()) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/de4fece0/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java b/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java index a54a6e9..13a37a9 100644 --- a/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java +++ b/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java @@ -37,6 +37,7 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.lang.management.ManagementFactory; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -70,7 +71,6 @@ import org.apache.blur.utils.BlurUtil; import org.apache.blur.zookeeper.ZkUtils; import org.apache.hadoop.conf.Configuration; import org.apache.thrift.protocol.TJSONProtocol; -import org.apache.thrift.protocol.TProtocolFactory; import org.apache.thrift.server.TServlet; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException.Code; @@ -120,7 +120,8 @@ public class ThriftBlurServer extends AbstractThriftServer { // in a slab when using a block size of 8,192 int numberOfBlocksPerSlab = 16384; int blockSize = BlockDirectory.BLOCK_SIZE; - int slabCount = configuration.getInt(BLUR_SHARD_BLOCKCACHE_SLAB_COUNT, 1); + int slabCount = configuration.getInt(BLUR_SHARD_BLOCKCACHE_SLAB_COUNT, -1); + slabCount = getSlabCount(slabCount, numberOfBlocksPerSlab, blockSize); Cache cache; Configuration config = new Configuration(); @@ -249,7 +250,7 @@ public class ThriftBlurServer extends AbstractThriftServer { int threadCount = configuration.getInt(BLUR_SHARD_SERVER_THRIFT_THREAD_COUNT, 32); Processor<Iface> processor = new Processor<Iface>(iface); - + if (context != null) { context.addServlet(new ServletHolder(new TServlet(processor, new TJSONProtocol.Factory())), "/rpc"); } @@ -275,6 +276,42 @@ public class ThriftBlurServer extends AbstractThriftServer { return server; } + private static int getSlabCount(int slabCount, int numberOfBlocksPerSlab, int blockSize) { + if (slabCount < 1) { + // -XX:MaxDirectMemorySize + + long slabSize = numberOfBlocksPerSlab * blockSize; + + List<String> inputArguments = ManagementFactory.getRuntimeMXBean().getInputArguments(); + for (String arg : inputArguments) { + if (arg.startsWith("-XX:MaxDirectMemorySize")) { + long maxDirectMemorySize = getMaxDirectMemorySize(arg); + maxDirectMemorySize -= 64 * 1024 * 1024; + return (int) (maxDirectMemorySize / slabSize); + } + } + + System.out.println(inputArguments); + + throw new RuntimeException(); + } + return 0; + } + + private static long getMaxDirectMemorySize(String arg) { + int index = arg.lastIndexOf('='); + return parseNumber(arg.substring(index + 1).toLowerCase().replace(" ", "")); + } + + private static long parseNumber(String number) { + if (number.endsWith("m")) { + return Long.parseLong(number.substring(0, number.length() - 1)) * 1024 * 1024; + } else if (number.endsWith("g")) { + return Long.parseLong(number.substring(0, number.length() - 1)) * 1024 * 1024 * 1024; + } + throw new RuntimeException("Cannot parse [" + number + "]"); + } + private static BlurFilterCache getFilterCache(BlurConfiguration configuration) { String _blurFilterCacheClass = configuration.get(BLUR_SHARD_FILTER_CACHE_CLASS); if (_blurFilterCacheClass != null) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/de4fece0/src/blur-util/src/main/resources/blur-default.properties ---------------------------------------------------------------------- diff --git a/src/blur-util/src/main/resources/blur-default.properties b/src/blur-util/src/main/resources/blur-default.properties index 9c40e54..c7a740d 100644 --- a/src/blur-util/src/main/resources/blur-default.properties +++ b/src/blur-util/src/main/resources/blur-default.properties @@ -36,7 +36,7 @@ blur.shard.cache.max.timetolive=60000 blur.shard.filter.cache.class=org.apache.blur.manager.DefaultBlurFilterCache blur.shard.index.warmup.class=org.apache.blur.manager.indexserver.DefaultBlurIndexWarmup blur.shard.blockcache.direct.memory.allocation=true -blur.shard.blockcache.slab.count=1 +blur.shard.blockcache.slab.count=-1 blur.shard.buffercache.1024=8192 blur.shard.buffercache.8192=8192 blur.shard.safemodedelay=30000
