Updated Branches: refs/heads/0.2-dev 80751ebab -> e3a319d98
Some fixes so that BufferStore doesn't double init. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/e3a319d9 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/e3a319d9 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/e3a319d9 Branch: refs/heads/0.2-dev Commit: e3a319d98c0203cd5c0d23f0ebf6b076cae05486 Parents: f3a69cb Author: Aaron McCurry <[email protected]> Authored: Wed Mar 6 11:28:24 2013 -0500 Committer: Aaron McCurry <[email protected]> Committed: Wed Mar 6 11:28:24 2013 -0500 ---------------------------------------------------------------------- .../org/apache/blur/thrift/ThriftBlurServer.java | 6 ++- .../org/apache/blur/store/buffer/BufferStore.java | 36 +++++++------- .../org/apache/blur/store/HdfsDirectoryTest.java | 2 + .../blur/store/blockcache/BlockDirectoryTest.java | 6 ++- 4 files changed, 29 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e3a319d9/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java b/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java index 0c4ebeb..7d8719a 100644 --- a/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java +++ b/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java @@ -127,7 +127,7 @@ public class ThriftBlurServer extends AbstractThriftServer { int bindPort = configuration.getInt(BLUR_SHARD_BIND_PORT, -1); bindPort += serverIndex; -// BlurMetrics blurMetrics = new BlurMetrics(config); + // BlurMetrics blurMetrics = new BlurMetrics(config); int baseGuiPort = Integer.parseInt(configuration.get(BLUR_GUI_SHARD_PORT)); final HttpJettyServer httpServer; @@ -151,7 +151,9 @@ public class ThriftBlurServer extends AbstractThriftServer { LOG.info("Number of slabs of block cache [{0}] with direct memory allocation set to [{1}]", slabCount, directAllocation); LOG.info("Block cache target memory usage, slab size of [{0}] will allocate [{1}] slabs and use ~[{2}] bytes", slabSize, slabCount, ((long) slabCount * (long) slabSize)); - BufferStore.init(configuration); + int _1024Size = configuration.getInt("blur.shard.buffercache.1024", 8192); + int _8192Size = configuration.getInt("blur.shard.buffercache.8192", 8192); + BufferStore.init(_1024Size, _8192Size); try { long totalMemory = (long) slabCount * (long) numberOfBlocksPerSlab * (long) blockSize; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e3a319d9/src/blur-store/src/main/java/org/apache/blur/store/buffer/BufferStore.java ---------------------------------------------------------------------- diff --git a/src/blur-store/src/main/java/org/apache/blur/store/buffer/BufferStore.java b/src/blur-store/src/main/java/org/apache/blur/store/buffer/BufferStore.java index 09e8d88..2701726 100644 --- a/src/blur-store/src/main/java/org/apache/blur/store/buffer/BufferStore.java +++ b/src/blur-store/src/main/java/org/apache/blur/store/buffer/BufferStore.java @@ -28,7 +28,6 @@ import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.TimeUnit; -import org.apache.blur.BlurConfiguration; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; @@ -38,34 +37,35 @@ import com.yammer.metrics.core.MetricName; public class BufferStore { - - private static final Log LOG = LogFactory.getLog(BufferStore.class); - private static BlockingQueue<byte[]> _1024 = setupBuffers(1024, 1); - private static BlockingQueue<byte[]> _8192 = setupBuffers(8192, 1); + private static BlockingQueue<byte[]> _1024; + private static BlockingQueue<byte[]> _8192; private static Meter _lost; private static Meter _8K; private static Meter _1K; private static Meter _other; - - public static void init(BlurConfiguration configuration) { - int _1024Size = configuration.getInt("blur.shard.buffercache.1024", 8192); - int _8192Size = configuration.getInt("blur.shard.buffercache.8192", 8192); - LOG.info("Initializing the 1024 buffers with [{0}] buffers.", _1024Size); - _1024 = setupBuffers(1024, _1024Size); - LOG.info("Initializing the 8192 buffers with [{0}] buffers.", _8192Size); - _8192 = setupBuffers(8192, _8192Size); - _lost = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, LUCENE, LOST, INTERNAL_BUFFERS), INTERNAL_BUFFERS, TimeUnit.SECONDS); - _8K = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, LUCENE, _1K_SIZE_ALLOCATED, INTERNAL_BUFFERS), INTERNAL_BUFFERS, TimeUnit.SECONDS); - _1K = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, LUCENE, _8K_SIZE_ALLOCATED, INTERNAL_BUFFERS), INTERNAL_BUFFERS, TimeUnit.SECONDS); - _other = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, LUCENE, OTHER_SIZES_ALLOCATED, INTERNAL_BUFFERS), INTERNAL_BUFFERS, TimeUnit.SECONDS); + private volatile static boolean setup = false; + + public static void init(int _1KSize, int _8KSize) { + if (!setup) { + _lost = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, LUCENE, LOST, INTERNAL_BUFFERS), INTERNAL_BUFFERS, TimeUnit.SECONDS); + _8K = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, LUCENE, _1K_SIZE_ALLOCATED, INTERNAL_BUFFERS), INTERNAL_BUFFERS, TimeUnit.SECONDS); + _1K = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, LUCENE, _8K_SIZE_ALLOCATED, INTERNAL_BUFFERS), INTERNAL_BUFFERS, TimeUnit.SECONDS); + _other = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, LUCENE, OTHER_SIZES_ALLOCATED, INTERNAL_BUFFERS), INTERNAL_BUFFERS, TimeUnit.SECONDS); + LOG.info("Initializing the 1024 buffers with [{0}] buffers.", _1KSize); + _1024 = setupBuffers(1024, _1KSize, _1K); + LOG.info("Initializing the 8192 buffers with [{0}] buffers.", _8KSize); + _8192 = setupBuffers(8192, _8KSize, _8K); + setup = true; + } } - private static BlockingQueue<byte[]> setupBuffers(int bufferSize, int count) { + private static BlockingQueue<byte[]> setupBuffers(int bufferSize, int count, Meter meter) { BlockingQueue<byte[]> queue = new ArrayBlockingQueue<byte[]>(count); for (int i = 0; i < count; i++) { + meter.mark(); queue.add(new byte[bufferSize]); } return queue; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e3a319d9/src/blur-store/src/test/java/org/apache/blur/store/HdfsDirectoryTest.java ---------------------------------------------------------------------- diff --git a/src/blur-store/src/test/java/org/apache/blur/store/HdfsDirectoryTest.java b/src/blur-store/src/test/java/org/apache/blur/store/HdfsDirectoryTest.java index f41daa9..fba2a42 100644 --- a/src/blur-store/src/test/java/org/apache/blur/store/HdfsDirectoryTest.java +++ b/src/blur-store/src/test/java/org/apache/blur/store/HdfsDirectoryTest.java @@ -28,6 +28,7 @@ import java.util.HashSet; import java.util.Random; import java.util.Set; +import org.apache.blur.store.buffer.BufferStore; import org.apache.blur.store.hdfs.HdfsDirectory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -55,6 +56,7 @@ public class HdfsDirectoryTest { @Before public void setUp() throws IOException { + BufferStore.init(128, 128); file = new File(TMPDIR, "hdfsdirectorytest"); rm(file); URI uri = new File(file, "hdfs").toURI(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e3a319d9/src/blur-store/src/test/java/org/apache/blur/store/blockcache/BlockDirectoryTest.java ---------------------------------------------------------------------- diff --git a/src/blur-store/src/test/java/org/apache/blur/store/blockcache/BlockDirectoryTest.java b/src/blur-store/src/test/java/org/apache/blur/store/blockcache/BlockDirectoryTest.java index c58e41a..f2ef236 100644 --- a/src/blur-store/src/test/java/org/apache/blur/store/blockcache/BlockDirectoryTest.java +++ b/src/blur-store/src/test/java/org/apache/blur/store/blockcache/BlockDirectoryTest.java @@ -24,8 +24,11 @@ import java.io.IOException; import java.util.Map; import java.util.Random; +import javax.jws.Oneway; + import org.apache.blur.store.blockcache.BlockDirectory; import org.apache.blur.store.blockcache.Cache; +import org.apache.blur.store.buffer.BufferStore; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.IOContext; @@ -100,9 +103,10 @@ public class BlockDirectoryTest { private long seed; private Random random; private MapperCache mapperCache; - + @Before public void setUp() throws IOException { + BufferStore.init(128, 128); file = new File(TMPDIR, "blockdirectorytest"); rm(file); file.mkdirs();
