Updated Branches: refs/heads/apache-blur-0.2 f2682d5b7 -> 8f6134c73
Passing table and shard into cache directory. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/09a7bc6d Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/09a7bc6d Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/09a7bc6d Branch: refs/heads/apache-blur-0.2 Commit: 09a7bc6d2a649b92f37224f2f920fd812787cb5a Parents: f2682d5 Author: Aaron McCurry <[email protected]> Authored: Sun Oct 20 21:27:56 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Sun Oct 20 21:27:56 2013 -0400 ---------------------------------------------------------------------- .../indexserver/DistributedIndexServer.java | 24 ++++++++++---------- .../blur/store/BlockCacheDirectoryFactory.java | 4 ++-- .../store/BlockCacheDirectoryFactoryV1.java | 5 ++-- .../store/BlockCacheDirectoryFactoryV2.java | 5 ++-- .../store/blockcache_v2/CacheDirectory.java | 6 ++--- .../blur/store/CacheDirectoryTestSuite.java | 12 +++++----- .../store/blockcache_v2/CacheDirectoryTest.java | 4 ++-- 7 files changed, 31 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/09a7bc6d/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java index ca826d1..d2f686f 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java +++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java @@ -120,8 +120,8 @@ public class DistributedIndexServer extends AbstractDistributedIndexServer { public DistributedIndexServer(Configuration configuration, ZooKeeper zookeeper, ClusterStatus clusterStatus, BlurIndexWarmup warmup, BlurFilterCache filterCache, BlockCacheDirectoryFactory blockCacheDirectoryFactory, DistributedLayoutFactory distributedLayoutFactory, String cluster, String nodeName, long safeModeDelay, - int shardOpenerThreadCount, int internalSearchThreads, int warmupThreads, int maxMergeThreads) throws KeeperException, - InterruptedException { + int shardOpenerThreadCount, int internalSearchThreads, int warmupThreads, int maxMergeThreads) + throws KeeperException, InterruptedException { super(clusterStatus, configuration, nodeName, cluster); _closer = Closer.create(); _shardOpenerThreadCount = shardOpenerThreadCount; @@ -133,23 +133,23 @@ public class DistributedIndexServer extends AbstractDistributedIndexServer { _warmupThreads = warmupThreads; _blockCacheDirectoryFactory = blockCacheDirectoryFactory; _distributedLayoutFactory = distributedLayoutFactory == null ? getDefaultLayoutFactory() : distributedLayoutFactory; - + _closer.register(_shardStateManager); BlurUtil.setupZookeeper(_zookeeper, _cluster); _openerService = Executors.newThreadPool("shard-opener", _shardOpenerThreadCount); _searchExecutor = Executors.newThreadPool("internal-search", _internalSearchThreads); _warmupExecutor = Executors.newThreadPool("warmup", _warmupThreads); - + _closer.register(CloseableExecutorService.close(_openerService)); _closer.register(CloseableExecutorService.close(_searchExecutor)); _closer.register(CloseableExecutorService.close(_warmupExecutor)); - + _gc = _closer.register(new DirectoryReferenceFileGC()); // @TODO allow for configuration of these _mergeScheduler = _closer.register(new SharedMergeScheduler(maxMergeThreads)); - + _refresher = _closer.register(new BlurIndexRefresher()); _indexCloser = _closer.register(new BlurIndexCloser()); _timerCacheFlush = setupFlushCacheTimer(); @@ -158,9 +158,9 @@ public class DistributedIndexServer extends AbstractDistributedIndexServer { String onlineShardsPath = ZookeeperPathConstants.getOnlineShardsPath(_cluster); String safemodePath = ZookeeperPathConstants.getSafemodePath(_cluster); - //Set the registerNode timeout value to zk sessionTimeout + {4} seconds - int registerNodeTimeOut = _zookeeper.getSessionTimeout()/1000 + 4; - + // Set the registerNode timeout value to zk sessionTimeout + {4} seconds + int registerNodeTimeOut = _zookeeper.getSessionTimeout() / 1000 + 4; + SafeMode safeMode = new SafeMode(_zookeeper, safemodePath, onlineShardsPath, TimeUnit.MILLISECONDS, _safeModeDelay, TimeUnit.SECONDS, registerNodeTimeOut); safeMode.registerNode(getNodeName(), BlurUtil.getVersion().getBytes()); @@ -169,7 +169,7 @@ public class DistributedIndexServer extends AbstractDistributedIndexServer { _timerTableWarmer = setupTableWarmer(); _watchOnlineShards = watchForShardServerChanges(); } - + @Override public void close() throws IOException { if (_running.get()) { @@ -456,7 +456,7 @@ public class DistributedIndexServer extends AbstractDistributedIndexServer { boolean blockCacheEnabled = _clusterStatus.isBlockCacheEnabled(_cluster, table); if (blockCacheEnabled) { Set<String> blockCacheFileTypes = _clusterStatus.getBlockCacheFileTypes(_cluster, table); - dir = _blockCacheDirectoryFactory.newDirectory(table + "_" + shard, directory, blockCacheFileTypes); + dir = _blockCacheDirectoryFactory.newDirectory(table, shard, directory, blockCacheFileTypes); } else { dir = directory; } @@ -575,7 +575,7 @@ public class DistributedIndexServer extends AbstractDistributedIndexServer { DistributedLayout layoutManager = _distributedLayoutFactory.createDistributedLayout(table, shardList, shardServerList, offlineShardServers); - + Map<String, String> layout = layoutManager.getLayout(); String nodeName = getNodeName(); Set<String> shardsToServeCache = new TreeSet<String>(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/09a7bc6d/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactory.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactory.java b/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactory.java index 4b10ced..a02d4cb 100644 --- a/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactory.java +++ b/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactory.java @@ -23,7 +23,7 @@ import org.apache.lucene.store.Directory; public abstract class BlockCacheDirectoryFactory { - public abstract Directory newDirectory(String name, Directory directory, Set<String> blockCacheFileTypes) - throws IOException; + public abstract Directory newDirectory(String table, String shard, Directory directory, + Set<String> blockCacheFileTypes) throws IOException; } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/09a7bc6d/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV1.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV1.java b/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV1.java index e4ad073..254b765 100644 --- a/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV1.java +++ b/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV1.java @@ -75,8 +75,9 @@ public class BlockCacheDirectoryFactoryV1 extends BlockCacheDirectoryFactory { } @Override - public Directory newDirectory(String name, Directory directory, Set<String> blockCacheFileTypes) throws IOException { - return new BlockDirectory(name, directory, _cache, blockCacheFileTypes); + public Directory newDirectory(String table, String shard, Directory directory, Set<String> blockCacheFileTypes) + throws IOException { + return new BlockDirectory(table + "_" + shard, directory, _cache, blockCacheFileTypes); } private static int getSlabCount(int slabCount, int numberOfBlocksPerSlab, int blockSize, long totalNumberOfBytes) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/09a7bc6d/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java b/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java index 6d1a47e..aff5823 100644 --- a/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java +++ b/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java @@ -90,8 +90,9 @@ public class BlockCacheDirectoryFactoryV2 extends BlockCacheDirectoryFactory { } @Override - public Directory newDirectory(String name, Directory directory, Set<String> blockCacheFileTypes) throws IOException { - return new CacheDirectory(name, directory, _cache); + public Directory newDirectory(String table, String shard, Directory directory, Set<String> blockCacheFileTypes) + throws IOException { + return new CacheDirectory(table, shard, directory, _cache); } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/09a7bc6d/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheDirectory.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheDirectory.java b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheDirectory.java index 73cc147..341c409 100644 --- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheDirectory.java +++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheDirectory.java @@ -35,12 +35,12 @@ public class CacheDirectory extends Directory implements DirectoryDecorator, Las private final String _directoryName; private final Cache _cache; - public CacheDirectory(String directoryName, Directory directory, Cache cache) { + public CacheDirectory(String table, String shard, Directory directory, Cache cache) { if (!(directory instanceof LastModified)) { throw new RuntimeException("Directory [" + directory + "] does not implement '" + LastModified.class.toString() + "'"); } - _directoryName = notNull(directoryName); + _directoryName = notNull(table + "_" + shard); _internal = notNull(directory); _cache = notNull(cache); } @@ -136,7 +136,7 @@ public class CacheDirectory extends Directory implements DirectoryDecorator, Las public Directory getOriginalDirectory() { return _internal; } - + private static <T> T notNull(T t) { if (t == null) { throw new IllegalArgumentException("Cannot be null"); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/09a7bc6d/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java ---------------------------------------------------------------------- diff --git a/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java b/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java index b03053e..38e448f 100644 --- a/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java +++ b/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java @@ -38,21 +38,21 @@ public abstract class CacheDirectoryTestSuite extends BaseDirectoryTestSuite { int totalNumberOfBytes = 1000000; final int fileBufferSizeInt = numberBetween(113, 215); final int cacheBlockSizeInt = numberBetween(111, 251); - + Size fileBufferSize = new Size() { @Override public int getSize(String directoryName, String fileName) { return fileBufferSizeInt; } }; - + Size cacheBlockSize = new Size() { @Override public int getSize(String directoryName, String fileName) { return cacheBlockSizeInt; } }; - + FileNameFilter writeFilter = new FileNameFilter() { @Override public boolean accept(String directoryName, String fileName) { @@ -71,16 +71,16 @@ public abstract class CacheDirectoryTestSuite extends BaseDirectoryTestSuite { return false; } }; - _cache = new BaseCache(totalNumberOfBytes, fileBufferSize, cacheBlockSize, readFilter, writeFilter,quiet, + _cache = new BaseCache(totalNumberOfBytes, fileBufferSize, cacheBlockSize, readFilter, writeFilter, quiet, getStore()); Directory dir = FSDirectory.open(new File(file, "cache")); BufferStore.init(128, 128); - return new CacheDirectory("test", wrapLastModified(dir), _cache); + return new CacheDirectory("test", "test", wrapLastModified(dir), _cache); } protected abstract STORE getStore(); - + public void close() throws IOException { _cache.close(); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/09a7bc6d/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryTest.java ---------------------------------------------------------------------- diff --git a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryTest.java b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryTest.java index 8867cff..b351fcf 100644 --- a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryTest.java +++ b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryTest.java @@ -91,9 +91,9 @@ public class CacheDirectoryTest { STORE.ON_HEAP); Directory directory = newDirectory(); BufferStore.init(128, 128); - _cacheDirectory = new CacheDirectory("test", directory, _cache); + _cacheDirectory = new CacheDirectory("test", "test", directory, _cache); } - + @After public void tearDown() throws IOException { _cache.close();
