Updated Branches: refs/heads/master 969c2e171 -> 745397cec
Fixed BLUR-157 Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/745397ce Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/745397ce Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/745397ce Branch: refs/heads/master Commit: 745397cec5a49272c628a4be995bf349805214a3 Parents: 969c2e1 Author: Aaron McCurry <[email protected]> Authored: Wed Jun 26 15:57:08 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Wed Jun 26 15:57:08 2013 -0400 ---------------------------------------------------------------------- .../indexserver/DistributedIndexServer.java | 45 +++++++++++++++----- .../blur/manager/writer/BlurNRTIndex.java | 2 +- .../blur/thrift/ThriftBlurShardServer.java | 4 ++ .../org/apache/blur/utils/BlurConstants.java | 2 + .../src/main/resources/blur-default.properties | 2 + 5 files changed, 43 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/745397ce/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java index 85c1810..46033e5 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java +++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java @@ -135,6 +135,9 @@ public class DistributedIndexServer extends AbstractIndexServer { private AtomicLong _indexMemoryUsage = new AtomicLong(); private BlurIndexRefresher _refresher; private BlurIndexCloser _indexCloser; + private int _internalSearchThreads; + private ExecutorService _warmupExecutor; + private int _warmupThreads; public static interface ReleaseReader { void release() throws IOException; @@ -158,7 +161,8 @@ public class DistributedIndexServer extends AbstractIndexServer { // @TODO allow for configuration of these _mergeScheduler = new SharedMergeScheduler(); - _searchExecutor = Executors.newThreadPool("internal-search", 16); + _searchExecutor = Executors.newThreadPool("internal-search", _internalSearchThreads); + _warmupExecutor = Executors.newThreadPool("warmup", _warmupThreads); _closer = new IndexInputCloser(); _closer.init(); _refresher = new BlurIndexRefresher(); @@ -372,7 +376,6 @@ public class DistributedIndexServer extends AbstractIndexServer { @Override public void close() { if (_running.get()) { - _shardStateManager.close(); _running.set(false); closeAllIndexes(); @@ -387,6 +390,8 @@ public class DistributedIndexServer extends AbstractIndexServer { _closer.close(); _gc.close(); _openerService.shutdownNow(); + _searchExecutor.shutdownNow(); + _warmupExecutor.shutdownNow(); } } @@ -478,20 +483,30 @@ public class DistributedIndexServer extends AbstractIndexServer { } _filterCache.opening(table, shard, index); TableDescriptor tableDescriptor = _clusterStatus.getTableDescriptor(true, _cluster, table); - return warmUp(index, tableDescriptor, shard); + warmUp(index, tableDescriptor, shard); + return index; } - private BlurIndex warmUp(BlurIndex index, TableDescriptor table, String shard) throws IOException { - final IndexSearcherClosable searcher = index.getIndexReader(); - IndexReader reader = searcher.getIndexReader(); - _warmup.warmBlurIndex(table, shard, reader, index.isClosed(), new ReleaseReader() { + private void warmUp(final BlurIndex index, final TableDescriptor table, final String shard) throws IOException { + _warmupExecutor.submit(new Runnable() { @Override - public void release() throws IOException { - // this will allow for closing of index - searcher.close(); + public void run() { + try { + final IndexSearcherClosable searcher = index.getIndexReader(); + IndexReader reader = searcher.getIndexReader(); + _warmup.warmBlurIndex(table, shard, reader, index.isClosed(), new ReleaseReader() { + @Override + public void release() throws IOException { + // this will allow for closing of index + searcher.close(); + } + }); + } catch (Exception e) { + LOG.error("Unknown error while trying to warmup index [" + index + "]", e); + } } }); - return index; + } private synchronized Map<String, BlurIndex> openMissingShards(final String table, Set<String> shardsToServe, @@ -742,4 +757,12 @@ public class DistributedIndexServer extends AbstractIndexServer { public void setClusterName(String cluster) { _cluster = cluster; } + + public void setInternalSearchThreads(int internalSearchThreads) { + _internalSearchThreads = internalSearchThreads; + } + + public void setWarmupThreads(int warmupThreads) { + _warmupThreads = warmupThreads; + } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/745397ce/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java index 4b0bb20..c53af2e 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java @@ -182,7 +182,7 @@ public class BlurNRTIndex extends BlurIndex { _refresher.close(); try { _recorder.close(); - _writer.close(); + _writer.close(false); getNRTManager().close(); } finally { _directory.close(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/745397ce/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java index cd3188c..1d89ea2 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java @@ -30,9 +30,11 @@ import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCKCACHE_SLAB_COU import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_FILTER_CACHE_CLASS; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_HOSTNAME; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INDEX_WARMUP_CLASS; +import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INTERNAL_SEARCH_THREAD_COUNT; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_OPENER_THREAD_COUNT; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_SAFEMODEDELAY; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_SERVER_THRIFT_THREAD_COUNT; +import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_WARMUP_THREAD_COUNT; import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_CONNECTION; import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT; import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT_DEFAULT; @@ -187,6 +189,8 @@ public class ThriftBlurShardServer extends ThriftServer { indexServer.setConfiguration(config); indexServer.setNodeName(nodeName); indexServer.setShardOpenerThreadCount(configuration.getInt(BLUR_SHARD_OPENER_THREAD_COUNT, 16)); + indexServer.setWarmupThreads(configuration.getInt(BLUR_SHARD_WARMUP_THREAD_COUNT, 16)); + indexServer.setInternalSearchThreads(configuration.getInt(BLUR_SHARD_INTERNAL_SEARCH_THREAD_COUNT, 16)); indexServer.setZookeeper(zooKeeper); indexServer.setFilterCache(filterCache); indexServer.setSafeModeDelay(configuration.getLong(BLUR_SHARD_SAFEMODEDELAY, 60000)); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/745397ce/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java ---------------------------------------------------------------------- diff --git a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java index 28ee666..de2bcf8 100644 --- a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java +++ b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java @@ -59,6 +59,8 @@ public class BlurConstants { public static final String BLUR_SHARD_INDEX_WARMUP_CLASS = "blur.shard.index.warmup.class"; public static final String BLUR_INDEXMANAGER_SEARCH_THREAD_COUNT = "blur.indexmanager.search.thread.count"; public static final String BLUR_SHARD_DATA_FETCH_THREAD_COUNT = "blur.shard.data.fetch.thread.count"; + public static final String BLUR_SHARD_INTERNAL_SEARCH_THREAD_COUNT = "blur.shard.internal.search.thread.count"; + public static final String BLUR_SHARD_WARMUP_THREAD_COUNT = "blur.shard.warmup.thread.count"; public static final String BLUR_MAX_CLAUSE_COUNT = "blur.max.clause.count"; public static final String BLUR_SHARD_CACHE_MAX_QUERYCACHE_ELEMENTS = "blur.shard.cache.max.querycache.elements"; public static final String BLUR_SHARD_OPENER_THREAD_COUNT = "blur.shard.opener.thread.count"; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/745397ce/blur-util/src/main/resources/blur-default.properties ---------------------------------------------------------------------- diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties index 9a703d8..9187a05 100644 --- a/blur-util/src/main/resources/blur-default.properties +++ b/blur-util/src/main/resources/blur-default.properties @@ -32,6 +32,8 @@ blur.shard.time.between.commits=30000 blur.shard.time.between.refreshs=3000 blur.max.clause.count=1024 blur.indexmanager.search.thread.count=8 +blur.shard.internal.search.thread.count=8 +blur.shard.warmup.thread.count=8 blur.controller.hostname= blur.controller.bind.address=0.0.0.0
