Updated Branches: refs/heads/master b9637861a -> 11d4e14a5
Added a config to be able to set the number of merge threads per shard server. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/11d4e14a Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/11d4e14a Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/11d4e14a Branch: refs/heads/master Commit: 11d4e14a54a58500dc00df1cd9ba40e6e74a4e05 Parents: b963786 Author: Aaron McCurry <[email protected]> Authored: Sat Oct 19 20:39:40 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Sat Oct 19 20:40:08 2013 -0400 ---------------------------------------------------------------------- .../apache/blur/manager/indexserver/DistributedIndexServer.java | 5 ++--- .../org/apache/blur/manager/indexserver/LocalIndexServer.java | 2 +- .../org/apache/blur/manager/writer/SharedMergeScheduler.java | 3 +-- .../main/java/org/apache/blur/thrift/ThriftBlurShardServer.java | 5 +++-- .../org/apache/blur/manager/writer/BlurIndexReaderTest.java | 2 +- .../java/org/apache/blur/manager/writer/BlurNRTIndexTest.java | 2 +- .../src/main/java/org/apache/blur/utils/BlurConstants.java | 1 + blur-util/src/main/resources/blur-default.properties | 3 +++ 8 files changed, 13 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/11d4e14a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java index bf50852..ca826d1 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java +++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java @@ -120,7 +120,7 @@ public class DistributedIndexServer extends AbstractDistributedIndexServer { public DistributedIndexServer(Configuration configuration, ZooKeeper zookeeper, ClusterStatus clusterStatus, BlurIndexWarmup warmup, BlurFilterCache filterCache, BlockCacheDirectoryFactory blockCacheDirectoryFactory, DistributedLayoutFactory distributedLayoutFactory, String cluster, String nodeName, long safeModeDelay, - int shardOpenerThreadCount, int internalSearchThreads, int warmupThreads) throws KeeperException, + int shardOpenerThreadCount, int internalSearchThreads, int warmupThreads, int maxMergeThreads) throws KeeperException, InterruptedException { super(clusterStatus, configuration, nodeName, cluster); _closer = Closer.create(); @@ -148,8 +148,7 @@ public class DistributedIndexServer extends AbstractDistributedIndexServer { _gc = _closer.register(new DirectoryReferenceFileGC()); // @TODO allow for configuration of these - _mergeScheduler = _closer.register(new SharedMergeScheduler()); - + _mergeScheduler = _closer.register(new SharedMergeScheduler(maxMergeThreads)); _refresher = _closer.register(new BlurIndexRefresher()); _indexCloser = _closer.register(new BlurIndexCloser()); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/11d4e14a/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java index f9652ce..5e42c23 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java +++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java @@ -69,7 +69,7 @@ public class LocalIndexServer extends AbstractIndexServer { public LocalIndexServer(TableDescriptor tableDescriptor) throws IOException { _closer = Closer.create(); _tableContext = TableContext.create(tableDescriptor); - _mergeScheduler = _closer.register(new SharedMergeScheduler()); + _mergeScheduler = _closer.register(new SharedMergeScheduler(3)); _gc = _closer.register(new DirectoryReferenceFileGC()); _searchExecutor = Executors.newCachedThreadPool(); _closer.register(new CloseableExecutorService(_searchExecutor)); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/11d4e14a/blur-core/src/main/java/org/apache/blur/manager/writer/SharedMergeScheduler.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/SharedMergeScheduler.java b/blur-core/src/main/java/org/apache/blur/manager/writer/SharedMergeScheduler.java index e53f464..6fe8205 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/SharedMergeScheduler.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/SharedMergeScheduler.java @@ -41,8 +41,7 @@ public class SharedMergeScheduler implements Runnable, Closeable { private AtomicBoolean _running = new AtomicBoolean(true); private ExecutorService _service; - public SharedMergeScheduler() { - int threads = 3; + public SharedMergeScheduler(int threads) { _service = Executors.newThreadPool(SHARED_MERGE_SCHEDULER, threads, false); for (int i = 0; i < threads; i++) { _service.submit(this); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/11d4e14a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java index f02bd3e..ffed891 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java @@ -34,6 +34,7 @@ import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_FILTER_CACHE_CLASS; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_HOSTNAME; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INDEX_WARMUP_CLASS; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INDEX_WARMUP_THROTTLE; +import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_MERGE_THREAD_COUNT; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_OPENER_THREAD_COUNT; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_SAFEMODEDELAY; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_SERVER_THRIFT_THREAD_COUNT; @@ -190,10 +191,10 @@ public class ThriftBlurShardServer extends ThriftServer { int shardOpenerThreadCount = configuration.getInt(BLUR_SHARD_OPENER_THREAD_COUNT, 16); int internalSearchThreads = configuration.getInt(BLUR_SHARD_WARMUP_THREAD_COUNT, 16); int warmupThreads = configuration.getInt(BLUR_SHARD_WARMUP_THREAD_COUNT, 16); - + int maxMergeThreads = configuration.getInt(BLUR_SHARD_MERGE_THREAD_COUNT, 3); final DistributedIndexServer indexServer = new DistributedIndexServer(config, zooKeeper, clusterStatus, indexWarmup, filterCache, blockCacheDirectoryFactory, distributedLayoutFactory, cluster, nodeName, - safeModeDelay, shardOpenerThreadCount, internalSearchThreads, warmupThreads); + safeModeDelay, shardOpenerThreadCount, internalSearchThreads, warmupThreads, maxMergeThreads); final IndexManager indexManager = new IndexManager(); indexManager.setIndexServer(indexServer); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/11d4e14a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java index 4a63b75..5c80a9e 100644 --- a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java +++ b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java @@ -68,7 +68,7 @@ public class BlurIndexReaderTest { rm(base); base.mkdirs(); - mergeScheduler = new SharedMergeScheduler(); + mergeScheduler = new SharedMergeScheduler(1); gc = new DirectoryReferenceFileGC(); configuration = new Configuration(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/11d4e14a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurNRTIndexTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurNRTIndexTest.java b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurNRTIndexTest.java index cafcf46..9800eab 100644 --- a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurNRTIndexTest.java +++ b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurNRTIndexTest.java @@ -70,7 +70,7 @@ public class BlurNRTIndexTest { rm(base); base.mkdirs(); - mergeScheduler = new SharedMergeScheduler(); + mergeScheduler = new SharedMergeScheduler(1); gc = new DirectoryReferenceFileGC(); configuration = new Configuration(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/11d4e14a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java ---------------------------------------------------------------------- diff --git a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java index fb5a0e6..ebebdb5 100644 --- a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java +++ b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java @@ -69,6 +69,7 @@ public class BlurConstants { public static final String BLUR_MAX_CLAUSE_COUNT = "blur.max.clause.count"; public static final String BLUR_SHARD_CACHE_MAX_QUERYCACHE_ELEMENTS = "blur.shard.cache.max.querycache.elements"; public static final String BLUR_SHARD_OPENER_THREAD_COUNT = "blur.shard.opener.thread.count"; + public static final String BLUR_SHARD_MERGE_THREAD_COUNT = "blur.shard.merge.thread.count"; public static final String BLUR_SHARD_INDEX_DELETION_POLICY_MAXAGE = "blur.shard.index.deletion.policy.maxage"; public static final String BLUR_ZOOKEEPER_SYSTEM_TIME_TOLERANCE = "blur.zookeeper.system.time.tolerance"; public static final String BLUR_SHARD_INDEX_SIMILARITY = "blur.shard.index.similarity"; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/11d4e14a/blur-util/src/main/resources/blur-default.properties ---------------------------------------------------------------------- diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties index 922dd6d..fdae2e7 100644 --- a/blur-util/src/main/resources/blur-default.properties +++ b/blur-util/src/main/resources/blur-default.properties @@ -96,6 +96,9 @@ blur.shard.time.between.commits=30000 # The default time between index refreshs blur.shard.time.between.refreshs=3000 +# The max number of threads used during index merges +blur.shard.merge.thread.count=3 + # The maximum number of clauses in a BooleanQuery blur.max.clause.count=1024
