Updated Branches: refs/heads/apache-blur-0.2 5d159cb4f -> dc5a192fe
Fixed BLUR-254 Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/dc5a192f Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/dc5a192f Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/dc5a192f Branch: refs/heads/apache-blur-0.2 Commit: dc5a192fe4b7c49f6a4844cf3c9c93bfb692a479 Parents: 5d159cb Author: Aaron McCurry <[email protected]> Authored: Mon Oct 7 09:34:53 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Mon Oct 7 09:34:53 2013 -0400 ---------------------------------------------------------------------- .../blur/thrift/ThriftBlurControllerServer.java | 6 ++++ .../blur/thrift/ThriftBlurShardServer.java | 6 ++++ .../org/apache/blur/thrift/ThriftServer.java | 33 ++++++++++++++++++++ .../org/apache/blur/utils/BlurConstants.java | 6 ++++ .../src/main/resources/blur-default.properties | 20 ++++++++++++ docs/cluster-setup.html | 18 +++++++++++ 6 files changed, 89 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dc5a192f/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java index 4669bc8..347f1ca 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java @@ -32,6 +32,9 @@ import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_RETRY_MUTATE_D import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_SERVER_REMOTE_THREAD_COUNT; import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_SERVER_THRIFT_THREAD_COUNT; import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_SHARD_CONNECTION_TIMEOUT; +import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_THRIFT_ACCEPT_QUEUE_SIZE_PER_THREAD; +import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_THRIFT_MAX_READ_BUFFER_BYTES; +import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_THRIFT_SELECTOR_THREADS; import static org.apache.blur.utils.BlurConstants.BLUR_GUI_CONTROLLER_PORT; import static org.apache.blur.utils.BlurConstants.BLUR_GUI_SHARD_PORT; import static org.apache.blur.utils.BlurConstants.BLUR_MAX_RECORDS_PER_ROW_FETCH_REQUEST; @@ -142,6 +145,9 @@ public class ThriftBlurControllerServer extends ThriftServer { server.setThreadCount(threadCount); server.setEventHandler(eventHandler); server.setIface(iface); + server.setAcceptQueueSizePerThread(configuration.getInt(BLUR_CONTROLLER_THRIFT_ACCEPT_QUEUE_SIZE_PER_THREAD, 4)); + server.setMaxReadBufferBytes(configuration.getLong(BLUR_CONTROLLER_THRIFT_MAX_READ_BUFFER_BYTES, Long.MAX_VALUE)); + server.setSelectorThreads(configuration.getInt(BLUR_CONTROLLER_THRIFT_SELECTOR_THREADS, 2)); int baseGuiPort = Integer.parseInt(configuration.get(BLUR_GUI_CONTROLLER_PORT)); final HttpJettyServer httpServer; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dc5a192f/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java index 4c7d133..9febe8b 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java @@ -40,6 +40,9 @@ import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INTERNAL_SEARCH_THR import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_OPENER_THREAD_COUNT; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_SAFEMODEDELAY; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_SERVER_THRIFT_THREAD_COUNT; +import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_THRIFT_ACCEPT_QUEUE_SIZE_PER_THREAD; +import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_THRIFT_MAX_READ_BUFFER_BYTES; +import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_THRIFT_SELECTOR_THREADS; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_WARMUP_THREAD_COUNT; import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_CONNECTION; import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT; @@ -275,6 +278,9 @@ public class ThriftBlurShardServer extends ThriftServer { server.setThreadCount(threadCount); server.setIface(iface); server.setEventHandler(eventHandler); + server.setAcceptQueueSizePerThread(configuration.getInt(BLUR_SHARD_THRIFT_ACCEPT_QUEUE_SIZE_PER_THREAD, 4)); + server.setMaxReadBufferBytes(configuration.getLong(BLUR_SHARD_THRIFT_MAX_READ_BUFFER_BYTES, Long.MAX_VALUE)); + server.setSelectorThreads(configuration.getInt(BLUR_SHARD_THRIFT_SELECTOR_THREADS, 2)); // This will shutdown the server when the correct path is set in zk BlurShutdown shutdown = new BlurShutdown() { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dc5a192f/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java index 130fd86..81423cd 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java @@ -52,6 +52,7 @@ import org.apache.blur.thirdparty.thrift_0_9_0.transport.TTransportException; import org.apache.blur.thrift.generated.Blur; import org.apache.blur.thrift.generated.Blur.Iface; import org.apache.blur.thrift.server.TThreadedSelectorServer; +import org.apache.blur.thrift.server.TThreadedSelectorServer.Args.AcceptPolicy; import com.yammer.metrics.Metrics; import com.yammer.metrics.core.Gauge; @@ -72,6 +73,9 @@ public class ThriftServer { private ExecutorService _mutateExecutorService; private TServerEventHandler _eventHandler; private TNonblockingServerSocket _serverTransport; + private int _acceptQueueSizePerThread = 4; + private long _maxReadBufferBytes = Long.MAX_VALUE; + private int _selectorThreads = 2; public TNonblockingServerSocket getServerTransport() { return _serverTransport; @@ -193,6 +197,11 @@ public class ThriftServer { args.executorService(_executorService); args.transportFactory(new TFramedTransport.Factory()); args.protocolFactory(new TBinaryProtocol.Factory(true, true)); + args.selectorThreads = _selectorThreads; + args.maxReadBufferBytes = _maxReadBufferBytes; + args.acceptQueueSizePerThread(_acceptQueueSizePerThread); + args.acceptPolicy(AcceptPolicy.FAIR_ACCEPT); + _server = new TThreadedSelectorServer(args); _server.setServerEventHandler(_eventHandler); LOG.info("Starting server [{0}]", _nodeName); @@ -284,4 +293,28 @@ public class ThriftServer { _eventHandler = eventHandler; } + + public int getAcceptQueueSizePerThread() { + return _acceptQueueSizePerThread; + } + + public void setAcceptQueueSizePerThread(int acceptQueueSizePerThread) { + _acceptQueueSizePerThread = acceptQueueSizePerThread; + } + + public long getMaxReadBufferBytes() { + return _maxReadBufferBytes; + } + + public void setMaxReadBufferBytes(long maxReadBufferBytes) { + _maxReadBufferBytes = maxReadBufferBytes; + } + + public int getSelectorThreads() { + return _selectorThreads; + } + + public void setSelectorThreads(int selectorThreads) { + _selectorThreads = selectorThreads; + } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dc5a192f/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java ---------------------------------------------------------------------- diff --git a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java index 888c72f..2176ec6 100644 --- a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java +++ b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java @@ -72,6 +72,9 @@ public class BlurConstants { public static final String BLUR_SHARD_INDEX_DELETION_POLICY_MAXAGE = "blur.shard.index.deletion.policy.maxage"; public static final String BLUR_ZOOKEEPER_SYSTEM_TIME_TOLERANCE = "blur.zookeeper.system.time.tolerance"; public static final String BLUR_SHARD_INDEX_SIMILARITY = "blur.shard.index.similarity"; + public static final String BLUR_SHARD_THRIFT_SELECTOR_THREADS = "blur.shard.thrift.selector.threads"; + public static final String BLUR_SHARD_THRIFT_MAX_READ_BUFFER_BYTES = "blur.shard.thrift.max.read.buffer.bytes"; + public static final String BLUR_SHARD_THRIFT_ACCEPT_QUEUE_SIZE_PER_THREAD = "blur.shard.thrift.accept.queue.size.per.thread"; public static final String BLUR_FIELDTYPE = "blur.fieldtype."; @@ -92,6 +95,9 @@ public class BlurConstants { public static final String BLUR_CONTROLLER_RETRY_MAX_MUTATE_DELAY = "blur.controller.retry.max.mutate.delay"; public static final String BLUR_CONTROLLER_RETRY_MAX_DEFAULT_DELAY = "blur.controller.retry.max.default.delay"; public static final String BLUR_CONTROLLER_RETRY_MAX_FETCH_RETRIES = "blur.controller.retry.max.fetch.retries"; + public static final String BLUR_CONTROLLER_THRIFT_SELECTOR_THREADS = "blur.controller.thrift.selector.threads"; + public static final String BLUR_CONTROLLER_THRIFT_MAX_READ_BUFFER_BYTES = "blur.controller.thrift.max.read.buffer.bytes"; + public static final String BLUR_CONTROLLER_THRIFT_ACCEPT_QUEUE_SIZE_PER_THREAD = "blur.controller.thrift.accept.queue.size.per.thread"; public static final String BLUR_GUI_CONTROLLER_PORT = "blur.gui.controller.port"; public static final String BLUR_GUI_SHARD_PORT = "blur.gui.shard.port"; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dc5a192f/blur-util/src/main/resources/blur-default.properties ---------------------------------------------------------------------- diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties index b96c6d9..48aaabf 100644 --- a/blur-util/src/main/resources/blur-default.properties +++ b/blur-util/src/main/resources/blur-default.properties @@ -45,6 +45,15 @@ blur.shard.data.fetch.thread.count=8 # The number of the thrift threads blur.shard.server.thrift.thread.count=8 +# The number of threads used for selector processing inside the thrift server. +blur.shard.thrift.selector.threads=2 + +# The maximum number of bytes used for reading requests in the thrift server. +blur.shard.thrift.max.read.buffer.bytes=9223372036854775807 + +# The size of the blocking queue per selector thread for passing accepted connections to the selector thread. +blur.shard.thrift.accept.queue.size.per.thread=4 + # The number of threads that are used for opening indexes blur.shard.opener.thread.count=8 @@ -116,6 +125,8 @@ blur.max.records.per.row.fetch.request=1000 blur.gui.shard.port=40090 + + # Controller Properties # Sets the hostname for the controller, if blank the hostname is automatically detected @@ -136,6 +147,15 @@ blur.controller.server.thrift.thread.count=32 # The number of threads used for remote thrift requests to the shards server. This should be a large number. blur.controller.server.remote.thread.count=64 +# The number of threads used for selector processing inside the thrift server. +blur.controller.thrift.selector.threads=2 + +# The maximum number of bytes used for reading requests in the thrift server. +blur.controller.thrift.max.read.buffer.bytes=9223372036854775807 + +# The size of the blocking queue per selector thread for passing accepted connections to the selector thread. +blur.controller.thrift.accept.queue.size.per.thread=4 + # The number of hits to fetch per request to the shard servers blur.controller.remote.fetch.count=100 http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dc5a192f/docs/cluster-setup.html ---------------------------------------------------------------------- diff --git a/docs/cluster-setup.html b/docs/cluster-setup.html index 4ef9e00..a2285dd 100644 --- a/docs/cluster-setup.html +++ b/docs/cluster-setup.html @@ -165,6 +165,15 @@ blur.controller.server.thrift.thread.count=32 # the shards server. This should be a large number. blur.controller.server.remote.thread.count=64 +# The number of threads used for selector processing inside the thrift server. +blur.controller.thrift.selector.threads=2 + +# The maximum number of bytes used for reading requests in the thrift server. +blur.controller.thrift.max.read.buffer.bytes=9223372036854775807 + +# The size of the blocking queue per selector thread for passing accepted connections to the selector thread. +blur.controller.thrift.accept.queue.size.per.thread=4 + # The number of hits to fetch per request to the shard servers blur.controller.remote.fetch.count=100 @@ -264,6 +273,15 @@ blur.shard.data.fetch.thread.count=8 # The number of the thrift threads blur.shard.server.thrift.thread.count=8 +# The number of threads used for selector processing inside the thrift server. +blur.shard.thrift.selector.threads=2 + +# The maximum number of bytes used for reading requests in the thrift server. +blur.shard.thrift.max.read.buffer.bytes=9223372036854775807 + +# The size of the blocking queue per selector thread for passing accepted connections to the selector thread. +blur.shard.thrift.accept.queue.size.per.thread=4 + # The number of threads that are used for opening indexes blur.shard.opener.thread.count=8
