Fixed BLUR-314
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/a348d49c Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/a348d49c Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/a348d49c Branch: refs/heads/apache-blur-0.2 Commit: a348d49c2bb7b07e0cf1881bdecd253526cf354f Parents: deef456 Author: Aaron McCurry <[email protected]> Authored: Tue Dec 3 13:16:03 2013 -0500 Committer: Aaron McCurry <[email protected]> Committed: Tue Dec 3 13:16:03 2013 -0500 ---------------------------------------------------------------------- .../blur/thrift/ThriftBlurControllerServer.java | 2 ++ .../org/apache/blur/thrift/ThriftBlurShardServer.java | 4 +++- .../java/org/apache/blur/thrift/ThriftServer.java | 14 ++++++++++++-- .../main/java/org/apache/blur/thrift/ClientPool.java | 11 ++++++++--- .../java/org/apache/blur/utils/BlurConstants.java | 2 ++ blur-util/src/main/resources/blur-default.properties | 3 +++ 6 files changed, 30 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a348d49c/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java index 2d78a13..b796f2b 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java @@ -39,6 +39,7 @@ import static org.apache.blur.utils.BlurConstants.BLUR_GUI_CONTROLLER_PORT; import static org.apache.blur.utils.BlurConstants.BLUR_GUI_SHARD_PORT; import static org.apache.blur.utils.BlurConstants.BLUR_MAX_RECORDS_PER_ROW_FETCH_REQUEST; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BIND_PORT; +import static org.apache.blur.utils.BlurConstants.BLUR_THRIFT_MAX_FRAME_SIZE; import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_CONNECTION; import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT; import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT_DEFAULT; @@ -164,6 +165,7 @@ public class ThriftBlurControllerServer extends ThriftServer { server.setAcceptQueueSizePerThread(configuration.getInt(BLUR_CONTROLLER_THRIFT_ACCEPT_QUEUE_SIZE_PER_THREAD, 4)); server.setMaxReadBufferBytes(configuration.getLong(BLUR_CONTROLLER_THRIFT_MAX_READ_BUFFER_BYTES, Long.MAX_VALUE)); server.setSelectorThreads(configuration.getInt(BLUR_CONTROLLER_THRIFT_SELECTOR_THREADS, 2)); + server.setMaxFrameSize(configuration.getInt(BLUR_THRIFT_MAX_FRAME_SIZE, 16384000)); int baseGuiPort = Integer.parseInt(configuration.get(BLUR_GUI_CONTROLLER_PORT)); final HttpJettyServer httpServer; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a348d49c/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java index d247d3a..4159b91 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java @@ -43,6 +43,7 @@ import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_THRIFT_ACCEPT_QUEUE import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_THRIFT_MAX_READ_BUFFER_BYTES; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_THRIFT_SELECTOR_THREADS; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_WARMUP_THREAD_COUNT; +import static org.apache.blur.utils.BlurConstants.BLUR_THRIFT_MAX_FRAME_SIZE; import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_CONNECTION; import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT; import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT_DEFAULT; @@ -252,7 +253,7 @@ public class ThriftBlurShardServer extends ThriftServer { int threadCount = configuration.getInt(BLUR_SHARD_SERVER_THRIFT_THREAD_COUNT, 32); ShardServerEventHandler eventHandler = new ShardServerEventHandler(); - + final ThriftBlurShardServer server = new ThriftBlurShardServer(); server.setNodeName(nodeName); server.setServerTransport(tNonblockingServerSocket); @@ -262,6 +263,7 @@ public class ThriftBlurShardServer extends ThriftServer { server.setAcceptQueueSizePerThread(configuration.getInt(BLUR_SHARD_THRIFT_ACCEPT_QUEUE_SIZE_PER_THREAD, 4)); server.setMaxReadBufferBytes(configuration.getLong(BLUR_SHARD_THRIFT_MAX_READ_BUFFER_BYTES, Long.MAX_VALUE)); server.setSelectorThreads(configuration.getInt(BLUR_SHARD_THRIFT_SELECTOR_THREADS, 2)); + server.setMaxFrameSize(config.getInt(BLUR_THRIFT_MAX_FRAME_SIZE, 16384000)); // This will shutdown the server when the correct path is set in zk BlurShutdown shutdown = new BlurShutdown() { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a348d49c/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java index 4cda152..48c0b27 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java @@ -63,7 +63,7 @@ import com.yammer.metrics.core.Gauge; import com.yammer.metrics.core.MetricName; public class ThriftServer { - + private static final Log LOG = LogFactory.getLog(ThriftServer.class); private String _nodeName; @@ -80,6 +80,15 @@ public class ThriftServer { private int _acceptQueueSizePerThread = 4; private long _maxReadBufferBytes = Long.MAX_VALUE; private int _selectorThreads = 2; + private int _maxFrameSize = 16384000; + + public int getMaxFrameSize() { + return _maxFrameSize; + } + + public void setMaxFrameSize(int maxFrameSize) { + _maxFrameSize = maxFrameSize; + } public TNonblockingServerSocket getServerTransport() { return _serverTransport; @@ -204,10 +213,11 @@ public class ThriftServer { public void start() throws TTransportException { _executorService = Executors.newThreadPool("thrift-processors", _threadCount); Blur.Processor<Blur.Iface> processor = new Blur.Processor<Blur.Iface>(_iface); + TThreadedSelectorServer.Args args = new TThreadedSelectorServer.Args(_serverTransport); args.processor(processor); args.executorService(_executorService); - args.transportFactory(new TFramedTransport.Factory()); + args.transportFactory(new TFramedTransport.Factory(_maxFrameSize)); args.protocolFactory(new TBinaryProtocol.Factory(true, true)); args.selectorThreads = _selectorThreads; args.maxReadBufferBytes = _maxReadBufferBytes; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a348d49c/blur-thrift/src/main/java/org/apache/blur/thrift/ClientPool.java ---------------------------------------------------------------------- diff --git a/blur-thrift/src/main/java/org/apache/blur/thrift/ClientPool.java b/blur-thrift/src/main/java/org/apache/blur/thrift/ClientPool.java index 440eea1..ee9e3db 100644 --- a/blur-thrift/src/main/java/org/apache/blur/thrift/ClientPool.java +++ b/blur-thrift/src/main/java/org/apache/blur/thrift/ClientPool.java @@ -17,6 +17,10 @@ package org.apache.blur.thrift; * limitations under the License. */ +import static org.apache.blur.utils.BlurConstants.BLUR_CLIENTPOOL_CLIENT_CLEAN_FREQUENCY; +import static org.apache.blur.utils.BlurConstants.BLUR_CLIENTPOOL_CLIENT_CLOSE_THRESHOLD; +import static org.apache.blur.utils.BlurConstants.BLUR_THRIFT_MAX_FRAME_SIZE; + import java.io.IOException; import java.net.InetSocketAddress; import java.net.Proxy; @@ -40,15 +44,15 @@ import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocol; import org.apache.blur.thirdparty.thrift_0_9_0.transport.TFramedTransport; import org.apache.blur.thirdparty.thrift_0_9_0.transport.TSocket; import org.apache.blur.thirdparty.thrift_0_9_0.transport.TTransportException; -import static org.apache.blur.utils.BlurConstants.BLUR_CLIENTPOOL_CLIENT_CLOSE_THRESHOLD; -import static org.apache.blur.utils.BlurConstants.BLUR_CLIENTPOOL_CLIENT_CLEAN_FREQUENCY; import org.apache.blur.thrift.generated.Blur.Client; public class ClientPool { + private static final Log LOG = LogFactory.getLog(ClientPool.class); private static final Map<Connection, BlockingQueue<Client>> _connMap = new ConcurrentHashMap<Connection, BlockingQueue<Client>>(); private int _maxConnectionsPerHost = Integer.MAX_VALUE; + private static final int _maxFrameSize; private static AtomicBoolean _running = new AtomicBoolean(true); private static long _idleTimeBeforeClosingClient; private static long _clientPoolCleanFrequency; @@ -61,6 +65,7 @@ public class ClientPool { TimeUnit.SECONDS.toMillis(30)); _clientPoolCleanFrequency = config.getLong(BLUR_CLIENTPOOL_CLIENT_CLEAN_FREQUENCY, TimeUnit.SECONDS.toMillis(300)); + _maxFrameSize = config.getInt(BLUR_THRIFT_MAX_FRAME_SIZE, 16384000); } catch (Exception e) { throw new RuntimeException(e); } @@ -211,7 +216,7 @@ public class ClientPool { socket.connect(new InetSocketAddress(host, port), timeout); trans = new TSocket(socket); - TProtocol proto = new TBinaryProtocol(new TFramedTransport(trans)); + TProtocol proto = new TBinaryProtocol(new TFramedTransport(trans,_maxFrameSize)); return new WeightedClient(proto); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a348d49c/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java ---------------------------------------------------------------------- diff --git a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java index 525b4bd..094815a 100644 --- a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java +++ b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java @@ -120,6 +120,8 @@ public class BlurConstants { public static final String BLUR_CLIENTPOOL_CLIENT_CLOSE_THRESHOLD = "blur.clientpool.client.close.threshold"; public static final String BLUR_CLIENTPOOL_CLIENT_CLEAN_FREQUENCY = "blur.clientpool.client.clean.frequency"; + public static final String BLUR_THRIFT_MAX_FRAME_SIZE = "blur.thrift.max.frame.size"; + public static final String BLUR_SHARD_FILTERED_SERVER_CLASS = "blur.shard.filtered.server.class"; public static final String BLUR_CONTROLLER_FILTERED_SERVER_CLASS = "blur.controller.filtered.server.class"; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a348d49c/blur-util/src/main/resources/blur-default.properties ---------------------------------------------------------------------- diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties index 393bfb2..213c931 100644 --- a/blur-util/src/main/resources/blur-default.properties +++ b/blur-util/src/main/resources/blur-default.properties @@ -31,6 +31,9 @@ blur.query.max.record.fetch=1000 # Setup metric reporter blur.metrics.reporters= +# Thrift max frame size +blur.thrift.max.frame.size=16384000 + ### Shard Server Configuration # The hostname for the shard, if blank the hostname is automatically detected
