Updated Branches: refs/heads/master ffa1d72a9 -> da43794ff
Adding configuration for socket timeouts. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/da380c1f Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/da380c1f Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/da380c1f Branch: refs/heads/master Commit: da380c1f6d7ee90e3791b29dc318ad15bc30c900 Parents: ffa1d72 Author: Aaron McCurry <[email protected]> Authored: Thu Aug 29 19:24:15 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Thu Aug 29 19:24:15 2013 -0400 ---------------------------------------------------------------------- .../blur/thrift/BlurControllerServer.java | 28 +++++++++++++------- .../blur/thrift/ThriftBlurControllerServer.java | 8 +++--- .../java/org/apache/blur/thrift/Connection.java | 8 +++++- .../org/apache/blur/utils/BlurConstants.java | 3 ++- .../src/main/resources/blur-default.properties | 3 +++ docs/cluster-setup.html | 3 +++ 6 files changed, 38 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/da380c1f/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java index e674a6c..0602367 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java @@ -94,10 +94,16 @@ public class BlurControllerServer extends TableAdmin implements Iface { } public static class BlurClientRemote extends BlurClient { + private int _timeout; + + public BlurClientRemote(int timeout) { + _timeout = timeout; + } + @Override public <T> T execute(String node, BlurCommand<T> command, int maxRetries, long backOffTime, long maxBackOffTime) throws BlurException, TException, IOException { - return BlurClientManager.execute(node, command, maxRetries, backOffTime, maxBackOffTime); + return BlurClientManager.execute(node + "#" + _timeout, command, maxRetries, backOffTime, maxBackOffTime); } } @@ -817,7 +823,7 @@ public class BlurControllerServer extends TableAdmin implements Iface { } })); } - + for (Future<Void> future : futures) { try { future.get(); @@ -830,7 +836,7 @@ public class BlurControllerServer extends TableAdmin implements Iface { } } } - + @Override public void createSnapshot(final String table, final String name) throws BlurException, TException { checkTable(table); @@ -844,10 +850,11 @@ public class BlurControllerServer extends TableAdmin implements Iface { }); } catch (Exception e) { LOG.error("Unknown error while trying to create a snapshot of table [{0}] snapshot name", e, table, name); - throw new BException("Unknown error while trying to create a snapshot of table [{0}] snapshot name", e, table, name); + throw new BException("Unknown error while trying to create a snapshot of table [{0}] snapshot name", e, table, + name); } } - + @Override public void removeSnapshot(final String table, final String name) throws BlurException, TException { checkTable(table); @@ -861,10 +868,11 @@ public class BlurControllerServer extends TableAdmin implements Iface { }); } catch (Exception e) { LOG.error("Unknown error while trying to remove a snapshot of table [{0}] snapshot name", e, table, name); - throw new BException("Unknown error while trying to remove a snapshot of table [{0}] snapshot name", e, table, name); + throw new BException("Unknown error while trying to remove a snapshot of table [{0}] snapshot name", e, table, + name); } } - + @Override public Map<String, List<String>> listSnapshots(final String table) throws BlurException, TException { checkTable(table); @@ -876,10 +884,12 @@ public class BlurControllerServer extends TableAdmin implements Iface { } }, new Merger<Map<String, List<String>>>() { @Override - public Map<String, List<String>> merge(BlurExecutorCompletionService<Map<String, List<String>>> service) throws BlurException { + public Map<String, List<String>> merge(BlurExecutorCompletionService<Map<String, List<String>>> service) + throws BlurException { Map<String, List<String>> result = new HashMap<String, List<String>>(); while (service.getRemainingCount() > 0) { - Future<Map<String, List<String>>> future = service.poll(_defaultParallelCallTimeout, TimeUnit.MILLISECONDS, true); + Future<Map<String, List<String>>> future = service.poll(_defaultParallelCallTimeout, TimeUnit.MILLISECONDS, + true); Map<String, List<String>> snapshotsOnAShardServer = service.getResultThrowException(future); for (Entry<String, List<String>> entry : snapshotsOnAShardServer.entrySet()) { List<String> snapshots = result.get(entry.getKey()); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/da380c1f/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java index 4d8a094..8017bf7 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java @@ -31,6 +31,7 @@ import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_RETRY_MAX_MUTA import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_RETRY_MUTATE_DELAY; import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_SERVER_REMOTE_THREAD_COUNT; import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_SERVER_THRIFT_THREAD_COUNT; +import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_SHARD_CONNECTION_TIMEOUT; import static org.apache.blur.utils.BlurConstants.BLUR_GUI_CONTROLLER_PORT; import static org.apache.blur.utils.BlurConstants.BLUR_GUI_SHARD_PORT; import static org.apache.blur.utils.BlurConstants.BLUR_MAX_RECORDS_PER_ROW_FETCH_REQUEST; @@ -64,7 +65,6 @@ public class ThriftBlurControllerServer extends ThriftServer { private static final Log LOG = LogFactory.getLog(ThriftBlurControllerServer.class); - public static void main(String[] args) throws Exception { int serverIndex = getServerIndex(args); LOG.info("Setting up Controller Server"); @@ -100,7 +100,8 @@ public class ThriftBlurControllerServer extends ThriftServer { final ZookeeperClusterStatus clusterStatus = new ZookeeperClusterStatus(zooKeeper, configuration); - BlurControllerServer.BlurClient client = new BlurControllerServer.BlurClientRemote(); + int timeout = configuration.getInt(BLUR_CONTROLLER_SHARD_CONNECTION_TIMEOUT, 60000); + BlurControllerServer.BlurClient client = new BlurControllerServer.BlurClientRemote(timeout); final BlurControllerServer controllerServer = new BlurControllerServer(); controllerServer.setClient(client); @@ -142,8 +143,7 @@ public class ThriftBlurControllerServer extends ThriftServer { final HttpJettyServer httpServer; if (baseGuiPort > 0) { int webServerPort = baseGuiPort + serverIndex; - // TODO: this got ugly, there has to be a better way to handle all these - // params + // TODO: this got ugly, there has to be a better way to handle all these params // without reversing the mvn dependancy and making blur-gui on top. httpServer = new HttpJettyServer(bindPort, webServerPort, configuration.getInt(BLUR_CONTROLLER_BIND_PORT, -1), configuration.getInt(BLUR_SHARD_BIND_PORT, -1), configuration.getInt(BLUR_GUI_CONTROLLER_PORT, -1), configuration.getInt(BLUR_GUI_SHARD_PORT, -1), "controller"); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/da380c1f/blur-thrift/src/main/java/org/apache/blur/thrift/Connection.java ---------------------------------------------------------------------- diff --git a/blur-thrift/src/main/java/org/apache/blur/thrift/Connection.java b/blur-thrift/src/main/java/org/apache/blur/thrift/Connection.java index 5124105..e2c5e5f 100644 --- a/blur-thrift/src/main/java/org/apache/blur/thrift/Connection.java +++ b/blur-thrift/src/main/java/org/apache/blur/thrift/Connection.java @@ -30,6 +30,13 @@ public class Connection { private final int _timeout; public Connection(String connectionStr) { + int indexOfTimeout = connectionStr.indexOf("#"); + if (indexOfTimeout > 0) { + _timeout = Integer.parseInt(connectionStr.substring(indexOfTimeout + 1)); + connectionStr = connectionStr.substring(0,indexOfTimeout); + } else { + _timeout = DEFAULT_TIMEOUT; + } int index = connectionStr.indexOf(':'); if (index >= 0) { int slashIndex = connectionStr.indexOf('/'); @@ -47,7 +54,6 @@ public class Connection { _proxyPort = -1; _proxy = false; } - _timeout = DEFAULT_TIMEOUT; } else { throw new RuntimeException("Connection string of [" + connectionStr + "] does not match 'host1:port' or 'host1:port/proxyhost1:proxyport'"); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/da380c1f/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java ---------------------------------------------------------------------- diff --git a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java index 59fde28..60368f8 100644 --- a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java +++ b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java @@ -77,7 +77,8 @@ public class BlurConstants { public static final String BLUR_CONTROLLER_SERVER_THRIFT_THREAD_COUNT = "blur.controller.server.thrift.thread.count"; public static final String BLUR_CONTROLLER_SERVER_REMOTE_THREAD_COUNT = "blur.controller.server.remote.thread.count"; public static final String BLUR_CONTROLLER_REMOTE_FETCH_COUNT = "blur.controller.remote.fetch.count"; - + + public static final String BLUR_CONTROLLER_SHARD_CONNECTION_TIMEOUT = "blur.controller.shard.connection.timeout"; public static final String BLUR_CONTROLLER_RETRY_MAX_MUTATE_RETRIES = "blur.controller.retry.max.mutate.retries"; public static final String BLUR_CONTROLLER_RETRY_MAX_DEFAULT_RETRIES = "blur.controller.retry.max.default.retries"; public static final String BLUR_CONTROLLER_RETRY_FETCH_DELAY = "blur.controller.retry.fetch.delay"; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/da380c1f/blur-util/src/main/resources/blur-default.properties ---------------------------------------------------------------------- diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties index 098cc1a..d28f39d 100644 --- a/blur-util/src/main/resources/blur-default.properties +++ b/blur-util/src/main/resources/blur-default.properties @@ -121,6 +121,9 @@ blur.controller.bind.address=0.0.0.0 # The default binding port of the controller server blur.controller.bind.port=40010 +# The connection timeout, NOTE: this will be the maximum amount of time you can wait for a query. +blur.controller.shard.connection.timeout=60000 + # The number of threads used for thrift requests blur.controller.server.thrift.thread.count=32 http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/da380c1f/docs/cluster-setup.html ---------------------------------------------------------------------- diff --git a/docs/cluster-setup.html b/docs/cluster-setup.html index 36452b5..d6ba58b 100644 --- a/docs/cluster-setup.html +++ b/docs/cluster-setup.html @@ -152,6 +152,9 @@ blur.controller.bind.address=0.0.0.0 # The default binding port of the controller server blur.controller.bind.port=40010 +# The connection timeout, NOTE: this will be the maximum amount of time you can wait for a query. +blur.controller.shard.connection.timeout=60000 + # The number of threads used for thrift requests blur.controller.server.thrift.thread.count=32
