Repository: incubator-blur Updated Branches: refs/heads/blur-342a [created] 9acdad258
initial cut at randomizing things - still a wip Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/d19cdf2a Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/d19cdf2a Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/d19cdf2a Branch: refs/heads/blur-342a Commit: d19cdf2a842ceee2de5416fe6f47731462c59b6d Parents: 355d14f Author: Tim <[email protected]> Authored: Wed Jul 16 16:41:20 2014 -0400 Committer: Tim <[email protected]> Committed: Wed Jul 23 16:04:06 2014 -0400 ---------------------------------------------------------------------- .../clusterstatus/ZookeeperClusterStatus.java | 1 + .../clusterstatus/ZookeeperPathConstants.java | 74 -------------------- .../indexserver/DistributedIndexServer.java | 2 +- .../MasterBasedDistributedLayoutFactory.java | 2 +- .../blur/thrift/BlurControllerServer.java | 2 +- .../java/org/apache/blur/thrift/Shutdown.java | 2 +- .../blur/thrift/ThriftBlurControllerServer.java | 24 +++---- .../blur/thrift/ThriftBlurShardServer.java | 24 +++---- .../java/org/apache/blur/utils/BlurUtil.java | 2 +- .../test/java/org/apache/blur/MiniCluster.java | 14 +++- .../ZookeeperClusterStatusTest.java | 1 + .../main/java/org/apache/blur/shell/Main.java | 69 ++++-------------- .../java/org/apache/blur/thrift/BlurClient.java | 45 ++++++++++++ .../java/org/apache/blur/BlurConfiguration.java | 8 +++ .../blur/zookeeper/ZookeeperPathConstants.java | 74 ++++++++++++++++++++ .../src/main/resources/blur-default.properties | 4 +- 16 files changed, 186 insertions(+), 162 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java b/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java index df45b73..bf10b8c 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java +++ b/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java @@ -48,6 +48,7 @@ import org.apache.blur.zookeeper.WatchChildren.OnChange; import org.apache.blur.zookeeper.WatchNodeData; import org.apache.blur.zookeeper.ZkUtils; import org.apache.blur.zookeeper.ZooKeeperLockManager; +import org.apache.blur.zookeeper.ZookeeperPathConstants; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java b/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java deleted file mode 100644 index a8099c1..0000000 --- a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java +++ /dev/null @@ -1,74 +0,0 @@ -package org.apache.blur.manager.clusterstatus; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -public class ZookeeperPathConstants { - - public static String getBasePath() { - return "/blur"; - } - - public static String getClusterPath(String cluster) { - return getClustersPath() + "/" + cluster; - } - - public static String getClustersPath() { - return getBasePath() + "/clusters"; - } - - public static String getOnlineControllersPath() { - return getBasePath() + "/online-controller-nodes"; - } - - public static String getControllersPath() { - return getBasePath() + "/controller-nodes"; - } - - public static String getOnlineShardsPath(String cluster) { - return getClusterPath(cluster) + "/online-nodes"; - } - - public static String getTablesPath(String cluster) { - return getClusterPath(cluster) + "/tables"; - } - - public static String getTablePath(String cluster, String table) { - return getTablesPath(cluster) + "/" + table; - } - - public static String getSafemodePath(String cluster) { - return getClusterPath(cluster) + "/safemode"; - } - - public static String getShutdownPath(String cluster) { - return getClusterPath(cluster) + "/shutdown"; - } - - public static String getRegisteredShardsPath(String cluster) { - return getClusterPath(cluster) + "/registered-nodes"; - } - - public static String getShardLayoutPath(String cluster) { - return getClusterPath(cluster) + "/layout"; - } - - public static String getShardLayoutPathLocks(String cluster) { - return getShardLayoutPath(cluster) + "/locks"; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java index 59d3f68..351d8fb 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java +++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java @@ -42,7 +42,6 @@ import org.apache.blur.log.LogFactory; import org.apache.blur.manager.BlurFilterCache; import org.apache.blur.manager.clusterstatus.ClusterStatus; import org.apache.blur.manager.clusterstatus.ClusterStatus.Action; -import org.apache.blur.manager.clusterstatus.ZookeeperPathConstants; import org.apache.blur.manager.writer.BlurIndex; import org.apache.blur.manager.writer.BlurIndexCloser; import org.apache.blur.manager.writer.BlurIndexReadOnly; @@ -59,6 +58,7 @@ import org.apache.blur.thrift.generated.ShardState; import org.apache.blur.thrift.generated.TableDescriptor; import org.apache.blur.utils.BlurUtil; import org.apache.blur.zookeeper.WatchChildren; +import org.apache.blur.zookeeper.ZookeeperPathConstants; import org.apache.blur.zookeeper.WatchChildren.OnChange; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java index 319ccf2..e7ad6fa 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java +++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java @@ -38,9 +38,9 @@ import java.util.concurrent.ConcurrentMap; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; -import org.apache.blur.manager.clusterstatus.ZookeeperPathConstants; import org.apache.blur.zookeeper.ZkUtils; import org.apache.blur.zookeeper.ZooKeeperLockManager; +import org.apache.blur.zookeeper.ZookeeperPathConstants; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.ZooDefs.Ids; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java index a1cec15..a0453bc 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java @@ -53,7 +53,6 @@ import org.apache.blur.log.LogFactory; import org.apache.blur.manager.BlurPartitioner; import org.apache.blur.manager.BlurQueryChecker; import org.apache.blur.manager.IndexManager; -import org.apache.blur.manager.clusterstatus.ZookeeperPathConstants; import org.apache.blur.manager.indexserver.DistributedLayout; import org.apache.blur.manager.indexserver.DistributedLayoutFactory; import org.apache.blur.manager.indexserver.DistributedLayoutFactoryImpl; @@ -100,6 +99,7 @@ import org.apache.blur.utils.ForkJoin; import org.apache.blur.utils.ForkJoin.Merger; import org.apache.blur.utils.ForkJoin.ParallelCall; import org.apache.blur.zookeeper.WatchChildren; +import org.apache.blur.zookeeper.ZookeeperPathConstants; import org.apache.blur.zookeeper.WatchChildren.OnChange; import org.apache.blur.zookeeper.WatchNodeExistance; import org.apache.zookeeper.CreateMode; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-core/src/main/java/org/apache/blur/thrift/Shutdown.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/Shutdown.java b/blur-core/src/main/java/org/apache/blur/thrift/Shutdown.java index 5f082e0..737794b 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/Shutdown.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/Shutdown.java @@ -25,8 +25,8 @@ import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT_DEFAULT import java.io.IOException; import org.apache.blur.BlurConfiguration; -import org.apache.blur.manager.clusterstatus.ZookeeperPathConstants; import org.apache.blur.zookeeper.ZkUtils; +import org.apache.blur.zookeeper.ZookeeperPathConstants; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.ZooDefs.Ids; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java index 76897f6..1297447 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java @@ -84,7 +84,7 @@ public class ThriftBlurControllerServer extends ThriftServer { ReporterSetup.setupReporters(configuration); MemoryReporter.enable(); setupJvmMetrics(); - ThriftServer server = createServer(serverIndex, configuration, false); + ThriftServer server = createServer(serverIndex, configuration); server.start(); } catch (Throwable t) { t.printStackTrace(); @@ -92,25 +92,25 @@ public class ThriftBlurControllerServer extends ThriftServer { } } - public static ThriftServer createServer(int serverIndex, BlurConfiguration configuration, boolean randomPort) + public static ThriftServer createServer(int serverIndex, BlurConfiguration configuration) throws Exception { Thread.setDefaultUncaughtExceptionHandler(new SimpleUncaughtExceptionHandler()); String bindAddress = configuration.get(BLUR_CONTROLLER_BIND_ADDRESS); - int bindPort = configuration.getInt(BLUR_CONTROLLER_BIND_PORT, -1); - bindPort += serverIndex; - if (randomPort) { - bindPort = 0; + int configBindPort = configuration.getInt(BLUR_CONTROLLER_BIND_PORT, -1); + int instanceBindPort = configBindPort + serverIndex; + if (configBindPort == 0) { + instanceBindPort = 0; } - TNonblockingServerSocket tNonblockingServerSocket = ThriftServer.getTNonblockingServerSocket(bindAddress, bindPort); - if (randomPort) { - bindPort = tNonblockingServerSocket.getServerSocket().getLocalPort(); + TNonblockingServerSocket tNonblockingServerSocket = ThriftServer.getTNonblockingServerSocket(bindAddress, instanceBindPort); + if (configBindPort == 0) { + instanceBindPort = tNonblockingServerSocket.getServerSocket().getLocalPort(); } - LOG.info("Controller Server using index [{0}] bind address [{1}] random port assignment [{2}]", serverIndex, - bindAddress + ":" + bindPort, randomPort); + LOG.info("Controller Server using index [{0}] bind address [{1}]", serverIndex, + bindAddress + ":" + instanceBindPort); String nodeName = getNodeName(configuration, BLUR_CONTROLLER_HOSTNAME); - nodeName = nodeName + ":" + bindPort; + nodeName = nodeName + ":" + instanceBindPort; String zkConnectionStr = isEmpty(configuration.get(BLUR_ZOOKEEPER_CONNECTION), BLUR_ZOOKEEPER_CONNECTION); BlurQueryChecker queryChecker = new BlurQueryChecker(configuration); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java index cd39485..52bd2ff 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java @@ -121,7 +121,7 @@ public class ThriftBlurShardServer extends ThriftServer { setupJvmMetrics(); // make this configurable GCWatcher.init(0.75); - ThriftServer server = createServer(serverIndex, configuration, false); + ThriftServer server = createServer(serverIndex, configuration); server.start(); } catch (Throwable t) { t.printStackTrace(); @@ -129,21 +129,21 @@ public class ThriftBlurShardServer extends ThriftServer { } } - public static ThriftServer createServer(int serverIndex, BlurConfiguration configuration, boolean randomPort) + public static ThriftServer createServer(int serverIndex, BlurConfiguration configuration) throws Exception { Configuration config = new Configuration(); TableContext.setSystemBlurConfiguration(configuration); TableContext.setSystemConfiguration(config); String bindAddress = configuration.get(BLUR_SHARD_BIND_ADDRESS); - int bindPort = configuration.getInt(BLUR_SHARD_BIND_PORT, -1); - bindPort += serverIndex; - if (randomPort) { - bindPort = 0; + int configBindPort = configuration.getInt(BLUR_SHARD_BIND_PORT, -1); + int instanceBindPort = configBindPort + serverIndex; + if (configBindPort == 0) { + instanceBindPort = 0; } - TNonblockingServerSocket tNonblockingServerSocket = ThriftServer.getTNonblockingServerSocket(bindAddress, bindPort); - if (randomPort) { - bindPort = tNonblockingServerSocket.getServerSocket().getLocalPort(); + TNonblockingServerSocket tNonblockingServerSocket = ThriftServer.getTNonblockingServerSocket(bindAddress, instanceBindPort); + if (configBindPort == 0) { + instanceBindPort = tNonblockingServerSocket.getServerSocket().getLocalPort(); } int baseGuiPort = Integer.parseInt(configuration.get(BLUR_GUI_SHARD_PORT)); @@ -180,11 +180,11 @@ public class ThriftBlurShardServer extends ThriftServer { } else { throw new RuntimeException("Unknown block cache version [" + blockCacheVersion + "] can be [v1,v2]"); } - LOG.info("Shard Server using index [{0}] bind address [{1}] random port assignment [{2}]", serverIndex, bindAddress - + ":" + bindPort, randomPort); + LOG.info("Shard Server using index [{0}] bind address [{1}]", serverIndex, bindAddress + + ":" + instanceBindPort); String nodeNameHostName = getNodeName(configuration, BLUR_SHARD_HOSTNAME); - String nodeName = nodeNameHostName + ":" + bindPort; + String nodeName = nodeNameHostName + ":" + instanceBindPort; String zkConnectionStr = isEmpty(configuration.get(BLUR_ZOOKEEPER_CONNECTION), BLUR_ZOOKEEPER_CONNECTION); BlurQueryChecker queryChecker = new BlurQueryChecker(configuration); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java index 52fd09f..b57573d 100644 --- a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java +++ b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java @@ -68,7 +68,6 @@ import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; import org.apache.blur.lucene.search.PrimeDocCache; import org.apache.blur.manager.BlurHighlighter; -import org.apache.blur.manager.clusterstatus.ZookeeperPathConstants; import org.apache.blur.manager.results.BlurResultComparator; import org.apache.blur.manager.results.BlurResultIterable; import org.apache.blur.manager.results.BlurResultPeekableIteratorComparator; @@ -104,6 +103,7 @@ import org.apache.blur.trace.Trace; import org.apache.blur.trace.Tracer; import org.apache.blur.user.User; import org.apache.blur.user.UserContext; +import org.apache.blur.zookeeper.ZookeeperPathConstants; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-core/src/test/java/org/apache/blur/MiniCluster.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/MiniCluster.java b/blur-core/src/test/java/org/apache/blur/MiniCluster.java index 3a2f826..f188d90 100644 --- a/blur-core/src/test/java/org/apache/blur/MiniCluster.java +++ b/blur-core/src/test/java/org/apache/blur/MiniCluster.java @@ -24,6 +24,8 @@ import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCKCACHE_SLAB_COU import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_HOSTNAME; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_SAFEMODEDELAY; import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_CONNECTION; +import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_BIND_PORT; +import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BIND_PORT; import java.io.File; import java.io.IOException; @@ -42,7 +44,6 @@ import java.util.concurrent.TimeUnit; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; -import org.apache.blur.manager.clusterstatus.ZookeeperPathConstants; import org.apache.blur.store.buffer.BufferStore; import org.apache.blur.thirdparty.thrift_0_9_0.TException; import org.apache.blur.thirdparty.thrift_0_9_0.transport.TTransportException; @@ -63,6 +64,7 @@ import org.apache.blur.thrift.util.BlurThriftHelper; import org.apache.blur.utils.BlurUtil; import org.apache.blur.zookeeper.ZkMiniCluster; import org.apache.blur.zookeeper.ZooKeeperClient; +import org.apache.blur.zookeeper.ZookeeperPathConstants; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -262,9 +264,12 @@ public class MiniCluster { public void startControllers(BlurConfiguration configuration, int num, boolean randomPort) { BlurConfiguration localConf = getBlurConfiguration(configuration); + if(randomPort) { + localConf.setInt(BLUR_CONTROLLER_BIND_PORT, 0); + } for (int i = 0; i < num; i++) { try { - ThriftServer server = ThriftBlurControllerServer.createServer(i, localConf, randomPort); + ThriftServer server = ThriftBlurControllerServer.createServer(i, localConf); controllers.add(server); startServer(server); } catch (Exception e) { @@ -281,6 +286,9 @@ public class MiniCluster { public void startShards(final BlurConfiguration configuration, int num, final boolean randomPort) { final BlurConfiguration localConf = getBlurConfiguration(configuration); + if(randomPort) { + localConf.setInt(BLUR_SHARD_BIND_PORT, 0); + } ExecutorService executorService = Executors.newFixedThreadPool(num); List<Future<ThriftServer>> futures = new ArrayList<Future<ThriftServer>>(); for (int i = 0; i < num; i++) { @@ -288,7 +296,7 @@ public class MiniCluster { futures.add(executorService.submit(new Callable<ThriftServer>() { @Override public ThriftServer call() throws Exception { - return ThriftBlurShardServer.createServer(index, localConf, randomPort); + return ThriftBlurShardServer.createServer(index, localConf); } })); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-core/src/test/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatusTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatusTest.java b/blur-core/src/test/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatusTest.java index d5236f8..deb930d 100644 --- a/blur-core/src/test/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatusTest.java +++ b/blur-core/src/test/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatusTest.java @@ -35,6 +35,7 @@ import org.apache.blur.thrift.generated.TableDescriptor; import org.apache.blur.utils.BlurUtil; import org.apache.blur.zookeeper.ZooKeeperClient; import org.apache.blur.zookeeper.ZooKeeperLockManager; +import org.apache.blur.zookeeper.ZookeeperPathConstants; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-shell/src/main/java/org/apache/blur/shell/Main.java ---------------------------------------------------------------------- diff --git a/blur-shell/src/main/java/org/apache/blur/shell/Main.java b/blur-shell/src/main/java/org/apache/blur/shell/Main.java index 8e9bfd3..b7654c9 100644 --- a/blur-shell/src/main/java/org/apache/blur/shell/Main.java +++ b/blur-shell/src/main/java/org/apache/blur/shell/Main.java @@ -99,7 +99,7 @@ public class Main { private static class CliShellOptions { - private String _controllerConnectionString; + private Iface _client; private boolean _shell; private String[] _args; private String _command; @@ -108,8 +108,8 @@ public class Main { return _shell; } - public String getControllerConnectionString() { - return _controllerConnectionString; + public Iface getClient() { + return _client; } public String getCommand() { @@ -120,8 +120,8 @@ public class Main { return _args; } - public void setControllerConnectionString(String controllerConnectionString) { - this._controllerConnectionString = controllerConnectionString; + public void setClient(Iface client) { + this._client = client; } public void setShell(boolean shell) { @@ -550,12 +550,12 @@ public class Main { } try { - Blur.Iface client = BlurClient.getClient(cliShellOptions.getControllerConnectionString()); + Blur.Iface client = cliShellOptions.getClient(); if (cliShellOptions.isShell()) { ConsoleReader reader = new ConsoleReader(); PrintWriter out = new PrintWriter(reader.getOutput()); setConsoleReader(commands, reader); - setPrompt(client, reader, cliShellOptions.getControllerConnectionString(), out); + setPrompt(client, reader, out); List<Completer> completors = new LinkedList<Completer>(); @@ -621,7 +621,7 @@ public class Main { out.println("Last command took " + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start) + "ms"); } } - setPrompt(client, reader, cliShellOptions.getControllerConnectionString(), out); + setPrompt(client, reader, out); } } } finally { @@ -704,18 +704,13 @@ public class Main { builder.put(command.name(), command); } - private static void setPrompt(Iface client, ConsoleReader reader, String connectionStr, PrintWriter out) + private static void setPrompt(Iface client, ConsoleReader reader, PrintWriter out) throws BlurException, TException, CommandException, IOException { List<String> shardClusterList; try { - shardClusterList = BlurClientManager.execute(connectionStr, new BlurCommand<List<String>>() { - @Override - public List<String> call(Client client) throws BlurException, TException { - return client.shardClusterList(); - } - }, 0, 0, 0); - } catch (BadConnectionException e) { - out.println(e.getMessage() + " Connection (" + connectionStr + ")"); + shardClusterList = client.shardClusterList(); + } catch (BlurException e) { + out.println("Unable to retrieve cluster information - " + e.getMessage()); out.flush(); if (debug) { e.printStackTrace(out); @@ -751,13 +746,7 @@ public class Main { private static CliShellOptions getCliShellOptions(String[] args) throws IOException { CliShellOptions cliShellOptions = new CliShellOptions(); if (args.length == 0) { - String controllerConnectionString = loadControllerConnectionString(); - if (controllerConnectionString == null) { - System.err - .println("Could not locate controller connection string in the blu-site.properties file and it was not passed in via command line args."); - return null; - } - cliShellOptions.setControllerConnectionString(controllerConnectionString); + cliShellOptions.setClient(BlurClient.getClient()); cliShellOptions.setShell(true); return cliShellOptions; } else { @@ -773,7 +762,7 @@ public class Main { + message + "]"); return null; } - cliShellOptions.setControllerConnectionString(arg0); + cliShellOptions.setClient(BlurClient.getClient(arg0)); if (args.length > 1) { // there's might be a command after the connection string cliShellOptions.setShell(false); @@ -794,13 +783,7 @@ public class Main { return cliShellOptions; } } else { - String controllerConnectionString = loadControllerConnectionString(); - if (controllerConnectionString == null) { - System.err - .println("Could not locate controller connection string in the blur-site.properties file and it was not passed in via command line args."); - return null; - } - cliShellOptions.setControllerConnectionString(controllerConnectionString); + cliShellOptions.setClient(BlurClient.getClient()); // command was found at arg0 cliShellOptions.setShell(false); cliShellOptions.setArgs(args); @@ -816,26 +799,4 @@ public class Main { } } - private static String loadControllerConnectionString() throws IOException { - StringBuilder builder = new StringBuilder(); - InputStream inputStream = Main.class.getResourceAsStream("/controllers"); - if (inputStream == null) { - return null; - } - BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream)); - String line; - while ((line = bufferedReader.readLine()) != null) { - if (builder.length() != 0) { - builder.append(','); - } - String trim = line.trim(); - if (trim.startsWith("#")) { - continue; - } - builder.append(trim).append(":40010"); - } - bufferedReader.close(); - return builder.toString(); - } - } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-thrift/src/main/java/org/apache/blur/thrift/BlurClient.java ---------------------------------------------------------------------- diff --git a/blur-thrift/src/main/java/org/apache/blur/thrift/BlurClient.java b/blur-thrift/src/main/java/org/apache/blur/thrift/BlurClient.java index 7d5e1d2..ad4813a 100644 --- a/blur-thrift/src/main/java/org/apache/blur/thrift/BlurClient.java +++ b/blur-thrift/src/main/java/org/apache/blur/thrift/BlurClient.java @@ -2,6 +2,7 @@ package org.apache.blur.thrift; /** * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 @@ -16,6 +17,11 @@ package org.apache.blur.thrift; * See the License for the specific language governing permissions and * limitations under the License. */ +import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_CONNECTION; +import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT; +import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT_DEFAULT; + +import java.io.IOException; import java.lang.reflect.InvocationHandler; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @@ -23,11 +29,19 @@ import java.lang.reflect.Proxy; import java.util.Arrays; import java.util.List; +import org.apache.blur.BlurConfiguration; import org.apache.blur.thirdparty.thrift_0_9_0.TException; import org.apache.blur.thrift.commands.BlurCommand; import org.apache.blur.thrift.generated.Blur.Client; import org.apache.blur.thrift.generated.Blur.Iface; import org.apache.blur.thrift.generated.BlurException; +import org.apache.blur.zookeeper.ZkUtils; +import org.apache.blur.zookeeper.ZookeeperPathConstants; +import org.apache.commons.lang.StringUtils; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooKeeper; + + public class BlurClient { @@ -77,6 +91,20 @@ public class BlurClient { } } + + public static Iface getClient() { + try { + return getClient(new BlurConfiguration()); + } catch (IOException e) { + throw new RuntimeException("Unable to load configurations.", e); + } + } + + public static Iface getClient(BlurConfiguration conf) { + List<String> onlineControllers = getOnlineControllers(conf); + + return getClient(StringUtils.join(onlineControllers, ",")); + } /** * Returns a client interface to Blur based on the connectionStr. @@ -123,5 +151,22 @@ public class BlurClient { return (Iface) Proxy.newProxyInstance(Iface.class.getClassLoader(), new Class[] { Iface.class }, new BlurClientInvocationHandler(connections, maxRetries, backOffTime, maxBackOffTime)); } + + private static List<String> getOnlineControllers(BlurConfiguration conf) { + String zkConn = conf.getExpected(BLUR_ZOOKEEPER_CONNECTION); + int zkSessionTimeout = conf.getInt(BLUR_ZOOKEEPER_TIMEOUT, BLUR_ZOOKEEPER_TIMEOUT_DEFAULT); + + ZooKeeper zkClient = null; + try { + zkClient = ZkUtils.newZooKeeper(zkConn, zkSessionTimeout); + return zkClient.getChildren(ZookeeperPathConstants.getOnlineControllersPath(), false); + } catch (KeeperException e) { + throw new RuntimeException("Error communicating with Zookeeper", e); + } catch (InterruptedException e) { + throw new RuntimeException("Error communicating with Zookeeper", e); + } catch (IOException e) { + throw new RuntimeException("Unable to initialize Zookeeper", e); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-util/src/main/java/org/apache/blur/BlurConfiguration.java ---------------------------------------------------------------------- diff --git a/blur-util/src/main/java/org/apache/blur/BlurConfiguration.java b/blur-util/src/main/java/org/apache/blur/BlurConfiguration.java index faeb11e..c6b256f 100644 --- a/blur-util/src/main/java/org/apache/blur/BlurConfiguration.java +++ b/blur-util/src/main/java/org/apache/blur/BlurConfiguration.java @@ -88,6 +88,14 @@ public class BlurConfiguration implements Cloneable { public String get(String name) { return get(name, null); } + + public String getExpected(String name) { + String val = get(name); + if (val == null || val.trim().isEmpty()) { + throw new IllegalArgumentException("Property [" + name + "] is missing or blank."); + } + return val; + } public String get(String name, String defaultValue) { String property = _properties.getProperty(name, defaultValue); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-util/src/main/java/org/apache/blur/zookeeper/ZookeeperPathConstants.java ---------------------------------------------------------------------- diff --git a/blur-util/src/main/java/org/apache/blur/zookeeper/ZookeeperPathConstants.java b/blur-util/src/main/java/org/apache/blur/zookeeper/ZookeeperPathConstants.java new file mode 100644 index 0000000..325ef6e --- /dev/null +++ b/blur-util/src/main/java/org/apache/blur/zookeeper/ZookeeperPathConstants.java @@ -0,0 +1,74 @@ +package org.apache.blur.zookeeper; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +public class ZookeeperPathConstants { + + public static String getBasePath() { + return "/blur"; + } + + public static String getClusterPath(String cluster) { + return getClustersPath() + "/" + cluster; + } + + public static String getClustersPath() { + return getBasePath() + "/clusters"; + } + + public static String getOnlineControllersPath() { + return getBasePath() + "/online-controller-nodes"; + } + + public static String getControllersPath() { + return getBasePath() + "/controller-nodes"; + } + + public static String getOnlineShardsPath(String cluster) { + return getClusterPath(cluster) + "/online-nodes"; + } + + public static String getTablesPath(String cluster) { + return getClusterPath(cluster) + "/tables"; + } + + public static String getTablePath(String cluster, String table) { + return getTablesPath(cluster) + "/" + table; + } + + public static String getSafemodePath(String cluster) { + return getClusterPath(cluster) + "/safemode"; + } + + public static String getShutdownPath(String cluster) { + return getClusterPath(cluster) + "/shutdown"; + } + + public static String getRegisteredShardsPath(String cluster) { + return getClusterPath(cluster) + "/registered-nodes"; + } + + public static String getShardLayoutPath(String cluster) { + return getClusterPath(cluster) + "/layout"; + } + + public static String getShardLayoutPathLocks(String cluster) { + return getShardLayoutPath(cluster) + "/locks"; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-util/src/main/resources/blur-default.properties ---------------------------------------------------------------------- diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties index 739f49e..0674f74 100644 --- a/blur-util/src/main/resources/blur-default.properties +++ b/blur-util/src/main/resources/blur-default.properties @@ -42,7 +42,7 @@ blur.shard.hostname= # The binding address of the shard blur.shard.bind.address=0.0.0.0 -# The default binding port of the shard server +# The default binding port of the shard server, 0 for random blur.shard.bind.port=40020 # The number of fetcher threads @@ -214,7 +214,7 @@ blur.controller.hostname= # The binding address of the controller blur.controller.bind.address=0.0.0.0 -# The default binding port of the controller server +# The default binding port of the controller server, 0 for random blur.controller.bind.port=40010 # The connection timeout, NOTE: this will be the maximum amount of time you can wait for a query.
