Repository: incubator-blur Updated Branches: refs/heads/master 36e3c85d7 -> 87d29a17d
Removing dependency on ZookeeperClient Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/2453507d Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/2453507d Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/2453507d Branch: refs/heads/master Commit: 2453507da6e14a24d7864056a4a624c5531a8d32 Parents: 36e3c85 Author: Chris Rohr <[email protected]> Authored: Fri Sep 26 07:08:15 2014 -0400 Committer: Chris Rohr <[email protected]> Committed: Thu Oct 2 07:18:31 2014 -0400 ---------------------------------------------------------------------- blur-console/pom.xml | 9 ++- .../blur/console/util/CachingBlurClient.java | 27 +++++++- .../org/apache/blur/console/util/Config.java | 41 +++-------- .../org/apache/blur/console/util/NodeUtil.java | 71 +++++++++++--------- .../apache/blur/console/ConsoleTestBase.java | 4 ++ .../apache/blur/console/ConsoleTestSuite.java | 6 +- .../org/apache/blur/console/RunMiniCluster.java | 9 ++- .../apache/blur/console/util/ConfigTest.java | 19 +----- .../apache/blur/console/util/NodeUtilTest.java | 9 +-- .../apache/blur/console/util/QueryUtilTest.java | 25 ++++--- 10 files changed, 122 insertions(+), 98 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2453507d/blur-console/pom.xml ---------------------------------------------------------------------- diff --git a/blur-console/pom.xml b/blur-console/pom.xml index 88a6fa1..0d82e2c 100644 --- a/blur-console/pom.xml +++ b/blur-console/pom.xml @@ -43,11 +43,11 @@ <artifactId>blur-thrift</artifactId> <version>${project.version}</version> </dependency> - <dependency> +<!-- <dependency> <groupId>org.apache.blur</groupId> <artifactId>blur-core</artifactId> <version>${project.version}</version> - </dependency> + </dependency> --> <dependency> <groupId>org.apache.blur</groupId> <artifactId>blur-util</artifactId> @@ -78,6 +78,11 @@ <!-- Test Dependancies --> <dependency> + <groupId>org.apache.blur</groupId> + <artifactId>blur-core</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> <groupId>org.apache.blur</groupId> <artifactId>blur-core</artifactId> <version>${project.version}</version> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2453507d/blur-console/src/main/java/org/apache/blur/console/util/CachingBlurClient.java ---------------------------------------------------------------------- diff --git a/blur-console/src/main/java/org/apache/blur/console/util/CachingBlurClient.java b/blur-console/src/main/java/org/apache/blur/console/util/CachingBlurClient.java index 3d1aca6..f722330 100644 --- a/blur-console/src/main/java/org/apache/blur/console/util/CachingBlurClient.java +++ b/blur-console/src/main/java/org/apache/blur/console/util/CachingBlurClient.java @@ -43,6 +43,8 @@ public class CachingBlurClient { private Map<String, Item> tableDescriptionCache = new HashMap<String, Item>(); private Map<String, Item> tableStatsCache = new HashMap<String, Item>(); private Map<String, Item> schemaCache = new HashMap<String, Item>(); + private Map<String, Item> controllerListCache = new HashMap<String, Item>(); + private Map<String, Item> shardListCache = new HashMap<String, Item>(); private long cacheHits; private long cacheMisses; @@ -76,6 +78,8 @@ public class CachingBlurClient { cleanup(tableDescriptionCache); cleanup(tableStatsCache); cleanup(schemaCache); + cleanup(controllerListCache); + cleanup(shardListCache); log.info("Cache: " + cacheHits + " hits, " + cacheMisses + " misses"); try { Thread.sleep(timeout * 2); @@ -105,9 +109,10 @@ public class CachingBlurClient { } private Iface getClient() { - return BlurClient.getClient(Config.getConnectionString()); + return BlurClient.getClient(Config.getBlurConfig()); } + @SuppressWarnings("unchecked") private <T extends Object> T getFromCache(String key, Map<String, Item> cache, Retriever<T> retriever) throws TException { synchronized (cache) { if(this.timeout <= 0) { @@ -139,6 +144,7 @@ public class CachingBlurClient { getClient().addColumnDefinition(table, def); } + @SuppressWarnings("unchecked") private void invalidateQuery(String table, String uuid) { synchronized (queryListCache) { Item item = queryListCache.get(null); @@ -151,6 +157,7 @@ public class CachingBlurClient { } } + @SuppressWarnings("unchecked") private void invalidateTable(String table) { synchronized (tableListCache) { for(Item item: tableListCache.values()) { @@ -189,6 +196,24 @@ public class CachingBlurClient { } }); } + + public List<String> controllerList() throws TException { + return getFromCache(null, controllerListCache, new Retriever<List<String>>() { + @Override + public List<String> retrieve() throws TException { + return getClient().controllerServerList(); + } + }); + } + + public List<String> shardList(final String cluster) throws TException { + return getFromCache(cluster, shardListCache, new Retriever<List<String>>() { + @Override + public List<String> retrieve() throws TException { + return getClient().shardServerList(cluster); + } + }); + } public List<String> queryStatusIdList(final String table) throws TException { return getFromCache(table, queryListCache, new Retriever<List<String>>() { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2453507d/blur-console/src/main/java/org/apache/blur/console/util/Config.java ---------------------------------------------------------------------- diff --git a/blur-console/src/main/java/org/apache/blur/console/util/Config.java b/blur-console/src/main/java/org/apache/blur/console/util/Config.java index a9dc634..83d7ea7 100644 --- a/blur-console/src/main/java/org/apache/blur/console/util/Config.java +++ b/blur-console/src/main/java/org/apache/blur/console/util/Config.java @@ -18,29 +18,25 @@ package org.apache.blur.console.util; */ +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Method; +import java.util.Map; +import java.util.Properties; + import org.apache.blur.BlurConfiguration; import org.apache.blur.console.providers.AuthenticationDenied; import org.apache.blur.console.providers.EmptyAuthorization; import org.apache.blur.console.providers.IAuthenticationProvider; import org.apache.blur.console.providers.IAuthorizationProvider; -import org.apache.blur.manager.clusterstatus.ZookeeperClusterStatus; import org.apache.blur.thrift.BlurClient; import org.apache.blur.thrift.generated.Blur.Iface; import org.apache.blur.user.User; import org.apache.blur.user.UserContext; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import java.io.File; -import java.io.IOException; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Properties; - public class Config { private static final File TMPDIR = new File(System.getProperty("blur.tmp.dir", "./target/mini-cluster")); @@ -50,8 +46,6 @@ public class Config { private static int port; private static BlurConfiguration blurConfig; - private static ZookeeperClusterStatus zk; - private static String blurConnection; private static Object cluster; private static IAuthenticationProvider authenticationProvider; private static IAuthorizationProvider authorizationProvider; @@ -73,8 +67,6 @@ public class Config { setDevelopmentZookeeperConnection(); setDevelopmentProperties(); } - zk = new ZookeeperClusterStatus(blurConfig.get("blur.zookeeper.connection"), blurConfig); - blurConnection = buildConnectionString(); port = blurConfig.getInt("blur.console.port", DEFAULT_PORT); cachingBlurClient = new CachingBlurClient(blurConfig.getInt("blur.console.refreshtime", DEFAULT_REFRESH_TIME)); setupProviders(); @@ -101,11 +93,11 @@ public class Config { } } + @SuppressWarnings("rawtypes") private static void setupProviders() throws Exception { String authenticationProviderClassName = blurConfig.get("blur.console.authentication.provider", "org.apache.blur.console.providers.AllAuthenticated"); String authorizationProviderClassName = blurConfig.get("blur.console.authorization.provider"); - Class authenticationProviderClass = Class.forName(authenticationProviderClassName, false, Config.class.getClassLoader()); if (authenticationProviderClass == null) { @@ -138,20 +130,6 @@ public class Config { } } - public static String getConnectionString() { - return blurConnection; - } - - public static ZookeeperClusterStatus getZookeeper() { - return zk; - } - - private static String buildConnectionString() { - List<String> allControllers = new ArrayList<String>(); - allControllers = zk.getControllerServerList(); - return StringUtils.join(allControllers, ","); - } - public static void shutdownMiniCluster() throws IOException { if (cluster != null) { try { @@ -164,6 +142,7 @@ public class Config { if (file.exists()) { FileUtils.deleteDirectory(file); } + cluster = null; } } @@ -193,7 +172,7 @@ public class Config { } public static Iface getClient(org.apache.blur.console.model.User user, String securityUser) throws IOException { - Iface client = BlurClient.getClient(getConnectionString()); + Iface client = BlurClient.getClient(blurConfig); Map<String, String> securityAttributes = user.getSecurityAttributes(securityUser); if (securityAttributes != null) { UserContext.setUser(new User(user.getName(), securityAttributes)); @@ -206,7 +185,7 @@ public class Config { } public static Iface getClient() { - return BlurClient.getClient(getConnectionString()); + return BlurClient.getClient(blurConfig); } public static CachingBlurClient getCachingBlurClient() { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2453507d/blur-console/src/main/java/org/apache/blur/console/util/NodeUtil.java ---------------------------------------------------------------------- diff --git a/blur-console/src/main/java/org/apache/blur/console/util/NodeUtil.java b/blur-console/src/main/java/org/apache/blur/console/util/NodeUtil.java index 3a1bb1a..08e20a4 100644 --- a/blur-console/src/main/java/org/apache/blur/console/util/NodeUtil.java +++ b/blur-console/src/main/java/org/apache/blur/console/util/NodeUtil.java @@ -17,10 +17,6 @@ package org.apache.blur.console.util; -import org.apache.blur.manager.clusterstatus.ZookeeperClusterStatus; -import org.apache.commons.collections.CollectionUtils; -import org.apache.blur.thrift.generated.Blur.Iface; - import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -28,47 +24,62 @@ import java.net.InetSocketAddress; import java.net.Socket; import java.net.URI; import java.nio.ByteBuffer; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.blur.thirdparty.thrift_0_9_0.TException; +import org.apache.commons.collections.CollectionUtils; public class NodeUtil { + private static Set<String> onlineControllers = new HashSet<String>(); + private static Map<String, Set<String>> onlineShards = new HashMap<String, Set<String>>(); @SuppressWarnings("unchecked") - public static Map<String, Object> getControllerStatus() throws IOException { - Iface client = Config.getClient(); - ZookeeperClusterStatus zk = Config.getZookeeper(); - - List<String> allControllers = new ArrayList<String>(); - List<String> oControllers = new ArrayList<String>(); - allControllers = zk.getOnlineControllerList(); - oControllers = zk.getControllerServerList(); - - Collection<String> onlineControllers = CollectionUtils.intersection(allControllers, oControllers); - Collection<String> offlineControllers = CollectionUtils.subtract(allControllers, oControllers); + public static Map<String, Object> getControllerStatus() throws IOException, TException { + CachingBlurClient client = Config.getCachingBlurClient(); + + List<String> currentControllers = client.controllerList(); + + Collection<String> newControllers = CollectionUtils.subtract(currentControllers, onlineControllers); + Collection<String> missingControllers = CollectionUtils.subtract(onlineControllers, currentControllers); + + onlineControllers.addAll(newControllers); Map<String, Object> data = new HashMap<String, Object>(); data.put("online", onlineControllers); - data.put("offline", offlineControllers); + data.put("offline", missingControllers); return data; } - public static List<Map<String, Object>> getClusterStatus() throws IOException { - ZookeeperClusterStatus zk = Config.getZookeeper(); - + @SuppressWarnings("unchecked") + public static List<Map<String, Object>> getClusterStatus() throws IOException, TException { List<Map<String, Object>> data = new ArrayList<Map<String, Object>>(); - List<String> clusters = zk.getClusterList(false); - - for (String cluster : clusters) { + CachingBlurClient client = Config.getCachingBlurClient(); + + for (String cluster : client.shardClusterList()) { + if (!onlineShards.containsKey(cluster)) { + onlineShards.put(cluster, new HashSet<String>()); + } + + Set<String> onlineShardsForCluster = onlineShards.get(cluster); + List<String> currentShards = client.shardList(cluster); + + Collection<String> newShards = CollectionUtils.subtract(currentShards, onlineShardsForCluster); + Collection<String> missingShards = CollectionUtils.subtract(onlineShardsForCluster, currentShards); + + onlineShardsForCluster.addAll(newShards); + Map<String, Object> clusterObj = new HashMap<String, Object>(); clusterObj.put("name", cluster); - - List<String> offlineShardServers = zk.getOfflineShardServers(false, cluster); - List<String> onlineShardServers = zk.getOnlineShardServers(false, cluster); - - clusterObj.put("online", onlineShardServers); - clusterObj.put("offline", offlineShardServers); - + clusterObj.put("online", onlineShardsForCluster); + clusterObj.put("offline", missingShards); data.add(clusterObj); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2453507d/blur-console/src/test/java/org/apache/blur/console/ConsoleTestBase.java ---------------------------------------------------------------------- diff --git a/blur-console/src/test/java/org/apache/blur/console/ConsoleTestBase.java b/blur-console/src/test/java/org/apache/blur/console/ConsoleTestBase.java index c49d591..1721196 100644 --- a/blur-console/src/test/java/org/apache/blur/console/ConsoleTestBase.java +++ b/blur-console/src/test/java/org/apache/blur/console/ConsoleTestBase.java @@ -18,6 +18,8 @@ package org.apache.blur.console; import org.apache.blur.console.util.Config; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -27,6 +29,7 @@ import java.io.IOException; public class ConsoleTestBase { protected static String TABLE_PATH = new File("./target/tmp/test-data/test-tables").getAbsolutePath(); private static boolean _managing; + private static Log log = LogFactory.getLog(ConsoleTestBase.class); @BeforeClass public static void startup() throws IOException { @@ -55,6 +58,7 @@ public class ConsoleTestBase { @AfterClass public static void shutdown() throws IOException { if (_managing) { + log.warn("SHUTTING DOWN MINI CLUSTER"); Config.shutdownMiniCluster(); } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2453507d/blur-console/src/test/java/org/apache/blur/console/ConsoleTestSuite.java ---------------------------------------------------------------------- diff --git a/blur-console/src/test/java/org/apache/blur/console/ConsoleTestSuite.java b/blur-console/src/test/java/org/apache/blur/console/ConsoleTestSuite.java index 9afd1a4..9b6b833 100644 --- a/blur-console/src/test/java/org/apache/blur/console/ConsoleTestSuite.java +++ b/blur-console/src/test/java/org/apache/blur/console/ConsoleTestSuite.java @@ -1,6 +1,8 @@ package org.apache.blur.console; import org.apache.blur.console.util.*; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.junit.*; import org.junit.rules.ExternalResource; import org.junit.runner.RunWith; @@ -28,7 +30,8 @@ import java.io.IOException; @RunWith(Suite.class) @Suite.SuiteClasses({ConfigTest.class, NodeUtilTest.class, QueryUtilTest.class, TableUtilTest.class}) public class ConsoleTestSuite { - + private static Log log = LogFactory.getLog(ConsoleTestSuite.class); + @ClassRule public static ExternalResource testCluster = new ExternalResource() { @@ -38,6 +41,7 @@ public class ConsoleTestSuite { protected void after() { if (_managing) { try { + log.warn("SHUTTING DOWN MINI CLUSTER"); Config.shutdownMiniCluster(); } catch (IOException e) { throw new RuntimeException(e); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2453507d/blur-console/src/test/java/org/apache/blur/console/RunMiniCluster.java ---------------------------------------------------------------------- diff --git a/blur-console/src/test/java/org/apache/blur/console/RunMiniCluster.java b/blur-console/src/test/java/org/apache/blur/console/RunMiniCluster.java index a595352..0e62097 100644 --- a/blur-console/src/test/java/org/apache/blur/console/RunMiniCluster.java +++ b/blur-console/src/test/java/org/apache/blur/console/RunMiniCluster.java @@ -16,8 +16,10 @@ */ package org.apache.blur.console; +import java.io.File; +import java.io.IOException; + import org.apache.blur.MiniCluster; -import org.apache.blur.utils.GCWatcher; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; @@ -25,15 +27,12 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; -import java.io.File; -import java.io.IOException; - public class RunMiniCluster { private static final File TMPDIR = new File(System.getProperty("blur.tmp.dir", "./target/mini-cluster")); public static void main(String[] args) throws IOException { - GCWatcher.init(0.60); + //GCWatcher.init(0.60); LocalFileSystem localFS = FileSystem.getLocal(new Configuration()); File testDirectory = new File(TMPDIR, "blur-cluster-test").getAbsoluteFile(); testDirectory.mkdirs(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2453507d/blur-console/src/test/java/org/apache/blur/console/util/ConfigTest.java ---------------------------------------------------------------------- diff --git a/blur-console/src/test/java/org/apache/blur/console/util/ConfigTest.java b/blur-console/src/test/java/org/apache/blur/console/util/ConfigTest.java index 797afa3..45c5fd0 100644 --- a/blur-console/src/test/java/org/apache/blur/console/util/ConfigTest.java +++ b/blur-console/src/test/java/org/apache/blur/console/util/ConfigTest.java @@ -16,17 +16,14 @@ */ package org.apache.blur.console.util; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + import org.apache.blur.BlurConfiguration; import org.apache.blur.console.ConsoleTestBase; -import org.apache.commons.lang.StringUtils; import org.junit.Before; import org.junit.Test; -import java.io.IOException; -import java.net.InetAddress; - -import static org.junit.Assert.*; - public class ConfigTest extends ConsoleTestBase { @Before @@ -44,14 +41,4 @@ public class ConfigTest extends ConsoleTestBase { BlurConfiguration blurConfig = Config.getBlurConfig(); assertNotNull(blurConfig); } - - @Test - public void testGetConnectionString() throws IOException { - assertTrue(StringUtils.contains(Config.getConnectionString(), InetAddress.getLocalHost().getHostName())); - } - - @Test - public void testGetZookeeper() { - assertNotNull(Config.getZookeeper()); - } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2453507d/blur-console/src/test/java/org/apache/blur/console/util/NodeUtilTest.java ---------------------------------------------------------------------- diff --git a/blur-console/src/test/java/org/apache/blur/console/util/NodeUtilTest.java b/blur-console/src/test/java/org/apache/blur/console/util/NodeUtilTest.java index 0c86126..0c4f953 100644 --- a/blur-console/src/test/java/org/apache/blur/console/util/NodeUtilTest.java +++ b/blur-console/src/test/java/org/apache/blur/console/util/NodeUtilTest.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Collection; import static org.junit.Assert.assertEquals; @@ -40,8 +41,8 @@ public class NodeUtilTest extends ConsoleTestBase { public void testGetControllerStatus() throws BlurException, IOException, TException { Map<String, Object> nodeStatus = NodeUtil.getControllerStatus(); - assertEquals(2, ((List<String>) nodeStatus.get("online")).size()); - assertEquals(0, ((List<String>) nodeStatus.get("offline")).size()); + assertEquals(2, ((Collection<String>) nodeStatus.get("online")).size()); + assertEquals(0, ((Collection<String>) nodeStatus.get("offline")).size()); } @SuppressWarnings("unchecked") @@ -58,7 +59,7 @@ public class NodeUtilTest extends ConsoleTestBase { public void testGetClusterStatus() throws BlurException, IOException, TException { List<Map<String, Object>> nodeStatus = NodeUtil.getClusterStatus(); - assertEquals(3, ((List<String>) nodeStatus.get(0).get("online")).size()); - assertEquals(0, ((List<String>) nodeStatus.get(0).get("offline")).size()); + assertEquals(3, ((Collection<String>) nodeStatus.get(0).get("online")).size()); + assertEquals(0, ((Collection<String>) nodeStatus.get(0).get("offline")).size()); } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2453507d/blur-console/src/test/java/org/apache/blur/console/util/QueryUtilTest.java ---------------------------------------------------------------------- diff --git a/blur-console/src/test/java/org/apache/blur/console/util/QueryUtilTest.java b/blur-console/src/test/java/org/apache/blur/console/util/QueryUtilTest.java index 9642ea0..dcc2d9a 100644 --- a/blur-console/src/test/java/org/apache/blur/console/util/QueryUtilTest.java +++ b/blur-console/src/test/java/org/apache/blur/console/util/QueryUtilTest.java @@ -17,13 +17,7 @@ package org.apache.blur.console.util; -import org.apache.blur.console.ConsoleTestBase; -import org.apache.blur.thirdparty.thrift_0_9_0.TException; -import org.apache.blur.thrift.BlurClient; -import org.apache.blur.thrift.generated.*; -import org.apache.blur.thrift.generated.Blur.Iface; -import org.junit.Before; -import org.junit.Test; +import static org.junit.Assert.assertEquals; import java.io.IOException; import java.util.Arrays; @@ -31,7 +25,22 @@ import java.util.List; import java.util.Map; import java.util.UUID; -import static org.junit.Assert.assertEquals; +import org.apache.blur.console.ConsoleTestBase; +import org.apache.blur.thirdparty.thrift_0_9_0.TException; +import org.apache.blur.thrift.generated.Blur.Iface; +import org.apache.blur.thrift.generated.BlurException; +import org.apache.blur.thrift.generated.BlurQuery; +import org.apache.blur.thrift.generated.Column; +import org.apache.blur.thrift.generated.Query; +import org.apache.blur.thrift.generated.Record; +import org.apache.blur.thrift.generated.RecordMutation; +import org.apache.blur.thrift.generated.RecordMutationType; +import org.apache.blur.thrift.generated.RowMutation; +import org.apache.blur.thrift.generated.RowMutationType; +import org.apache.blur.thrift.generated.ScoreType; +import org.apache.blur.thrift.generated.TableDescriptor; +import org.junit.Before; +import org.junit.Test; public class QueryUtilTest extends ConsoleTestBase { @Before
