Repository: hbase Updated Branches: refs/heads/master 0d40a52ee -> 5bc518b38
HBASE-16774 [shell] Add coverage to TestShell when ZooKeeper is not reachable Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5bc518b3 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5bc518b3 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5bc518b3 Branch: refs/heads/master Commit: 5bc518b38717413c844968a29f817b5c99b9a136 Parents: 0d40a52 Author: Esteban Gutierrez <este...@apache.org> Authored: Wed Oct 5 15:25:54 2016 -0700 Committer: Esteban Gutierrez <este...@apache.org> Committed: Tue Oct 18 09:08:33 2016 -0700 ---------------------------------------------------------------------- .../hbase/client/ConnectionImplementation.java | 2 +- .../apache/hadoop/hbase/client/HBaseAdmin.java | 40 ++++++--- .../apache/hadoop/hbase/client/Registry.java | 4 +- .../hadoop/hbase/client/ZooKeeperRegistry.java | 7 +- .../hadoop/hbase/mapred/TableOutputFormat.java | 13 +++ .../hadoop/hbase/client/TestClientTimeouts.java | 8 +- .../hadoop/hbase/client/TestShellNoCluster.java | 60 +++++++++++++ .../ruby/hbase/test_connection_no_cluster.rb | 46 ++++++++++ .../src/test/ruby/no_cluster_tests_runner.rb | 92 ++++++++++++++++++++ 9 files changed, 247 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/5bc518b3/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java index 922168d..53eb522 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java @@ -439,7 +439,7 @@ class ConnectionImplementation implements ClusterConnection, Closeable { protected String clusterId = null; - protected void retrieveClusterId() throws IOException { + protected void retrieveClusterId() { if (clusterId != null) { return; } http://git-wip-us.apache.org/repos/asf/hbase/blob/5bc518b3/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index e7f6929..51d07e3 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -2112,10 +2112,12 @@ public class HBaseAdmin implements Admin { /** * Is HBase available? Throw an exception if not. * @param conf system configuration - * @throws ZooKeeperConnectionException if unable to connect to zookeeper] + * @throws MasterNotRunningException if the master is not running. + * @throws ZooKeeperConnectionException if unable to connect to zookeeper. + * // TODO do not expose ZKConnectionException. */ public static void available(final Configuration conf) - throws ZooKeeperConnectionException, InterruptedIOException { + throws MasterNotRunningException, ZooKeeperConnectionException, IOException { Configuration copyOfConf = HBaseConfiguration.create(conf); // We set it to make it fail as soon as possible if HBase is not available copyOfConf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1); @@ -2124,19 +2126,29 @@ public class HBaseAdmin implements Admin { // Check ZK first. // If the connection exists, we may have a connection to ZK that does not work anymore try (ClusterConnection connection = - (ClusterConnection) ConnectionFactory.createConnection(copyOfConf); - ZooKeeperKeepAliveConnection zkw = ((ConnectionImplementation) connection). - getKeepAliveZooKeeperWatcher();) { - // This is NASTY. FIX!!!! Dependent on internal implementation! TODO - zkw.getRecoverableZooKeeper().getZooKeeper().exists(zkw.znodePaths.baseZNode, false); + (ClusterConnection) ConnectionFactory.createConnection(copyOfConf)) { + // Check ZK first. + // If the connection exists, we may have a connection to ZK that does not work anymore + ZooKeeperKeepAliveConnection zkw = null; + try { + // This is NASTY. FIX!!!! Dependent on internal implementation! TODO + zkw = ((ConnectionImplementation) connection) + .getKeepAliveZooKeeperWatcher(); + zkw.getRecoverableZooKeeper().getZooKeeper().exists(zkw.znodePaths.baseZNode, false); + } catch (IOException e) { + throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e); + } catch (InterruptedException e) { + throw (InterruptedIOException) + new InterruptedIOException("Can't connect to ZooKeeper").initCause(e); + } catch (KeeperException e){ + throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e); + } finally { + if (zkw != null) { + zkw.close(); + } + } + // can throw MasterNotRunningException connection.isMasterRunning(); - } catch (IOException e) { - throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e); - } catch (InterruptedException e) { - throw (InterruptedIOException) - new InterruptedIOException("Can't connect to ZooKeeper").initCause(e); - } catch (KeeperException e) { - throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/5bc518b3/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Registry.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Registry.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Registry.java index d20a4bd..412e4fa 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Registry.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Registry.java @@ -32,7 +32,7 @@ interface Registry { /** * @param connection */ - void init(Connection connection) throws IOException; + void init(Connection connection); /** * @return Meta region location @@ -43,7 +43,7 @@ interface Registry { /** * @return Cluster id. */ - String getClusterId() throws IOException; + String getClusterId(); /** * @return Count of 'running' regionservers http://git-wip-us.apache.org/repos/asf/hbase/blob/5bc518b3/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZooKeeperRegistry.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZooKeeperRegistry.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZooKeeperRegistry.java index 9fca027..f38a3d3 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZooKeeperRegistry.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZooKeeperRegistry.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.RegionLocations; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; import org.apache.hadoop.hbase.zookeeper.ZKClusterId; import org.apache.hadoop.hbase.zookeeper.ZKUtil; @@ -34,6 +35,7 @@ import org.apache.zookeeper.KeeperException; /** * A cluster registry that stores to zookeeper. */ +@InterfaceAudience.Private class ZooKeeperRegistry implements Registry { private static final Log LOG = LogFactory.getLog(ZooKeeperRegistry.class); // Needs an instance of hci to function. Set after construct this instance. @@ -50,7 +52,6 @@ class ZooKeeperRegistry implements Registry { @Override public RegionLocations getMetaRegionLocation() throws IOException { ZooKeeperKeepAliveConnection zkw = hci.getKeepAliveZooKeeperWatcher(); - try { if (LOG.isTraceEnabled()) { LOG.trace("Looking up meta region location in ZK," + " connection=" + this); @@ -92,7 +93,7 @@ class ZooKeeperRegistry implements Registry { private String clusterId = null; @Override - public String getClusterId() throws IOException { + public String getClusterId() { if (this.clusterId != null) return this.clusterId; // No synchronized here, worse case we will retrieve it twice, that's // not an issue. @@ -105,10 +106,8 @@ class ZooKeeperRegistry implements Registry { } } catch (KeeperException e) { LOG.warn("Can't retrieve clusterId from ZooKeeper", e); - throw new IOException("ZooKeeperException ", e); } catch (IOException e) { LOG.warn("Can't retrieve clusterId from ZooKeeper", e); - throw e; } finally { if (zkw != null) zkw.close(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/5bc518b3/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableOutputFormat.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableOutputFormat.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableOutputFormat.java index 0c5d5f6..3fe5a90 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableOutputFormat.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableOutputFormat.java @@ -55,6 +55,19 @@ public class TableOutputFormat extends FileOutputFormat<ImmutableBytesWritable, private BufferedMutator m_mutator; private Connection conn; + + /** + * Instantiate a TableRecordWriter with the HBase HClient for writing. + * + * @deprecated Please use {@code #TableRecordWriter(JobConf)} This version does not clean up + * connections and will leak connections (removed in 2.0) + */ + @Deprecated + public TableRecordWriter(final BufferedMutator mutator) throws IOException { + this.m_mutator = mutator; + this.conn = null; + } + /** * Instantiate a TableRecordWriter with a BufferedMutator for batch writing. */ http://git-wip-us.apache.org/repos/asf/hbase/blob/5bc518b3/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestClientTimeouts.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestClientTimeouts.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestClientTimeouts.java index 684d339..8f2a222 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestClientTimeouts.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestClientTimeouts.java @@ -33,8 +33,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MasterNotRunningException; import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.ZooKeeperConnectionException; import org.apache.hadoop.hbase.ipc.AbstractRpcClient; import org.apache.hadoop.hbase.ipc.RpcClientFactory; import org.apache.hadoop.hbase.ipc.BlockingRpcClient; @@ -103,9 +103,9 @@ public class TestClientTimeouts { // run some admin commands HBaseAdmin.available(conf); admin.setBalancerRunning(false, false); - } catch (ZooKeeperConnectionException ex) { + } catch (MasterNotRunningException ex) { // Since we are randomly throwing SocketTimeoutExceptions, it is possible to get - // a ZooKeeperConnectionException. It's a bug if we get other exceptions. + // a MasterNotRunningException. It's a bug if we get other exceptions. lastFailed = true; } finally { if(admin != null) { @@ -173,4 +173,4 @@ public class TestClientTimeouts { return super.callBlockingMethod(md, controller, param, returnType); } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hbase/blob/5bc518b3/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestShellNoCluster.java ---------------------------------------------------------------------- diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestShellNoCluster.java b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestShellNoCluster.java new file mode 100644 index 0000000..3f11853 --- /dev/null +++ b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestShellNoCluster.java @@ -0,0 +1,60 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.hbase.testclassification.ClientTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.jruby.embed.PathType; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +@Category({ ClientTests.class, LargeTests.class }) +public class TestShellNoCluster extends AbstractTestShell { + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + // no cluster + List<String> loadPaths = new ArrayList(); + loadPaths.add("src/main/ruby"); + loadPaths.add("src/test/ruby"); + jruby.getProvider().setLoadPaths(loadPaths); + jruby.put("$TEST_CLUSTER", TEST_UTIL); + System.setProperty("jruby.jit.logging.verbose", "true"); + System.setProperty("jruby.jit.logging", "true"); + System.setProperty("jruby.native.verbose", "true"); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + // no cluster + } + + @Test + public void testRunNoClusterShellTests() throws IOException { + // Start ruby tests without cluster + jruby.runScriptlet(PathType.ABSOLUTE, "src/test/ruby/no_cluster_tests_runner.rb"); + } + +} http://git-wip-us.apache.org/repos/asf/hbase/blob/5bc518b3/hbase-shell/src/test/ruby/hbase/test_connection_no_cluster.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/test/ruby/hbase/test_connection_no_cluster.rb b/hbase-shell/src/test/ruby/hbase/test_connection_no_cluster.rb new file mode 100644 index 0000000..d240ff8 --- /dev/null +++ b/hbase-shell/src/test/ruby/hbase/test_connection_no_cluster.rb @@ -0,0 +1,46 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'shell' +require 'stringio' +require 'hbase_constants' +require 'hbase/hbase' +require 'hbase/table' + +include HBaseConstants + +module Hbase + class NoClusterConnectionTest < Test::Unit::TestCase + include TestHelpers + + def setup + puts "starting shell" + end + + def teardown + # nothing to teardown + end + + define_test "start_hbase_shell_no_cluster" do + assert_nothing_raised do + setup_hbase + end + end + end +end http://git-wip-us.apache.org/repos/asf/hbase/blob/5bc518b3/hbase-shell/src/test/ruby/no_cluster_tests_runner.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/test/ruby/no_cluster_tests_runner.rb b/hbase-shell/src/test/ruby/no_cluster_tests_runner.rb new file mode 100644 index 0000000..77b16df --- /dev/null +++ b/hbase-shell/src/test/ruby/no_cluster_tests_runner.rb @@ -0,0 +1,92 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'rubygems' +require 'rake' +require 'set' + + +# This runner will only launch shell tests that don't require a HBase cluster running. + +unless defined?($TEST_CLUSTER) + include Java + + # Set logging level to avoid verboseness + org.apache.log4j.Logger.getRootLogger.setLevel(org.apache.log4j.Level::OFF) + org.apache.log4j.Logger.getLogger("org.apache.zookeeper").setLevel(org.apache.log4j.Level::OFF) + org.apache.log4j.Logger.getLogger("org.apache.hadoop.hdfs").setLevel(org.apache.log4j.Level::OFF) + org.apache.log4j.Logger.getLogger("org.apache.hadoop.hbase").setLevel(org.apache.log4j.Level::OFF) + org.apache.log4j.Logger.getLogger("org.apache.hadoop.ipc.HBaseServer").setLevel(org.apache.log4j.Level::OFF) + + java_import org.apache.hadoop.hbase.HBaseTestingUtility + + $TEST_CLUSTER = HBaseTestingUtility.new + $TEST_CLUSTER.configuration.setInt("hbase.regionserver.msginterval", 100) + $TEST_CLUSTER.configuration.setInt("hbase.client.pause", 250) + $TEST_CLUSTER.configuration.setInt(org.apache.hadoop.hbase.HConstants::HBASE_CLIENT_RETRIES_NUMBER, 6) +end + +require 'test_helper' + +puts "Running tests without a cluster..." + +if java.lang.System.get_property('shell.test.include') + includes = Set.new(java.lang.System.get_property('shell.test.include').split(',')) +end + +if java.lang.System.get_property('shell.test.exclude') + excludes = Set.new(java.lang.System.get_property('shell.test.exclude').split(',')) +end + +files = Dir[ File.dirname(__FILE__) + "/**/*_no_cluster.rb" ] +files.each do |file| + filename = File.basename(file) + if includes != nil && !includes.include?(filename) + puts "Skip #{filename} because of not included" + next + end + if excludes != nil && excludes.include?(filename) + puts "Skip #{filename} because of excluded" + next + end + begin + load(file) + rescue => e + puts "ERROR: #{e}" + raise + end +end + +# If this system property is set, we'll use it to filter the test cases. +runner_args = [] +if java.lang.System.get_property('shell.test') + shell_test_pattern = java.lang.System.get_property('shell.test') + puts "Only running tests that match #{shell_test_pattern}" + runner_args << "--testcase=#{shell_test_pattern}" +end +# first couple of args are to match the defaults, so we can pass options to limit the tests run +if !(Test::Unit::AutoRunner.run(false, nil, runner_args)) + raise "Shell unit tests failed. Check output file for details." +end + +puts "Done with tests! Shutting down the cluster..." +if @own_cluster + $TEST_CLUSTER.shutdownMiniCluster + java.lang.System.exit(0) +end
