HIVE-13149: Remove some unnecessary HMS connections from HS2 (Reviewed by Szehon Ho, Chaoyu Tang)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/37e6e1bf Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/37e6e1bf Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/37e6e1bf Branch: refs/heads/llap Commit: 37e6e1bf56d7d2fd557730380b147c745fc051ce Parents: 1cb4ce8 Author: Aihua Xu <aihu...@apache.org> Authored: Fri Mar 18 10:27:49 2016 -0400 Committer: Aihua Xu <aihu...@apache.org> Committed: Mon Apr 11 14:58:59 2016 -0400 ---------------------------------------------------------------------- .../hadoop/hive/metastore/TestMetastoreVersion.java | 7 ++++--- .../metastore/hbase/TestHBaseMetastoreMetrics.java | 4 +--- .../org/apache/hadoop/hive/hbase/HBaseQTestUtil.java | 10 +++++++++- .../org/apache/hadoop/hive/hbase/HBaseTestSetup.java | 3 --- .../java/org/apache/hadoop/hive/ql/QTestUtil.java | 14 ++++++++++---- .../hadoop/hive/metastore/HiveMetaStoreClient.java | 10 ++++++---- .../apache/hadoop/hive/ql/session/SessionState.java | 8 -------- 7 files changed, 30 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/37e6e1bf/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java index 53f0d0e..5ceb3d2 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hive.metastore; import java.io.File; import java.lang.reflect.Field; -import java.util.Random; import junit.framework.TestCase; @@ -32,6 +31,7 @@ import org.apache.hive.common.util.HiveStringUtils; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.ObjectStore; import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; @@ -96,8 +96,9 @@ public class TestMetastoreVersion extends TestCase { // session creation should fail since the schema didn't get created try { SessionState.start(new CliSessionState(hiveConf)); - fail("Expected exception"); - } catch (RuntimeException re) { + Hive.get(hiveConf).getMSC(); + fail("An exception is expected since schema is not created."); + } catch (Exception re) { LOG.info("Exception in testVersionRestriction: " + re, re); String msg = HiveStringUtils.stringifyException(re); assertTrue("Expected 'Version information not found in metastore' in: " + msg, msg http://git-wip-us.apache.org/repos/asf/hive/blob/37e6e1bf/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java index 3ed88f2..aefafe0 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java @@ -41,8 +41,6 @@ import java.io.IOException; */ public class TestHBaseMetastoreMetrics extends HBaseIntegrationTests { - private CodahaleMetrics metrics; - @BeforeClass public static void startup() throws Exception { HBaseIntegrationTests.startMiniCluster(); @@ -66,7 +64,6 @@ public class TestHBaseMetastoreMetrics extends HBaseIntegrationTests { conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, MetricsReporting.JSON_FILE.name() + "," + MetricsReporting.JMX.name()); SessionState.start(new CliSessionState(conf)); driver = new Driver(conf); - metrics = (CodahaleMetrics) MetricsFactory.getInstance(); } @Test @@ -107,6 +104,7 @@ public class TestHBaseMetastoreMetrics extends HBaseIntegrationTests { driver.run("use default"); driver.run("drop database tempdb cascade"); + CodahaleMetrics metrics = (CodahaleMetrics) MetricsFactory.getInstance(); String json = metrics.dumpJson(); MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, MetricsConstant.CREATE_TOTAL_DATABASES, 2); MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, MetricsConstant.CREATE_TOTAL_TABLES, 7); http://git-wip-us.apache.org/repos/asf/hive/blob/37e6e1bf/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java index 3ff5742..70c0b13 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java @@ -39,12 +39,14 @@ public class HBaseQTestUtil extends QTestUtil { /** A handle to this harness's cluster */ private final HConnection conn; + private HBaseTestSetup setup; + public HBaseQTestUtil( String outDir, String logDir, MiniClusterType miniMr, HBaseTestSetup setup, String initScript, String cleanupScript) throws Exception { - super(outDir, logDir, miniMr, null, "0.20", initScript, cleanupScript, false, false); + this.setup = setup; setup.preTest(conf); this.conn = setup.getConnection(); super.init(); @@ -69,6 +71,12 @@ public class HBaseQTestUtil extends QTestUtil { } @Override + protected void initConfFromSetup() throws Exception { + super.initConfFromSetup(); + setup.preTest(conf); + } + + @Override public void createSources(String tname) throws Exception { super.createSources(tname); http://git-wip-us.apache.org/repos/asf/hive/blob/37e6e1bf/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java index e6383dc..cee7158 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java @@ -22,9 +22,6 @@ import java.io.IOException; import java.net.ServerSocket; import java.util.Arrays; -import junit.extensions.TestSetup; -import junit.framework.Test; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HColumnDescriptor; http://git-wip-us.apache.org/repos/asf/hive/blob/37e6e1bf/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 8473436..2f109ab 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -510,6 +510,7 @@ public class QTestUtil { dfs.shutdown(); dfs = null; } + Hive.closeCurrent(); } public String readEntireFileIntoString(File queryFile) throws IOException { @@ -731,8 +732,9 @@ public class QTestUtil { return; } - db.getConf().set("hive.metastore.filter.hook", + conf.set("hive.metastore.filter.hook", "org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl"); + db = Hive.get(conf); // Delete any tables other than the source tables // and any databases other than the default database. for (String dbName : db.getAllDatabases()) { @@ -800,16 +802,20 @@ public class QTestUtil { return; } - clearTablesCreatedDuringTests(); - clearKeysCreatedInTests(); - // allocate and initialize a new conf since a test can // modify conf by using 'set' commands conf = new HiveConf(Driver.class); initConf(); + initConfFromSetup(); + // renew the metastore since the cluster type is unencrypted db = Hive.get(conf); // propagate new conf to meta store + clearTablesCreatedDuringTests(); + clearKeysCreatedInTests(); + } + + protected void initConfFromSetup() throws Exception { setup.preTest(conf); } http://git-wip-us.apache.org/repos/asf/hive/blob/37e6e1bf/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index cdd12ab..64a26ac 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -178,7 +178,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient { private boolean isConnected = false; private URI metastoreUris[]; private final HiveMetaHookLoader hookLoader; - protected final HiveConf conf; + protected final HiveConf conf; // Keep a copy of HiveConf so if Session conf changes, we may need to get a new HMS client. protected boolean fastpath = false; private String tokenStrForm; private final boolean localMetaStore; @@ -205,8 +205,10 @@ public class HiveMetaStoreClient implements IMetaStoreClient { this.hookLoader = hookLoader; if (conf == null) { conf = new HiveConf(HiveMetaStoreClient.class); + this.conf = conf; + } else { + this.conf = new HiveConf(conf); } - this.conf = conf; filterHook = loadFilterHooks(); fileMetadataBatchSize = HiveConf.getIntVar( conf, HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_OBJECTS_MAX); @@ -221,10 +223,10 @@ public class HiveMetaStoreClient implements IMetaStoreClient { // instantiate the metastore server handler directly instead of connecting // through the network if (conf.getBoolVar(ConfVars.METASTORE_FASTPATH)) { - client = new HiveMetaStore.HMSHandler("hive client", conf, true); + client = new HiveMetaStore.HMSHandler("hive client", this.conf, true); fastpath = true; } else { - client = HiveMetaStore.newRetryingHMSHandler("hive client", conf, true); + client = HiveMetaStore.newRetryingHMSHandler("hive client", this.conf, true); } isConnected = true; snapshotActiveConf(); http://git-wip-us.apache.org/repos/asf/hive/blob/37e6e1bf/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index 23b8a96..f7a0f31 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -546,10 +546,6 @@ public class SessionState { // Get the following out of the way when you start the session these take a // while and should be done when we start up. try { - // Hive object instance should be created with a copy of the conf object. If the conf is - // shared with SessionState, other parts of the code might update the config, but - // Hive.get(HiveConf) would not recognize the case when it needs refreshing - Hive.get(new HiveConf(startSs.conf)).getMSC(); UserGroupInformation sessionUGI = Utils.getUGI(); FileSystem.get(startSs.conf); @@ -575,10 +571,6 @@ public class SessionState { } } catch (RuntimeException e) { throw e; - } catch (Hive.SchemaException e) { - RuntimeException ex = new RuntimeException(e.getMessage()); - ex.setStackTrace(new StackTraceElement[0]); - throw ex; } catch (Exception e) { // Catch-all due to some exec time dependencies on session state // that would cause ClassNoFoundException otherwise
