Repository: hive Updated Branches: refs/heads/master 525cfa7be -> 542b165c3
HIVE-18147 : Tests can fail with java.net.BindException: Address already in use (Janaki Lahorani, reviewed by Andrew Sherman and Vihang Karajgaonkar) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/542b165c Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/542b165c Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/542b165c Branch: refs/heads/master Commit: 542b165c3f042f9d726d2ed831adfc5a89ce369f Parents: 525cfa7 Author: Vihang Karajgaonkar <vih...@cloudera.com> Authored: Tue Nov 28 11:11:39 2017 -0800 Committer: Vihang Karajgaonkar <vih...@cloudera.com> Committed: Tue Nov 28 11:30:57 2017 -0800 ---------------------------------------------------------------------- .../apache/hive/hcatalog/cli/TestPermsGrp.java | 4 +- .../mapreduce/TestHCatPartitionPublish.java | 3 +- .../hive/hcatalog/templeton/TestWebHCatE2e.java | 43 ++++++++++++++------ .../AbstractTestAuthorizationApiAuthorizer.java | 3 +- .../hadoop/hive/metastore/TestFilterHooks.java | 3 +- ...TestHiveMetaStoreWithEnvironmentContext.java | 3 +- .../TestMetaStoreEndFunctionListener.java | 3 +- .../metastore/TestMetaStoreEventListener.java | 3 +- .../TestMetaStoreEventListenerOnlyOnCommit.java | 3 +- .../metastore/TestMetaStoreInitListener.java | 3 +- .../hive/metastore/TestMetaStoreMetrics.java | 7 ++-- .../hive/metastore/TestRemoteHiveMetaStore.java | 3 +- .../TestRemoteHiveMetaStoreIpAddress.java | 5 +-- .../hive/metastore/TestRetryingHMSHandler.java | 3 +- ...estDDLWithRemoteMetastoreSecondNamenode.java | 3 +- .../security/StorageBasedMetastoreTestBase.java | 5 +-- .../TestAuthorizationPreEventListener.java | 4 +- .../TestClientSideAuthorizationProvider.java | 4 +- .../TestMetastoreAuthorizationProvider.java | 5 +-- .../TestMultiAuthorizationPreEventListener.java | 6 +-- .../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 37 ++++++++++++++--- .../hive/metastore/MetaStoreTestUtils.java | 32 +++++++++++++++ .../thrift/ThriftCliServiceTestWithCookie.java | 26 ++++++++---- 23 files changed, 134 insertions(+), 77 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java ---------------------------------------------------------------------- diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java index 80c5d63..e5aa2b8 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java @@ -80,9 +80,7 @@ public class TestPermsGrp extends TestCase { return; } - - msPort = MetaStoreTestUtils.findFreePort(); - MetaStoreTestUtils.startMetaStore(msPort, HadoopThriftAuthBridge.getBridge()); + msPort = MetaStoreTestUtils.startMetaStoreWithRetry(); isServerRunning = true; http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java ---------------------------------------------------------------------- diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java index a000642..6711a3e 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java @@ -105,9 +105,8 @@ public class TestHCatPartitionPublish { return; } - msPort = MetaStoreTestUtils.findFreePort(); + msPort = MetaStoreTestUtils.startMetaStoreWithRetry(); - MetaStoreTestUtils.startMetaStore(msPort, HadoopThriftAuthBridge.getBridge()); Thread.sleep(10000); isServerRunning = true; securityManager = System.getSecurityManager(); http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java ---------------------------------------------------------------------- diff --git a/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java b/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java index 341314b..55cd5b1 100644 --- a/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java +++ b/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java @@ -70,21 +70,38 @@ public class TestWebHCatE2e { private static final String charSet = "UTF-8"; @BeforeClass - public static void startHebHcatInMem() { - int webhcatPort = 50111; - try { - //in case concurrent tests are running on the same machine - webhcatPort = MetaStoreTestUtils.findFreePort(); + public static void startHebHcatInMem() throws Exception { + Exception webhcatException = null; + int webhcatPort = 0; + boolean webhcatStarted = false; + + for (int tryCount = 0; tryCount < MetaStoreTestUtils.RETRY_COUNT; tryCount++) { + try { + if (tryCount == MetaStoreTestUtils.RETRY_COUNT - 1) { + /* Last try to get a port. Just use default 50111. */ + webhcatPort = 50111; + LOG.warn("Unable to find free port; using default: " + webhcatPort); + } + else { + webhcatPort = MetaStoreTestUtils.findFreePort(); + } + templetonBaseUrl = templetonBaseUrl.replace("50111", Integer.toString(webhcatPort)); + templetonServer = new Main(new String[] { "-D" + AppConfig.UNIT_TEST_MODE + "=true", + "-D" + AppConfig.PORT + "=" + webhcatPort }); + LOG.info("Starting Main; WebHCat using port: " + webhcatPort); + templetonServer.run(); + LOG.info("Main started"); + webhcatStarted = true; + break; + } catch (Exception ce) { + LOG.info("Attempt to Start WebHCat using port: " + webhcatPort + " failed"); + webhcatException = ce; + } } - catch (IOException ex) { - LOG.warn("Unable to find free port; using default: " + webhcatPort); + + if (!webhcatStarted) { + throw webhcatException; } - templetonBaseUrl = templetonBaseUrl.replace("50111", Integer.toString(webhcatPort)); - templetonServer = new Main(new String[] {"-D" + - AppConfig.UNIT_TEST_MODE + "=true", "-D" + AppConfig.PORT + "=" + webhcatPort}); - LOG.info("Starting Main; WebHCat using port: " + webhcatPort); - templetonServer.run(); - LOG.info("Main started"); } @AfterClass http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java index ebdfab2..bd8332c 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java @@ -57,8 +57,7 @@ public abstract class AbstractTestAuthorizationApiAuthorizer { hiveConf = new HiveConf(); if (isRemoteMetastoreMode) { - int port = MetaStoreTestUtils.findFreePort(); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); } hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java index 6a2bbd2..8366fab 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java @@ -174,9 +174,8 @@ public class TestFilterHooks { hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); hiveConf.setVar(ConfVars.METASTORE_FILTER_HOOK, DummyMetaStoreFilterHookImpl.class.getName()); UtilsForTest.setNewDerbyDbLocation(hiveConf, TestFilterHooks.class.getSimpleName()); - int port = MetaStoreTestUtils.findFreePort(); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(hiveConf); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), hiveConf); SessionState.start(new CliSessionState(hiveConf)); msc = new HiveMetaStoreClient(hiveConf); http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java index 3fa4b1e..5267dca 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java @@ -73,8 +73,7 @@ public class TestHiveMetaStoreWithEnvironmentContext extends TestCase { System.setProperty("hive.metastore.event.listeners", DummyListener.class.getName()); - int port = MetaStoreTestUtils.findFreePort(); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java index 27bc791..3bb340e 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java @@ -48,8 +48,7 @@ public class TestMetaStoreEndFunctionListener extends TestCase { DummyPreListener.class.getName()); System.setProperty("hive.metastore.end.function.listeners", DummyEndFunctionListener.class.getName()); - int port = MetaStoreTestUtils.findFreePort(); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java index 4df2f86..7219774 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java @@ -94,11 +94,10 @@ public class TestMetaStoreEventListener extends TestCase { System.setProperty("hive.metastore.pre.event.listeners", DummyPreListener.class.getName()); - int port = MetaStoreTestUtils.findFreePort(); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTORE_PARTITION_NAME_WHITELIST_PATTERN, metaConfVal); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), hiveConf); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(hiveConf); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java index d9da422..a905324 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java @@ -50,8 +50,7 @@ public class TestMetaStoreEventListenerOnlyOnCommit extends TestCase { System.setProperty(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL.varname, DummyRawStoreControlledCommit.class.getName()); - int port = MetaStoreTestUtils.findFreePort(); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java index 9578ef9..3d58819 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java @@ -41,8 +41,7 @@ public class TestMetaStoreInitListener extends TestCase { super.setUp(); System.setProperty("hive.metastore.init.hooks", DummyMetaStoreInitListener.class.getName()); - int port = MetaStoreTestUtils.findFreePort(); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java index a899739..9d241a8 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java @@ -40,10 +40,7 @@ public class TestMetaStoreMetrics { @BeforeClass public static void before() throws Exception { - int port = MetaStoreTestUtils.findFreePort(); - hiveConf = new HiveConf(TestMetaStoreMetrics.class); - hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); hiveConf.setBoolVar(HiveConf.ConfVars.METASTORE_METRICS, true); hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); @@ -52,7 +49,9 @@ public class TestMetaStoreMetrics { "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); //Increments one HMS connection - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), hiveConf); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(hiveConf); + + hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); //Increments one HMS connection (Hive.get()) SessionState.start(new CliSessionState(hiveConf)); http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java index 19bed9d..d4a809f 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java @@ -42,9 +42,8 @@ public class TestRemoteHiveMetaStore extends TestHiveMetaStore { return; } - port = MetaStoreTestUtils.findFreePort(); + port = MetaStoreTestUtils.startMetaStoreWithRetry(hiveConf); System.out.println("Starting MetaStore Server on port " + port); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), hiveConf); isServerStarted = true; // This is default case with setugi off for both client and server http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java index 449fe71..9ba1d75 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java @@ -48,11 +48,10 @@ public class TestRemoteHiveMetaStoreIpAddress extends TestCase { return; } - int port = MetaStoreTestUtils.findFreePort(); - System.out.println("Starting MetaStore Server on port " + port); System.setProperty(ConfVars.METASTORE_EVENT_LISTENERS.varname, IpAddressListener.class.getName()); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(); + System.out.println("Started MetaStore Server on port " + port); isServerStarted = true; // This is default case with setugi off for both client and server http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java index 41494d1..2b7c5a1 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java @@ -53,8 +53,7 @@ public class TestRetryingHMSHandler extends TestCase { super.setUp(); System.setProperty("hive.metastore.pre.event.listeners", AlternateFailurePreListener.class.getName()); - int port = MetaStoreTestUtils.findFreePort(); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java index 4ce2731..179eed9 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java @@ -80,8 +80,7 @@ public class TestDDLWithRemoteMetastoreSecondNamenode extends TestCase { SessionState.start(conf); // Test with remote metastore service - int port = MetaStoreTestUtils.findFreePort(); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(); conf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); conf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); conf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java index fa5ae80..dc3af3c 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java @@ -62,9 +62,6 @@ public class StorageBasedMetastoreTestBase { @Before public void setUp() throws Exception { - - int port = MetaStoreTestUtils.findFreePort(); - // Turn on metastore-side authorization System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, AuthorizationPreEventListener.class.getName()); @@ -74,7 +71,7 @@ public class StorageBasedMetastoreTestBase { InjectableDummyAuthenticator.class.getName()); clientHiveConf = createHiveConf(); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), clientHiveConf); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(clientHiveConf); // Turn off client-side authorization clientHiveConf.setBoolVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED,false); http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java index 102af6a..6a668aa 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java @@ -53,8 +53,6 @@ public class TestAuthorizationPreEventListener extends TestCase { super.setUp(); - int port = MetaStoreTestUtils.findFreePort(); - System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, AuthorizationPreEventListener.class.getName()); System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHORIZATION_MANAGER.varname, @@ -62,7 +60,7 @@ public class TestAuthorizationPreEventListener extends TestCase { System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHENTICATOR_MANAGER.varname, HadoopDefaultMetastoreAuthenticator.class.getName()); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(); clientHiveConf = new HiveConf(this.getClass()); http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java index 60273e7..57ff8c9 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java @@ -58,13 +58,11 @@ public class TestClientSideAuthorizationProvider extends TestCase { super.setUp(); - int port = MetaStoreTestUtils.findFreePort(); - // Turn off metastore-side authorization System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, ""); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(); clientHiveConf = new HiveConf(this.getClass()); http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java index f0196c6..871cf3d 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java @@ -84,8 +84,6 @@ public class TestMetastoreAuthorizationProvider extends TestCase { super.setUp(); - int port = MetaStoreTestUtils.findFreePort(); - // Turn on metastore-side authorization System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, AuthorizationPreEventListener.class.getName()); @@ -98,8 +96,7 @@ public class TestMetastoreAuthorizationProvider extends TestCase { InjectableDummyAuthenticator.class.getName()); System.setProperty(HiveConf.ConfVars.HIVE_AUTHORIZATION_TABLE_OWNER_GRANTS.varname, ""); - - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(); clientHiveConf = createHiveConf(); http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java index 62835b0..2059370 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java @@ -47,10 +47,6 @@ public class TestMultiAuthorizationPreEventListener { @BeforeClass public static void setUp() throws Exception { - - - int port = MetaStoreTestUtils.findFreePort(); - System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, AuthorizationPreEventListener.class.getName()); @@ -62,7 +58,7 @@ public class TestMultiAuthorizationPreEventListener { System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHENTICATOR_MANAGER.varname, HadoopDefaultMetastoreAuthenticator.class.getName()); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(); clientHiveConf = new HiveConf(); http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java b/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java index e554cc8..ae78c19 100644 --- a/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java +++ b/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java @@ -36,7 +36,6 @@ import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.llap.LlapItUtils; import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster; import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; -import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.util.ZooKeeperHiveHelper; import org.apache.hadoop.hive.shims.HadoopShims.MiniDFSShim; @@ -75,6 +74,7 @@ public class MiniHS2 extends AbstractHiveService { private final boolean cleanupLocalDirOnStartup; private final boolean isMetastoreSecure; private MiniClusterType miniClusterType = MiniClusterType.LOCALFS_ONLY; + private boolean usePortsFromConf = false; public enum MiniClusterType { MR, @@ -227,6 +227,7 @@ public class MiniHS2 extends AbstractHiveService { this.isMetastoreRemote = isMetastoreRemote; this.isMetastoreSecure = isMetastoreSecure; this.cleanupLocalDirOnStartup = cleanupLocalDirOnStartup; + this.usePortsFromConf = usePortsFromConf; baseDir = getBaseDir(); localFS = FileSystem.getLocal(hiveConf); FileSystem fs; @@ -334,18 +335,42 @@ public class MiniHS2 extends AbstractHiveService { public void start(Map<String, String> confOverlay) throws Exception { if (isMetastoreRemote) { - int metaStorePort = MetaStoreTestUtils.findFreePort(); + int metaStorePort = MetaStoreTestUtils.startMetaStoreWithRetry(getHiveConf()); getHiveConf().setVar(ConfVars.METASTOREURIS, "thrift://localhost:" + metaStorePort); - MetaStoreTestUtils.startMetaStore(metaStorePort, HadoopThriftAuthBridge.getBridge(), getHiveConf()); } - hiveServer2 = new HiveServer2(); // Set confOverlay parameters for (Map.Entry<String, String> entry : confOverlay.entrySet()) { setConfProperty(entry.getKey(), entry.getValue()); } - hiveServer2.init(getHiveConf()); - hiveServer2.start(); + + Exception hs2Exception = null; + boolean hs2Started = false; + for (int tryCount = 0; (tryCount < MetaStoreTestUtils.RETRY_COUNT); tryCount++) { + try { + hiveServer2 = new HiveServer2(); + hiveServer2.init(getHiveConf()); + hiveServer2.start(); + hs2Started = true; + break; + } catch (Exception t) { + hs2Exception = t; + if (usePortsFromConf) { + hs2Started = false; + break; + } else { + HiveConf.setIntVar(getHiveConf(), HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT, + MetaStoreTestUtils.findFreePort()); + HiveConf.setIntVar(getHiveConf(), HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT, + MetaStoreTestUtils.findFreePort()); + } + } + } + + if (!hs2Started) { + throw(hs2Exception); + } + waitForStartup(); setStarted(true); } http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/metastore/src/test/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java ---------------------------------------------------------------------- diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java b/metastore/src/test/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java index 28353c3..7df7ac5 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.metastore; import java.io.IOException; +import java.net.ConnectException; import java.net.InetSocketAddress; import java.net.ServerSocket; import java.net.Socket; @@ -35,6 +36,7 @@ import org.slf4j.LoggerFactory; public class MetaStoreTestUtils { private static final Logger LOG = LoggerFactory.getLogger("hive.log"); + public static final int RETRY_COUNT = 10; public static int startMetaStore() throws Exception { return MetaStoreTestUtils.startMetaStore(HadoopThriftAuthBridge.getBridge(), null); @@ -76,6 +78,36 @@ public class MetaStoreTestUtils { MetaStoreTestUtils.loopUntilHMSReady(port); } + public static int startMetaStoreWithRetry(final HadoopThriftAuthBridge bridge) throws Exception { + return MetaStoreTestUtils.startMetaStoreWithRetry(bridge, null); + } + + public static int startMetaStoreWithRetry(HiveConf conf) throws Exception { + return MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), conf); + } + + public static int startMetaStoreWithRetry() throws Exception { + return MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), null); + } + + public static int startMetaStoreWithRetry(final HadoopThriftAuthBridge bridge, HiveConf conf) + throws Exception { + Exception metaStoreException = null; + int metaStorePort = 0; + + for (int tryCount = 0; tryCount < MetaStoreTestUtils.RETRY_COUNT; tryCount++) { + try { + metaStorePort = MetaStoreTestUtils.findFreePort(); + MetaStoreTestUtils.startMetaStore(metaStorePort, bridge, conf); + return metaStorePort; + } catch (ConnectException ce) { + metaStoreException = ce; + } + } + + throw metaStoreException; + } + /** * A simple connect test to make sure that the metastore is up * @throws Exception http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java ---------------------------------------------------------------------- diff --git a/service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java b/service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java index 6660097..1dac783 100644 --- a/service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java +++ b/service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java @@ -96,15 +96,27 @@ public class ThriftCliServiceTestWithCookie { } protected static void startHiveServer2WithConf(HiveConf hiveConf) throws Exception { - hiveServer2.init(hiveConf); - // Start HiveServer2 with given config - // Fail if server doesn't start - try { - hiveServer2.start(); - } catch (Throwable t) { - t.printStackTrace(); + Exception HS2Exception = null; + boolean HS2Started = false; + for (int tryCount = 0; tryCount < MetaStoreTestUtils.RETRY_COUNT; tryCount++) { + try { + hiveServer2.init(hiveConf); + hiveServer2.start(); + HS2Started = true; + break; + } catch (Exception t) { + HS2Exception = t; + port = MetaStoreTestUtils.findFreePort(); + hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT, port); + hiveServer2 = new HiveServer2(); + } + } + + if (!HS2Started) { + HS2Exception.printStackTrace(); fail(); } + // Wait for startup to complete Thread.sleep(2000); System.out.println("HiveServer2 started on port " + port);
