Repository: hive Updated Branches: refs/heads/branch-2 059844e48 -> 1f7e3be05
HIVE-18147 : Tests can fail with java.net.BindException: Address already in use (Janaki Lahorani, reviewed by Andrew Sherman and Vihang Karajgaonkar) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1f7e3be0 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1f7e3be0 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1f7e3be0 Branch: refs/heads/branch-2 Commit: 1f7e3be05982a4cb41f9316bf2c012767b5b2a5f Parents: 059844e Author: Vihang Karajgaonkar <vih...@cloudera.com> Authored: Sun Dec 3 15:36:15 2017 +0530 Committer: Vihang Karajgaonkar <vih...@cloudera.com> Committed: Sun Dec 3 15:43:15 2017 +0530 ---------------------------------------------------------------------- .../apache/hive/hcatalog/cli/TestPermsGrp.java | 4 +- .../mapreduce/TestHCatPartitionPublish.java | 4 +- .../hive/hcatalog/templeton/TestWebHCatE2e.java | 41 +++++++++++++------- .../AbstractTestAuthorizationApiAuthorizer.java | 3 +- .../hadoop/hive/metastore/TestFilterHooks.java | 3 +- .../metastore/TestHiveMetaStoreStatsMerge.java | 3 +- ...TestHiveMetaStoreWithEnvironmentContext.java | 3 +- .../TestMetaStoreEndFunctionListener.java | 3 +- .../metastore/TestMetaStoreEventListener.java | 3 +- .../TestMetaStoreEventListenerOnlyOnCommit.java | 3 +- .../metastore/TestMetaStoreInitListener.java | 3 +- .../hive/metastore/TestMetaStoreMetrics.java | 7 ++-- .../hive/metastore/TestRemoteHiveMetaStore.java | 3 +- .../TestRemoteHiveMetaStoreIpAddress.java | 5 +-- .../hive/metastore/TestRetryingHMSHandler.java | 3 +- ...estDDLWithRemoteMetastoreSecondNamenode.java | 3 +- .../hive/ql/security/FolderPermissionBase.java | 3 +- .../security/StorageBasedMetastoreTestBase.java | 5 +-- .../TestAuthorizationPreEventListener.java | 4 +- .../TestClientSideAuthorizationProvider.java | 4 +- .../TestMetastoreAuthorizationProvider.java | 5 +-- .../TestMultiAuthorizationPreEventListener.java | 6 +-- .../thrift/ThriftCliServiceMessageSizeTest.java | 24 +++++++++--- .../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 37 +++++++++++++++--- .../hadoop/hive/metastore/MetaStoreUtils.java | 33 ++++++++++++++++ .../thrift/ThriftCliServiceTestWithCookie.java | 25 +++++++++--- 26 files changed, 155 insertions(+), 85 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java ---------------------------------------------------------------------- diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java index 66a5dd4..2375f5d 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java @@ -79,9 +79,7 @@ public class TestPermsGrp extends TestCase { return; } - - msPort = MetaStoreUtils.findFreePort(); - MetaStoreUtils.startMetaStore(msPort, ShimLoader.getHadoopThriftAuthBridge()); + msPort = MetaStoreUtils.startMetaStoreWithRetry(); isServerRunning = true; http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java ---------------------------------------------------------------------- diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java index 69874bc..777fc9c 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java @@ -104,10 +104,8 @@ public class TestHCatPartitionPublish { return; } - msPort = MetaStoreUtils.findFreePort(); + msPort = MetaStoreUtils.startMetaStoreWithRetry(); - MetaStoreUtils.startMetaStore(msPort, ShimLoader - .getHadoopThriftAuthBridge()); Thread.sleep(10000); isServerRunning = true; securityManager = System.getSecurityManager(); http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java ---------------------------------------------------------------------- diff --git a/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java b/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java index b62716e..841e2ff 100644 --- a/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java +++ b/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java @@ -70,21 +70,36 @@ public class TestWebHCatE2e { private static final String charSet = "UTF-8"; @BeforeClass - public static void startHebHcatInMem() { - int webhcatPort = 50111; - try { - //in case concurrent tests are running on the same machine - webhcatPort = MetaStoreUtils.findFreePort(); + public static void startHebHcatInMem() throws Exception { + Exception webhcatException = null; + int webhcatPort = 0; + boolean webhcatStarted = false; + + for (int tryCount = 0; tryCount < MetaStoreUtils.RETRY_COUNT; tryCount++) { + try { + if (tryCount == MetaStoreUtils.RETRY_COUNT - 1) { + // down to the last try. try default port 50111 + webhcatPort = 50111; + } else { + webhcatPort = MetaStoreUtils.findFreePort(); + } + templetonBaseUrl = templetonBaseUrl.replace("50111", Integer.toString(webhcatPort)); + templetonServer = new Main(new String[] { "-D" + AppConfig.UNIT_TEST_MODE + "=true", + "-D" + AppConfig.PORT + "=" + webhcatPort }); + LOG.info("Starting Main; WebHCat using port: " + webhcatPort); + templetonServer.run(); + LOG.info("Main started"); + webhcatStarted = true; + break; + } catch (Exception ce) { + LOG.info("Attempt to Start WebHCat using port: " + webhcatPort + " failed"); + webhcatException = ce; + } } - catch (IOException ex) { - LOG.warn("Unable to find free port; using default: " + webhcatPort); + + if (!webhcatStarted) { + throw webhcatException; } - templetonBaseUrl = templetonBaseUrl.replace("50111", Integer.toString(webhcatPort)); - templetonServer = new Main(new String[] {"-D" + - AppConfig.UNIT_TEST_MODE + "=true", "-D" + AppConfig.PORT + "=" + webhcatPort}); - LOG.info("Starting Main; WebHCat using port: " + webhcatPort); - templetonServer.run(); - LOG.info("Main started"); } @AfterClass http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java index 341c7ba..e72a585 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java @@ -57,8 +57,7 @@ public abstract class AbstractTestAuthorizationApiAuthorizer { hiveConf = new HiveConf(); if (isRemoteMetastoreMode) { - int port = MetaStoreUtils.findFreePort(); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + int port = MetaStoreUtils.startMetaStoreWithRetry(); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); } hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java index 1073abb..42859d4 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java @@ -173,9 +173,8 @@ public class TestFilterHooks { hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); hiveConf.setVar(ConfVars.METASTORE_FILTER_HOOK, DummyMetaStoreFilterHookImpl.class.getName()); UtilsForTest.setNewDerbyDbLocation(hiveConf, TestFilterHooks.class.getSimpleName()); - int port = MetaStoreUtils.findFreePort(); + int port = MetaStoreUtils.startMetaStoreWithRetry(hiveConf); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge(), hiveConf); SessionState.start(new CliSessionState(hiveConf)); msc = new HiveMetaStoreClient(hiveConf); http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreStatsMerge.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreStatsMerge.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreStatsMerge.java index 600aef4..4aa527c 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreStatsMerge.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreStatsMerge.java @@ -76,8 +76,7 @@ public class TestHiveMetaStoreStatsMerge extends TestCase { System.setProperty("hive.metastore.event.listeners", DummyListener.class.getName()); - int port = MetaStoreUtils.findFreePort(); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + int port = MetaStoreUtils.startMetaStoreWithRetry(); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java index d6e4fb7..e999bfb 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java @@ -73,8 +73,7 @@ public class TestHiveMetaStoreWithEnvironmentContext extends TestCase { System.setProperty("hive.metastore.event.listeners", DummyListener.class.getName()); - int port = MetaStoreUtils.findFreePort(); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + int port = MetaStoreUtils.startMetaStoreWithRetry(); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java index 1e78ff1..1158b76 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java @@ -47,8 +47,7 @@ public class TestMetaStoreEndFunctionListener extends TestCase { DummyPreListener.class.getName()); System.setProperty("hive.metastore.end.function.listeners", DummyEndFunctionListener.class.getName()); - int port = MetaStoreUtils.findFreePort(); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + int port = MetaStoreUtils.startMetaStoreWithRetry(); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java index fd4527e..2ad58cf 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java @@ -92,8 +92,7 @@ public class TestMetaStoreEventListener extends TestCase { System.setProperty("hive.metastore.pre.event.listeners", DummyPreListener.class.getName()); - int port = MetaStoreUtils.findFreePort(); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + int port = MetaStoreUtils.startMetaStoreWithRetry(); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java index 0c3e703..84ebbe1 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java @@ -50,8 +50,7 @@ public class TestMetaStoreEventListenerOnlyOnCommit extends TestCase { System.setProperty(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL.varname, DummyRawStoreControlledCommit.class.getName()); - int port = MetaStoreUtils.findFreePort(); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + int port = MetaStoreUtils.startMetaStoreWithRetry(); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java index e8171e5..b6426ff 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java @@ -42,8 +42,7 @@ public class TestMetaStoreInitListener extends TestCase { super.setUp(); System.setProperty("hive.metastore.init.hooks", DummyMetaStoreInitListener.class.getName()); - int port = MetaStoreUtils.findFreePort(); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + int port = MetaStoreUtils.startMetaStoreWithRetry(); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java index b462b2a..8d6a45b 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java @@ -42,10 +42,7 @@ public class TestMetaStoreMetrics { @BeforeClass public static void before() throws Exception { - int port = MetaStoreUtils.findFreePort(); - hiveConf = new HiveConf(TestMetaStoreMetrics.class); - hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); hiveConf.setBoolVar(HiveConf.ConfVars.METASTORE_METRICS, true); hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); @@ -58,7 +55,9 @@ public class TestMetaStoreMetrics { metrics = (CodahaleMetrics) MetricsFactory.getInstance(); //Increments one HMS connection - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge(), hiveConf); + int port = MetaStoreUtils.startMetaStoreWithRetry(hiveConf); + + hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); //Increments one HMS connection (Hive.get()) SessionState.start(new CliSessionState(hiveConf)); http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java index ef02968..225cf0a 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java @@ -42,9 +42,8 @@ public class TestRemoteHiveMetaStore extends TestHiveMetaStore { return; } - port = MetaStoreUtils.findFreePort(); + port = MetaStoreUtils.startMetaStoreWithRetry(hiveConf); System.out.println("Starting MetaStore Server on port " + port); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge(), hiveConf); isServerStarted = true; // This is default case with setugi off for both client and server http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java index 63eea27..1c1c3e6 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java @@ -48,11 +48,10 @@ public class TestRemoteHiveMetaStoreIpAddress extends TestCase { return; } - int port = MetaStoreUtils.findFreePort(); - System.out.println("Starting MetaStore Server on port " + port); System.setProperty(ConfVars.METASTORE_EVENT_LISTENERS.varname, IpAddressListener.class.getName()); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + int port = MetaStoreUtils.startMetaStoreWithRetry(); + System.out.println("Starting MetaStore Server on port " + port); isServerStarted = true; // This is default case with setugi off for both client and server http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java index 99c09b7..abe2416 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java @@ -53,8 +53,7 @@ public class TestRetryingHMSHandler extends TestCase { super.setUp(); System.setProperty("hive.metastore.pre.event.listeners", AlternateFailurePreListener.class.getName()); - int port = MetaStoreUtils.findFreePort(); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + int port = MetaStoreUtils.startMetaStoreWithRetry(); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java index bfb25aa..a9bb4ad 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java @@ -79,8 +79,7 @@ public class TestDDLWithRemoteMetastoreSecondNamenode extends TestCase { SessionState.start(conf); // Test with remote metastore service - int port = MetaStoreUtils.findFreePort(); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + int port = MetaStoreUtils.startMetaStoreWithRetry(); conf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); conf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); conf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java index bf6b7e1..63cd732 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java @@ -100,8 +100,7 @@ public abstract class FolderPermissionBase { conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); conf.setBoolVar(HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS, true); conf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict"); - int port = MetaStoreUtils.findFreePort(); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + MetaStoreUtils.startMetaStoreWithRetry(); SessionState.start(new CliSessionState(conf)); driver = new Driver(conf); http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java index 36f31f4..f6ac493 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java @@ -63,9 +63,6 @@ public class StorageBasedMetastoreTestBase { @Before public void setUp() throws Exception { - - int port = MetaStoreUtils.findFreePort(); - // Turn on metastore-side authorization System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, AuthorizationPreEventListener.class.getName()); @@ -75,7 +72,7 @@ public class StorageBasedMetastoreTestBase { InjectableDummyAuthenticator.class.getName()); clientHiveConf = createHiveConf(); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge(), clientHiveConf); + int port = MetaStoreUtils.startMetaStoreWithRetry(clientHiveConf); // Turn off client-side authorization clientHiveConf.setBoolVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED,false); http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java index b0da884..b357a55 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java @@ -53,8 +53,6 @@ public class TestAuthorizationPreEventListener extends TestCase { super.setUp(); - int port = MetaStoreUtils.findFreePort(); - System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, AuthorizationPreEventListener.class.getName()); System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHORIZATION_MANAGER.varname, @@ -62,7 +60,7 @@ public class TestAuthorizationPreEventListener extends TestCase { System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHENTICATOR_MANAGER.varname, HadoopDefaultMetastoreAuthenticator.class.getName()); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + int port = MetaStoreUtils.startMetaStoreWithRetry(); clientHiveConf = new HiveConf(this.getClass()); http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java index d895da8..312d4aa 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java @@ -58,13 +58,11 @@ public class TestClientSideAuthorizationProvider extends TestCase { super.setUp(); - int port = MetaStoreUtils.findFreePort(); - // Turn off metastore-side authorization System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, ""); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + int port = MetaStoreUtils.startMetaStoreWithRetry(); clientHiveConf = new HiveConf(this.getClass()); http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java index 19dc9cf..e11d28c 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java @@ -84,8 +84,6 @@ public class TestMetastoreAuthorizationProvider extends TestCase { super.setUp(); - int port = MetaStoreUtils.findFreePort(); - // Turn on metastore-side authorization System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, AuthorizationPreEventListener.class.getName()); @@ -98,8 +96,7 @@ public class TestMetastoreAuthorizationProvider extends TestCase { InjectableDummyAuthenticator.class.getName()); System.setProperty(HiveConf.ConfVars.HIVE_AUTHORIZATION_TABLE_OWNER_GRANTS.varname, ""); - - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + int port = MetaStoreUtils.startMetaStoreWithRetry(); clientHiveConf = createHiveConf(); http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java index 5c9bf05..45b443f 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java @@ -47,10 +47,6 @@ public class TestMultiAuthorizationPreEventListener { @BeforeClass public static void setUp() throws Exception { - - - int port = MetaStoreUtils.findFreePort(); - System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, AuthorizationPreEventListener.class.getName()); @@ -62,7 +58,7 @@ public class TestMultiAuthorizationPreEventListener { System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHENTICATOR_MANAGER.varname, HadoopDefaultMetastoreAuthenticator.class.getName()); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + int port = MetaStoreUtils.startMetaStoreWithRetry(); clientHiveConf = new HiveConf(); http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/ThriftCliServiceMessageSizeTest.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/ThriftCliServiceMessageSizeTest.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/ThriftCliServiceMessageSizeTest.java index 7148307..a73ea9f 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/ThriftCliServiceMessageSizeTest.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/ThriftCliServiceMessageSizeTest.java @@ -68,13 +68,27 @@ public class ThriftCliServiceMessageSizeTest { protected static void startHiveServer2WithConf(HiveServer2 hiveServer2, HiveConf hiveConf) throws Exception { - hiveServer2.init(hiveConf); // Start HiveServer2 with given config // Fail if server doesn't start - try { - hiveServer2.start(); - } catch (Throwable t) { - t.printStackTrace(); + + Throwable hs2Exception = null; + boolean hs2Started = false; + for (int tryCount = 0; tryCount < MetaStoreUtils.RETRY_COUNT; tryCount++) { + try { + hiveServer2.init(hiveConf); + hiveServer2.start(); + hs2Started = true; + break; + } catch (Throwable t) { + hs2Exception = t; + port = MetaStoreUtils.findFreePort(); + hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT, port); + hiveServer2 = new HiveServer2(); + } + } + + if (!hs2Started) { + hs2Exception.printStackTrace(); fail(); } // Wait for startup to complete http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java b/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java index ebc4c10..2bf2461 100644 --- a/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java +++ b/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java @@ -74,6 +74,7 @@ public class MiniHS2 extends AbstractHiveService { private final boolean cleanupLocalDirOnStartup; private final boolean isMetastoreSecure; private MiniClusterType miniClusterType = MiniClusterType.LOCALFS_ONLY; + private boolean usePortsFromConf = false; public enum MiniClusterType { MR, @@ -226,6 +227,7 @@ public class MiniHS2 extends AbstractHiveService { this.isMetastoreRemote = isMetastoreRemote; this.isMetastoreSecure = isMetastoreSecure; this.cleanupLocalDirOnStartup = cleanupLocalDirOnStartup; + this.usePortsFromConf = usePortsFromConf; baseDir = getBaseDir(); localFS = FileSystem.getLocal(hiveConf); FileSystem fs; @@ -333,19 +335,42 @@ public class MiniHS2 extends AbstractHiveService { public void start(Map<String, String> confOverlay) throws Exception { if (isMetastoreRemote) { - int metaStorePort = MetaStoreUtils.findFreePort(); + int metaStorePort = MetaStoreUtils.startMetaStoreWithRetry(getHiveConf()); getHiveConf().setVar(ConfVars.METASTOREURIS, "thrift://localhost:" + metaStorePort); - MetaStoreUtils.startMetaStore(metaStorePort, - ShimLoader.getHadoopThriftAuthBridge(), getHiveConf()); } - hiveServer2 = new HiveServer2(); // Set confOverlay parameters for (Map.Entry<String, String> entry : confOverlay.entrySet()) { setConfProperty(entry.getKey(), entry.getValue()); } - hiveServer2.init(getHiveConf()); - hiveServer2.start(); + + Exception hs2Exception = null; + boolean hs2Started = false; + for (int tryCount = 0; tryCount < MetaStoreUtils.RETRY_COUNT; tryCount++) { + try { + hiveServer2 = new HiveServer2(); + hiveServer2.init(getHiveConf()); + hiveServer2.start(); + hs2Started = true; + break; + } catch (Exception t) { + hs2Exception = t; + if (usePortsFromConf) { + hs2Started = false; + break; + } else { + HiveConf.setIntVar(getHiveConf(), HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT, + MetaStoreUtils.findFreePort()); + HiveConf.setIntVar(getHiveConf(), HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT, + MetaStoreUtils.findFreePort()); + } + } + } + + if (!hs2Started) { + throw(hs2Exception); + } + waitForStartup(); setStarted(true); } http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index 3eb8c36..13bdf8e 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.IOException; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; +import java.net.ConnectException; import java.net.InetSocketAddress; import java.net.ServerSocket; import java.net.Socket; @@ -101,6 +102,8 @@ public class MetaStoreUtils { public static final String DATABASE_WAREHOUSE_SUFFIX = ".db"; + public static final int RETRY_COUNT = 10; + // Right now we only support one special character '/'. // More special characters can be added accordingly in the future. // NOTE: @@ -1250,6 +1253,36 @@ public class MetaStoreUtils { return port; } + public static int startMetaStoreWithRetry(final HadoopThriftAuthBridge bridge) throws Exception { + return startMetaStoreWithRetry(bridge, null); + } + + public static int startMetaStoreWithRetry(HiveConf conf) throws Exception { + return startMetaStoreWithRetry(ShimLoader.getHadoopThriftAuthBridge(), conf); + } + + public static int startMetaStoreWithRetry() throws Exception { + return startMetaStoreWithRetry(ShimLoader.getHadoopThriftAuthBridge(), null); + } + + public static int startMetaStoreWithRetry(final HadoopThriftAuthBridge bridge, HiveConf conf) + throws Exception { + Exception metaStoreException = null; + int metaStorePort = 0; + + for (int tryCount = 0; tryCount < MetaStoreUtils.RETRY_COUNT; tryCount++) { + try { + metaStorePort = findFreePort(); + startMetaStore(metaStorePort, bridge, conf); + return metaStorePort; + } catch (ConnectException ce) { + metaStoreException = ce; + } + } + + throw metaStoreException; + } + public static int startMetaStore(HiveConf conf) throws Exception { return startMetaStore(ShimLoader.getHadoopThriftAuthBridge(), conf); } http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java ---------------------------------------------------------------------- diff --git a/service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java b/service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java index 6fec947..8977a96 100644 --- a/service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java +++ b/service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java @@ -95,15 +95,30 @@ public class ThriftCliServiceTestWithCookie { } protected static void startHiveServer2WithConf(HiveConf hiveConf) throws Exception { - hiveServer2.init(hiveConf); // Start HiveServer2 with given config // Fail if server doesn't start - try { - hiveServer2.start(); - } catch (Throwable t) { - t.printStackTrace(); + + Throwable hs2Exception = null; + boolean hs2Started = false; + for (int tryCount = 0; tryCount < MetaStoreUtils.RETRY_COUNT; tryCount++) { + try { + hiveServer2.init(hiveConf); + hiveServer2.start(); + hs2Started = true; + break; + } catch (Throwable t) { + hs2Exception = t; + port = MetaStoreUtils.findFreePort(); + hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT, port); + hiveServer2 = new HiveServer2(); + } + } + + if (!hs2Started) { + hs2Exception.printStackTrace(); fail(); } + // Wait for startup to complete Thread.sleep(2000); System.out.println("HiveServer2 started on port " + port);