Repository: hive Updated Branches: refs/heads/master fdd8fabdc -> 466f51034
HIVE-18449 : Add configurable policy for choosing the HMS URI from hive.metastore.uris (Janaki Lahorani, reviewed by Vihang Karajgaonkar) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/466f5103 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/466f5103 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/466f5103 Branch: refs/heads/master Commit: 466f51034e18d0fdf8527f1eec5d421cebb41db4 Parents: fdd8fab Author: Vihang Karajgaonkar <[email protected]> Authored: Fri Feb 2 10:50:47 2018 -0800 Committer: Vihang Karajgaonkar <[email protected]> Committed: Fri Feb 2 10:50:47 2018 -0800 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/conf/HiveConf.java | 8 +++++++- .../hive/metastore/HiveMetaStoreClient.java | 18 +++++++++++------- .../hadoop/hive/metastore/conf/MetastoreConf.java | 6 ++++++ 3 files changed, 24 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/466f5103/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index b7d3e99..4f2e6d3 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -216,6 +216,7 @@ public class HiveConf extends Configuration { HiveConf.ConfVars.METASTOREWAREHOUSE, HiveConf.ConfVars.REPLDIR, HiveConf.ConfVars.METASTOREURIS, + HiveConf.ConfVars.METASTORESELECTION, HiveConf.ConfVars.METASTORE_SERVER_PORT, HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, HiveConf.ConfVars.METASTORETHRIFTFAILURERETRIES, @@ -632,7 +633,12 @@ public class HiveConf extends Configuration { "location of default database for the warehouse"), METASTOREURIS("hive.metastore.uris", "", "Thrift URI for the remote metastore. Used by metastore client to connect to remote metastore."), - + METASTORESELECTION("hive.metastore.uri.selection", "RANDOM", + new StringSet("SEQUENTIAL", "RANDOM"), + "Determines the selection mechanism used by metastore client to connect to remote " + + "metastore. SEQUENTIAL implies that the first valid metastore from the URIs specified " + + "as part of hive.metastore.uris will be picked. RANDOM implies that the metastore " + + "will be picked randomly"), METASTORE_CAPABILITY_CHECK("hive.metastore.client.capability.check", true, "Whether to check client capabilities for potentially breaking API usage."), METASTORE_FASTPATH("hive.metastore.fastpath", false, http://git-wip-us.apache.org/repos/asf/hive/blob/466f5103/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 3a468b1..a3cb17b 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -195,9 +195,11 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { } // make metastore URIS random - List<?> uriList = Arrays.asList(metastoreUris); - Collections.shuffle(uriList); - metastoreUris = (URI[]) uriList.toArray(); + if (MetastoreConf.getVar(conf, ConfVars.THRIFT_URI_SELECTION).equalsIgnoreCase("RANDOM")) { + List uriList = Arrays.asList(metastoreUris); + Collections.shuffle(uriList); + metastoreUris = (URI[]) uriList.toArray(); + } } catch (IllegalArgumentException e) { throw (e); } catch (Exception e) { @@ -322,10 +324,12 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { " at the client level."); } else { close(); - // Swap the first element of the metastoreUris[] with a random element from the rest - // of the array. Rationale being that this method will generally be called when the default - // connection has died and the default connection is likely to be the first array element. - promoteRandomMetaStoreURI(); + if (MetastoreConf.getVar(conf, ConfVars.THRIFT_URI_SELECTION).equalsIgnoreCase("RANDOM")) { + // Swap the first element of the metastoreUris[] with a random element from the rest + // of the array. Rationale being that this method will generally be called when the default + // connection has died and the default connection is likely to be the first array element. + promoteRandomMetaStoreURI(); + } open(); } } http://git-wip-us.apache.org/repos/asf/hive/blob/466f5103/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 3c8d005..dba68ac 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -718,6 +718,12 @@ public class MetastoreConf { "Number of retries upon failure of Thrift metastore calls"), THRIFT_URIS("metastore.thrift.uris", "hive.metastore.uris", "", "Thrift URI for the remote metastore. Used by metastore client to connect to remote metastore."), + THRIFT_URI_SELECTION("metastore.thrift.uri.selection", "hive.metastore.uri.selection", "RANDOM", + new Validator.StringSet("RANDOM", "SEQUENTIAL"), + "Determines the selection mechanism used by metastore client to connect to remote " + + "metastore. SEQUENTIAL implies that the first valid metastore from the URIs specified " + + "as part of hive.metastore.uris will be picked. RANDOM implies that the metastore " + + "will be picked randomly"), TIMEDOUT_TXN_REAPER_START("metastore.timedout.txn.reaper.start", "hive.timedout.txn.reaper.start", 100, TimeUnit.SECONDS, "Time delay of 1st reaper run after metastore start"),
