Repository: hive Updated Branches: refs/heads/master 208e202f5 -> 3635c4dba
HIVE-20507: Beeline: Add a utility command to retrieve all uris from beeline-site.xml (Vaibhav Gumashta, reviewed by Daniel Dai) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3635c4db Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3635c4db Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3635c4db Branch: refs/heads/master Commit: 3635c4dba1613f0aef697b652dbe9e45bb64cb33 Parents: 208e202f Author: Vaibhav Gumashta <[email protected]> Authored: Thu Sep 20 15:23:12 2018 -0700 Committer: Vaibhav Gumashta <[email protected]> Committed: Thu Sep 20 15:23:12 2018 -0700 ---------------------------------------------------------------------- .../java/org/apache/hive/beeline/BeeLine.java | 57 ++++++++++++++- .../org/apache/hive/beeline/BeeLineOpts.java | 10 +++ beeline/src/main/resources/BeeLine.properties | 1 + .../org/apache/hive/jdbc/HiveConnection.java | 73 ++++++++++++++++++++ jdbc/src/java/org/apache/hive/jdbc/Utils.java | 2 +- 5 files changed, 141 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/3635c4db/beeline/src/java/org/apache/hive/beeline/BeeLine.java ---------------------------------------------------------------------- diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLine.java b/beeline/src/java/org/apache/hive/beeline/BeeLine.java index 29ec2de..e54e818 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLine.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLine.java @@ -65,6 +65,7 @@ import java.util.LinkedList; import java.util.List; import java.util.ListIterator; import java.util.Map; +import java.util.Map.Entry; import java.util.Properties; import java.util.ResourceBundle; import java.util.ServiceLoader; @@ -94,6 +95,7 @@ import org.apache.hive.beeline.hs2connection.HS2ConnectionFileUtils; import org.apache.hive.beeline.hs2connection.HiveSiteHS2ConnectionFileParser; import org.apache.hive.beeline.hs2connection.UserHS2ConnectionFileParser; import org.apache.hive.common.util.ShutdownHookManager; +import org.apache.hive.jdbc.HiveConnection; import org.apache.hive.jdbc.JdbcUriParseException; import org.apache.hive.jdbc.Utils; import org.apache.hive.jdbc.Utils.JdbcConnectionParams; @@ -389,6 +391,12 @@ public class BeeLine implements Closeable { .withLongOpt("help") .withDescription("Display this message") .create('h')); + + // -getUrlsFromBeelineSite + options.addOption(OptionBuilder + .withLongOpt("getUrlsFromBeelineSite") + .withDescription("Print all urls from beeline-site.xml, if it is present in the classpath") + .create()); // Substitution option --hivevar options.addOption(OptionBuilder @@ -712,7 +720,7 @@ public class BeeLine implements Closeable { private boolean isBeeLineOpt(String arg) { return arg.startsWith("--") && !(HIVE_VAR_PREFIX.equals(arg) || (HIVE_CONF_PREFIX.equals(arg)) - || "--help".equals(arg) || PROP_FILE_PREFIX.equals(arg)); + || "--help".equals(arg) || PROP_FILE_PREFIX.equals(arg) || "--getUrlsFromBeelineSite".equals(arg)); } } @@ -846,6 +854,12 @@ public class BeeLine implements Closeable { getOpts().setHelpAsked(true); return true; } + + if (cl.hasOption("getUrlsFromBeelineSite")) { + printBeelineSiteUrls(); + getOpts().setBeelineSiteUrlsAsked(true); + return true; + } Properties hiveVars = cl.getOptionProperties("hivevar"); for (String key : hiveVars.stringPropertyNames()) { @@ -922,6 +936,44 @@ public class BeeLine implements Closeable { return false; } + private void printBeelineSiteUrls() { + BeelineSiteParser beelineSiteParser = getUserBeelineSiteParser(); + if (!beelineSiteParser.configExists()) { + output("No beeline-site.xml in the path", true); + } + if (beelineSiteParser.configExists()) { + // Get the named url from user specific config file if present + try { + Properties userNamedConnectionURLs = beelineSiteParser.getConnectionProperties(); + userNamedConnectionURLs.remove(BeelineSiteParser.DEFAULT_NAMED_JDBC_URL_PROPERTY_KEY); + StringBuilder sb = new StringBuilder("urls: "); + for (Entry<Object, Object> entry : userNamedConnectionURLs.entrySet()) { + String urlFromBeelineSite = (String) entry.getValue(); + if (isZkBasedUrl(urlFromBeelineSite)) { + List<String> jdbcUrls = HiveConnection.getAllUrlStrings(urlFromBeelineSite); + for (String jdbcUrl : jdbcUrls) { + sb.append(jdbcUrl + ", "); + } + } else { + sb.append(urlFromBeelineSite + ", "); + } + } + output(sb.toString(), true); + } catch (Exception e) { + output(e.getMessage(), true); + return; + } + } + } + + private boolean isZkBasedUrl(String urlFromBeelineSite) { + String zkJdbcUriParam = ("serviceDiscoveryMode=zooKeeper").toLowerCase(); + if (urlFromBeelineSite.toLowerCase().contains(zkJdbcUriParam)) { + return true; + } + return false; + } + private void setHiveConfVar(String key, String val) { getOpts().getHiveConfVariables().put(key, val); if (HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname.equals(key) && "mr".equals(val)) { @@ -1068,6 +1120,9 @@ public class BeeLine implements Closeable { if (getOpts().isHelpAsked()) { return 0; } + if (getOpts().isBeelineSiteUrlsAsked()) { + return 0; + } if (getOpts().getScriptFile() != null) { return executeFile(getOpts().getScriptFile()); } http://git-wip-us.apache.org/repos/asf/hive/blob/3635c4db/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java ---------------------------------------------------------------------- diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java index 062436a..21675dc 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java @@ -114,6 +114,7 @@ public class BeeLineOpts implements Completer { private Map<String, String> hiveVariables = new HashMap<String, String>(); private Map<String, String> hiveConfVariables = new HashMap<String, String>(); private boolean helpAsked; + private boolean beelineSiteUrlsAsked; private String lastConnectedUrl = null; @@ -679,7 +680,16 @@ public class BeeLineOpts implements Completer { public boolean isHelpAsked() { return helpAsked; } + + public void setBeelineSiteUrlsAsked(boolean beelineSiteUrlsAsked) { + this.beelineSiteUrlsAsked = beelineSiteUrlsAsked; + } + + public boolean isBeelineSiteUrlsAsked() { + return beelineSiteUrlsAsked; + } + public String getLastConnectedUrl(){ return lastConnectedUrl; } http://git-wip-us.apache.org/repos/asf/hive/blob/3635c4db/beeline/src/main/resources/BeeLine.properties ---------------------------------------------------------------------- diff --git a/beeline/src/main/resources/BeeLine.properties b/beeline/src/main/resources/BeeLine.properties index 30b4ef9..46a5173 100644 --- a/beeline/src/main/resources/BeeLine.properties +++ b/beeline/src/main/resources/BeeLine.properties @@ -211,6 +211,7 @@ cmd-usage: Usage: java org.apache.hive.cli.beeline.BeeLine \n \ \ --delimiter=DELIMITER set the query delimiter; multi-char delimiters are allowed, but quotation\n \ \ marks, slashes, and -- are not allowed; defaults to ;\n \ \ --convertBinaryArrayToString=[true/false] display binary column data as string or as byte array \n \ +\ --getUrlsFromBeelineSite Print all urls from beeline-site.xml, if it is present in the classpath\n \ \ --help display this message\n \ \n \ \ Example:\n \ http://git-wip-us.apache.org/repos/asf/hive/blob/3635c4db/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java ---------------------------------------------------------------------- diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java index 9f457a2..335995c 100644 --- a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java +++ b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java @@ -157,6 +157,79 @@ public class HiveConnection implements java.sql.Connection { } return ZooKeeperHiveClientHelper.getDirectParamsList(params); } + + public static List<String> getAllUrlStrings(String zookeeperBasedHS2Url) throws Exception { + List<String> jdbcUrls = new ArrayList<>(); + List<JdbcConnectionParams> allConnectionParams = getAllUrls(zookeeperBasedHS2Url); + for (JdbcConnectionParams cp : allConnectionParams) { + String jdbcUrl = makeDirectJDBCUrlFromConnectionParams(cp); + if ((jdbcUrl != null) && (!jdbcUrl.isEmpty())) { + jdbcUrls.add(jdbcUrl); + } + } + return jdbcUrls; + } + + private static String makeDirectJDBCUrlFromConnectionParams(JdbcConnectionParams cp) { + // Direct JDBC Url format: + // jdbc:hive2://<host1>:<port1>/dbName;sess_var_list?hive_conf_list#hive_var_list + StringBuilder url = new StringBuilder(""); + if (cp != null) { + if (cp.getHost() != null) { + url.append(cp.getHost()); + url.append(":"); + url.append(cp.getPort()); + url.append("/"); + url.append(cp.getDbName()); + // Add session vars + if ((cp.getSessionVars() != null) && (!cp.getSessionVars().isEmpty())) { + for (Entry<String, String> sessVar : cp.getSessionVars().entrySet()) { + if ((sessVar.getKey().equalsIgnoreCase(JdbcConnectionParams.SERVICE_DISCOVERY_MODE)) + || (sessVar.getKey().equalsIgnoreCase(JdbcConnectionParams.ZOOKEEPER_NAMESPACE))) { + continue; + } + url.append(";"); + url.append(sessVar.getKey()); + url.append("="); + url.append(sessVar.getValue()); + } + } + // Add hive confs + if ((cp.getHiveConfs() != null) && (!cp.getHiveConfs().isEmpty())) { + url.append("?"); + boolean firstKV = true; + for (Entry<String, String> hiveConf : cp.getHiveConfs().entrySet()) { + if (!firstKV) { + url.append(";"); + } else { + firstKV = false; + } + url.append(hiveConf.getKey()); + url.append("="); + url.append(hiveConf.getValue()); + } + } + // Add hive vars + if ((cp.getHiveVars() != null) && (!cp.getHiveVars().isEmpty())) { + url.append("#"); + boolean firstKV = true; + for (Entry<String, String> hiveVar : cp.getHiveVars().entrySet()) { + if (!firstKV) { + url.append(";"); + } else { + firstKV = false; + } + url.append(hiveVar.getKey()); + url.append("="); + url.append(hiveVar.getValue()); + } + } + } else { + return url.toString(); + } + } + return url.toString(); + } public HiveConnection(String uri, Properties info) throws SQLException { setupLoginTimeout(); http://git-wip-us.apache.org/repos/asf/hive/blob/3635c4db/jdbc/src/java/org/apache/hive/jdbc/Utils.java ---------------------------------------------------------------------- diff --git a/jdbc/src/java/org/apache/hive/jdbc/Utils.java b/jdbc/src/java/org/apache/hive/jdbc/Utils.java index 42b3975..852942e 100644 --- a/jdbc/src/java/org/apache/hive/jdbc/Utils.java +++ b/jdbc/src/java/org/apache/hive/jdbc/Utils.java @@ -114,7 +114,7 @@ public class Utils { // Use ZooKeeper for indirection while using dynamic service discovery public static final String SERVICE_DISCOVERY_MODE_ZOOKEEPER = "zooKeeper"; public static final String SERVICE_DISCOVERY_MODE_ZOOKEEPER_HA = "zooKeeperHA"; - static final String ZOOKEEPER_NAMESPACE = "zooKeeperNamespace"; + public static final String ZOOKEEPER_NAMESPACE = "zooKeeperNamespace"; // Default namespace value on ZooKeeper. // This value is used if the param "zooKeeperNamespace" is not specified in the JDBC Uri. static final String ZOOKEEPER_DEFAULT_NAMESPACE = "hiveserver2";
