HIVE-14063: beeline to auto connect to the HiveServer2 (Vihang Karajgaonkar, reviewed by Aihua Xu, Szehon Ho, Sergio Peña)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8029e11b Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8029e11b Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8029e11b Branch: refs/heads/hive-14535 Commit: 8029e11b3c913e0aebc0a719a19b85d96db28a32 Parents: 9caf230 Author: Aihua Xu <[email protected]> Authored: Mon Oct 17 13:06:30 2016 -0400 Committer: Aihua Xu <[email protected]> Committed: Mon Oct 17 13:08:29 2016 -0400 ---------------------------------------------------------------------- .../java/org/apache/hive/beeline/BeeLine.java | 176 ++++++++++----- .../java/org/apache/hive/beeline/Commands.java | 52 +++-- .../BeelineHS2ConnectionFileParseException.java | 30 +++ .../hs2connection/HS2ConnectionFileParser.java | 88 ++++++++ .../hs2connection/HS2ConnectionFileUtils.java | 119 +++++++++++ .../HiveSiteHS2ConnectionFileParser.java | 172 +++++++++++++++ .../UserHS2ConnectionFileParser.java | 117 ++++++++++ .../apache/hive/beeline/TestBeeLineHistory.java | 4 +- .../TestUserHS2ConnectionFileParser.java | 211 ++++++++++++++++++ beeline/src/test/resources/hive-site.xml | 5 + .../test-hs2-conn-conf-kerberos-http.xml | 48 +++++ .../test-hs2-conn-conf-kerberos-nossl.xml | 32 +++ .../test-hs2-conn-conf-kerberos-ssl.xml | 40 ++++ .../resources/test-hs2-connection-conf-list.xml | 36 ++++ .../test-hs2-connection-config-noauth.xml | 28 +++ .../test-hs2-connection-multi-conf-list.xml | 37 ++++ .../test-hs2-connection-zookeeper-config.xml | 32 +++ .../TestBeelineConnectionUsingHiveSite.java | 109 ++++++++++ .../TestBeelineWithHS2ConnectionFile.java | 214 +++++++++++++++++++ .../TestBeelineWithUserHs2ConnectionFile.java | 129 +++++++++++ 20 files changed, 1612 insertions(+), 67 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/8029e11b/beeline/src/java/org/apache/hive/beeline/BeeLine.java ---------------------------------------------------------------------- diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLine.java b/beeline/src/java/org/apache/hive/beeline/BeeLine.java index 79922d2..fdbe0a2 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLine.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLine.java @@ -28,6 +28,7 @@ import java.io.Closeable; import java.io.EOFException; import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; @@ -93,8 +94,15 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.io.IOUtils; import org.apache.hive.beeline.cli.CliOptionsProcessor; import org.apache.hive.common.util.ShutdownHookManager; +import org.apache.hive.beeline.hs2connection.BeelineHS2ConnectionFileParseException; +import org.apache.hive.beeline.hs2connection.HS2ConnectionFileUtils; +import org.apache.hive.beeline.hs2connection.UserHS2ConnectionFileParser; +import org.apache.hive.beeline.hs2connection.HS2ConnectionFileParser; +import org.apache.hive.beeline.hs2connection.HiveSiteHS2ConnectionFileParser; import org.apache.thrift.transport.TTransportException; +import com.google.common.annotations.VisibleForTesting; + import org.apache.hive.jdbc.Utils; import org.apache.hive.jdbc.Utils.JdbcConnectionParams; @@ -279,7 +287,6 @@ public class BeeLine implements Closeable { "org.apache.hadoop.hive.jdbc.HiveDriver", })); - static { try { Class.forName("jline.console.ConsoleReader"); @@ -728,6 +735,46 @@ public class BeeLine implements Closeable { return -1; } + boolean connSuccessful = connectUsingArgs(cl); + // checks if default hs2 connection configuration file is present + // and uses it to connect if found + // no-op if the file is not present + if(!connSuccessful && !exit) { + connSuccessful = defaultBeelineConnect(); + } + + int code = 0; + if (cl.getOptionValues('e') != null) { + commands = Arrays.asList(cl.getOptionValues('e')); + } + + if (!commands.isEmpty() && getOpts().getScriptFile() != null) { + error("The '-e' and '-f' options cannot be specified simultaneously"); + return 1; + } else if(!commands.isEmpty() && !connSuccessful) { + error("Cannot run commands specified using -e. No current connection"); + return 1; + } + if (!commands.isEmpty()) { + for (Iterator<String> i = commands.iterator(); i.hasNext();) { + String command = i.next().toString(); + debug(loc("executing-command", command)); + if (!dispatch(command)) { + code++; + } + } + exit = true; // execute and exit + } + return code; + } + + + /* + * Connects using the command line arguments. There are two + * possible ways to connect here 1. using the cmd line arguments like -u + * or using !properties <property-file> + */ + private boolean connectUsingArgs(CommandLine cl) { String driver = null, user = null, pass = null, url = null; String auth = null; @@ -735,7 +782,7 @@ public class BeeLine implements Closeable { if (cl.hasOption("help")) { usage(); getOpts().setHelpAsked(true); - return 0; + return true; } Properties hiveVars = cl.getOptionProperties("hivevar"); @@ -764,24 +811,7 @@ public class BeeLine implements Closeable { } getOpts().setInitFiles(cl.getOptionValues("i")); getOpts().setScriptFile(cl.getOptionValue("f")); - if (cl.getOptionValues('e') != null) { - commands = Arrays.asList(cl.getOptionValues('e')); - } - - if (!commands.isEmpty() && getOpts().getScriptFile() != null) { - System.err.println("The '-e' and '-f' options cannot be specified simultaneously"); - return 1; - } - // TODO: temporary disable this for easier debugging - /* - if (url == null) { - url = BEELINE_DEFAULT_JDBC_URL; - } - if (driver == null) { - driver = BEELINE_DEFAULT_JDBC_DRIVER; - } - */ if (url != null) { if (user == null) { @@ -795,38 +825,25 @@ public class BeeLine implements Closeable { String com = constructCmd(url, user, pass, driver, false); String comForDebug = constructCmd(url, user, pass, driver, true); debug("issuing: " + comForDebug); - dispatch(com); - } - - // load property file - String propertyFile = cl.getOptionValue("property-file"); - if (propertyFile != null) { - try { - this.consoleReader = new ConsoleReader(); - } catch (IOException e) { - handleException(e); - } - if (!dispatch("!properties " + propertyFile)) { - exit = true; - return 1; - } - } - - int code = 0; - if (!commands.isEmpty()) { - for (Iterator<String> i = commands.iterator(); i.hasNext();) { - String command = i.next().toString(); - debug(loc("executing-command", command)); - if (!dispatch(command)) { - code++; + return dispatch(com); + } else { + // load property file + String propertyFile = cl.getOptionValue("property-file"); + if (propertyFile != null) { + try { + this.consoleReader = new ConsoleReader(); + } catch (IOException e) { + handleException(e); + } + if (!dispatch("!properties " + propertyFile)) { + exit = true; + return false; } } - exit = true; // execute and exit } - return code; + return false; } - private void setHiveConfVar(String key, String val) { getOpts().getHiveConfVariables().put(key, val); if (HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname.equals(key) && "mr".equals(val)) { @@ -906,13 +923,74 @@ public class BeeLine implements Closeable { } catch (Exception e) { // ignore } - ConsoleReader reader = getConsoleReader(inputStream); + ConsoleReader reader = initializeConsoleReader(inputStream); return execute(reader, false); } finally { close(); } } + /* + * Attempts to make a connection using default HS2 connection config file if available + * if there connection is not made return false + * + */ + private boolean defaultBeelineConnect() { + String url; + try { + initializeConsoleReader(null); + url = getDefaultConnectionUrl(); + if (url == null) { + debug("Default hs2 connection config file not found"); + return false; + } + } catch (BeelineHS2ConnectionFileParseException | IOException e) { + error(e); + return false; + } + return dispatch("!connect " + url); + } + + + private String getDefaultConnectionUrl() throws BeelineHS2ConnectionFileParseException { + HS2ConnectionFileParser userHS2ConnFileParser = getUserHS2ConnFileParser(); + if (!userHS2ConnFileParser.configExists()) { + // nothing to do if there is no user HS2 connection configuration file + return null; + } + // get the connection properties from user specific config file + Properties userConnectionProperties = userHS2ConnFileParser.getConnectionProperties(); + // load the HS2 connection url properties from hive-site.xml if it is present in the classpath + HS2ConnectionFileParser hiveSiteParser = getHiveSiteHS2ConnectionFileParser(); + Properties hiveSiteConnectionProperties = hiveSiteParser.getConnectionProperties(); + // add/override properties found from hive-site with user-specific properties + for (String key : userConnectionProperties.stringPropertyNames()) { + if (hiveSiteConnectionProperties.containsKey(key)) { + debug("Overriding connection url property " + key + + " from user connection configuration file"); + } + hiveSiteConnectionProperties.setProperty(key, userConnectionProperties.getProperty(key)); + } + // return the url based on the aggregated connection properties + return HS2ConnectionFileUtils.getUrl(hiveSiteConnectionProperties); + } + + /* + * Increased visibility of this method is only for providing better test coverage + */ + @VisibleForTesting + public HS2ConnectionFileParser getUserHS2ConnFileParser() { + return new UserHS2ConnectionFileParser(); + } + + /* + * Increased visibility of this method is only for providing better test coverage + */ + @VisibleForTesting + public HS2ConnectionFileParser getHiveSiteHS2ConnectionFileParser() { + return new HiveSiteHS2ConnectionFileParser(); + } + int runInit() { String initFiles[] = getOpts().getInitFiles(); if (initFiles != null && initFiles.length != 0) { @@ -972,7 +1050,7 @@ public class BeeLine implements Closeable { } else { initStream = new FileInputStream(fileName); } - return execute(getConsoleReader(initStream), !getOpts().getForce()); + return execute(initializeConsoleReader(initStream), !getOpts().getForce()); } catch (Throwable t) { handleException(t); return ERRNO_OTHER; @@ -1017,7 +1095,7 @@ public class BeeLine implements Closeable { commands.closeall(null); } - public ConsoleReader getConsoleReader(InputStream inputStream) throws IOException { + public ConsoleReader initializeConsoleReader(InputStream inputStream) throws IOException { if (inputStream != null) { // ### NOTE: fix for sf.net bug 879425. // Working around an issue in jline-2.1.2, see https://github.com/jline/jline/issues/10 http://git-wip-us.apache.org/repos/asf/hive/blob/8029e11b/beeline/src/java/org/apache/hive/beeline/Commands.java ---------------------------------------------------------------------- diff --git a/beeline/src/java/org/apache/hive/beeline/Commands.java b/beeline/src/java/org/apache/hive/beeline/Commands.java index 039e354..6c3e7f7 100644 --- a/beeline/src/java/org/apache/hive/beeline/Commands.java +++ b/beeline/src/java/org/apache/hive/beeline/Commands.java @@ -22,12 +22,6 @@ */ package org.apache.hive.beeline; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveVariableSource; -import org.apache.hadoop.hive.conf.SystemVariables; -import org.apache.hadoop.hive.conf.VariableSubstitution; -import org.apache.hadoop.io.IOUtils; - import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; @@ -46,8 +40,8 @@ import java.sql.DatabaseMetaData; import java.sql.Driver; import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; import java.sql.SQLWarning; +import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -60,6 +54,12 @@ import java.util.Set; import java.util.TreeSet; import org.apache.hadoop.hive.common.cli.ShellCmdExecutor; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveVariableSource; +import org.apache.hadoop.hive.conf.SystemVariables; +import org.apache.hadoop.hive.conf.VariableSubstitution; +import org.apache.hadoop.io.IOUtils; import org.apache.hive.jdbc.HiveStatement; import org.apache.hive.jdbc.Utils; import org.apache.hive.jdbc.Utils.JdbcConnectionParams; @@ -1037,9 +1037,7 @@ public class Commands { } public String handleMultiLineCmd(String line) throws IOException { - //When using -e, console reader is not initialized and command is a single line - while (beeLine.getConsoleReader() != null && !(line.trim().endsWith(";")) && beeLine.getOpts() - .isAllowMultiLineCommand()) { + while (isMultiLine(line) && beeLine.getOpts().isAllowMultiLineCommand()) { StringBuilder prompt = new StringBuilder(beeLine.getPrompt()); if (!beeLine.getOpts().isSilent()) { @@ -1049,7 +1047,6 @@ public class Commands { } } } - String extra; if (beeLine.getOpts().isSilent() && beeLine.getOpts().getScriptFile() != null) { extra = beeLine.getConsoleReader().readLine(null, jline.console.ConsoleReader.NULL_MASK); @@ -1067,6 +1064,26 @@ public class Commands { return line; } + //returns true if statement represented by line is + //not complete and needs additional reading from + //console. Used in handleMultiLineCmd method + //assumes line would never be null when this method is called + private boolean isMultiLine(String line) { + line = line.trim(); + if (line.endsWith(";")) { + return false; + } + if (beeLine.isComment(line)) { + return false; + } + // handles the case like line = show tables; --test comment + List<String> cmds = getCmdList(line, false); + if(!cmds.isEmpty() && cmds.get(cmds.size()-1).trim().startsWith("--")) { + return false; + } + return true; + } + public boolean sql(String line, boolean entireLineAsCommand) { return execute(line, false, entireLineAsCommand); } @@ -1409,6 +1426,10 @@ public class Commands { value = Utils.parsePropertyFromUrl(url, JdbcConnectionParams.AUTH_PASSWD); if (value != null) { props.setProperty(JdbcConnectionParams.AUTH_PASSWD, value); + } else { + //if the password is not provided, beeline assumes a empty string as + //password + props.setProperty(JdbcConnectionParams.AUTH_PASSWD, ""); } } @@ -1465,7 +1486,6 @@ public class Commands { return null; } - public boolean connect(Properties props) throws IOException { String url = getProperty(props, new String[] { JdbcConnectionParams.PROPERTY_URL, @@ -1507,13 +1527,14 @@ public class Commands { beeLine.info("Connecting to " + url); if (Utils.parsePropertyFromUrl(url, JdbcConnectionParams.AUTH_PRINCIPAL) == null) { + String urlForPrompt = url.substring(0, url.contains(";") ? url.indexOf(';') : url.length()); if (username == null) { - username = beeLine.getConsoleReader().readLine("Enter username for " + url + ": "); + username = beeLine.getConsoleReader().readLine("Enter username for " + urlForPrompt + ": "); } props.setProperty(JdbcConnectionParams.AUTH_USER, username); if (password == null) { - password = beeLine.getConsoleReader().readLine("Enter password for " + url + ": ", - new Character('*')); + password = beeLine.getConsoleReader().readLine("Enter password for " + urlForPrompt + ": ", + new Character('*')); } props.setProperty(JdbcConnectionParams.AUTH_PASSWD, password); } @@ -1539,7 +1560,6 @@ public class Commands { } } - public boolean rehash(String line) { try { if (!(beeLine.assertConnection())) { http://git-wip-us.apache.org/repos/asf/hive/blob/8029e11b/beeline/src/java/org/apache/hive/beeline/hs2connection/BeelineHS2ConnectionFileParseException.java ---------------------------------------------------------------------- diff --git a/beeline/src/java/org/apache/hive/beeline/hs2connection/BeelineHS2ConnectionFileParseException.java b/beeline/src/java/org/apache/hive/beeline/hs2connection/BeelineHS2ConnectionFileParseException.java new file mode 100644 index 0000000..6e60186 --- /dev/null +++ b/beeline/src/java/org/apache/hive/beeline/hs2connection/BeelineHS2ConnectionFileParseException.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline.hs2connection; + +public class BeelineHS2ConnectionFileParseException extends Exception { + private static final long serialVersionUID = -748635913718300617L; + + BeelineHS2ConnectionFileParseException(String msg, Exception e) { + super(msg, e); + } + + public BeelineHS2ConnectionFileParseException(String msg) { + super(msg); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/8029e11b/beeline/src/java/org/apache/hive/beeline/hs2connection/HS2ConnectionFileParser.java ---------------------------------------------------------------------- diff --git a/beeline/src/java/org/apache/hive/beeline/hs2connection/HS2ConnectionFileParser.java b/beeline/src/java/org/apache/hive/beeline/hs2connection/HS2ConnectionFileParser.java new file mode 100644 index 0000000..7173ee8 --- /dev/null +++ b/beeline/src/java/org/apache/hive/beeline/hs2connection/HS2ConnectionFileParser.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline.hs2connection; + +import java.util.Properties; + +/** + * HS2ConnectionFileParser provides a interface to be used by Beeline to parse a configuration (file + * or otherwise) to return a Java Properties object which contain key value pairs to be used to construct + * connection URL automatically for the Beeline connection + */ +public interface HS2ConnectionFileParser { + /** + * prefix string used for the keys + */ + public static final String BEELINE_CONNECTION_PROPERTY_PREFIX = "beeline.hs2.connection."; + /** + * Property key used to provide the URL prefix for the connection URL + */ + public static final String URL_PREFIX_PROPERTY_KEY = "url_prefix"; + /** + * Property key used to provide the default database in the connection URL + */ + public static final String DEFAULT_DB_PROPERTY_KEY = "defaultDB"; + /** + * Property key used to provide the hosts in the connection URL + */ + public static final String HOST_PROPERTY_KEY = "hosts"; + /** + * Property key used to provide the hive configuration key value pairs in the connection URL + */ + public static final String HIVE_CONF_PROPERTY_KEY = "hiveconf"; + /** + * Property key used to provide the hive variables in the connection URL + */ + public static final String HIVE_VAR_PROPERTY_KEY = "hivevar"; + + /** + * Returns a Java properties object which contain the key value pairs which can be used in the + * Beeline connection URL<p> + * The properties returned must include url_prefix and hosts<p> + * Following are some examples of the URLs and returned properties object<p> + * <ul> + * <li> jdbc:hive2://hs2-instance1.example.com:10000/default;user=hive;password=mypassword should + * return [ "url_prefix"="jdbc:hive2://", "hosts"="hs2-instance1.example.com:10000", + * "defaultDB"=default, "user"="hive", "password"="mypassword" ] + * <li>jdbc:hive2://zk-instance1:10001,zk-instance2:10002,zk-instance3:10003/default; + * serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2 should return [ + * "url_prefix"="jdbc:hive2://", + * "hosts"="zk-instance1:10001,zk-instance2:10002,zk-instance3:10003", "defaultDB"="default", + * "serviceDiscoveryMode"="zooKeeper", "zooKeeperNamespace"="hiveserver2" ] + * + * <li>If hive_conf_list and hive_var_list is present in the url it should return a comma separated + * key=value pairs for each of them<p> + * For example :<p> + * jdbc:hive2://hs2-instance1.example.com:10000/default;user=hive?hive.cli.print.currentdb=true; + * hive.cli.print.header=true#hivevar:mytbl=customers;hivevar:mycol=id it should return [ + * "url_prefix"="jdbc:hive2://", "hosts"="hs2-instance1.example.com:10000", "defaultDB"="default", + * "user"="hive", "hiveconf"="hive.cli.print.currentdb=true, hive.cli.print.header=true", + * "hivevar"="hivevar:mytb1=customers, hivevar:mycol=id" ] + * </ul> + * + * @return Properties object which contain connection URL properties for Beeline connection. Returns an empty properties + * object if the connection configuration is not found + * @throws BeelineHS2ConnectionFileParseException if there is invalid key with appropriate message + */ + Properties getConnectionProperties() throws BeelineHS2ConnectionFileParseException; + /** + * + * @return returns true if the configuration exists else returns false + */ + boolean configExists(); +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/8029e11b/beeline/src/java/org/apache/hive/beeline/hs2connection/HS2ConnectionFileUtils.java ---------------------------------------------------------------------- diff --git a/beeline/src/java/org/apache/hive/beeline/hs2connection/HS2ConnectionFileUtils.java b/beeline/src/java/org/apache/hive/beeline/hs2connection/HS2ConnectionFileUtils.java new file mode 100644 index 0000000..ae9c4df --- /dev/null +++ b/beeline/src/java/org/apache/hive/beeline/hs2connection/HS2ConnectionFileUtils.java @@ -0,0 +1,119 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline.hs2connection; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +public class HS2ConnectionFileUtils { + + public static String getUrl(Properties props) throws BeelineHS2ConnectionFileParseException { + if (props == null || props.isEmpty()) { + return null; + } + // use remove instead of get so that it is not parsed again + // in the for loop below + String urlPrefix = (String) props.remove(HS2ConnectionFileParser.URL_PREFIX_PROPERTY_KEY); + if (urlPrefix == null || urlPrefix.isEmpty()) { + throw new BeelineHS2ConnectionFileParseException("url_prefix parameter cannot be empty"); + } + + String hosts = (String) props.remove(HS2ConnectionFileParser.HOST_PROPERTY_KEY); + if (hosts == null || hosts.isEmpty()) { + throw new BeelineHS2ConnectionFileParseException("hosts parameter cannot be empty"); + } + String defaultDB = (String) props.remove(HS2ConnectionFileParser.DEFAULT_DB_PROPERTY_KEY); + if (defaultDB == null) { + defaultDB = "default"; + } + // collect the hiveConfList and HiveVarList separately so that they can be + // appended once all the session list are added to the url + String hiveConfProperties = ""; + if (props.containsKey(HS2ConnectionFileParser.HIVE_CONF_PROPERTY_KEY)) { + hiveConfProperties = extractHiveVariables( + (String) props.remove(HS2ConnectionFileParser.HIVE_CONF_PROPERTY_KEY), true); + } + + String hiveVarProperties = ""; + if (props.containsKey(HS2ConnectionFileParser.HIVE_VAR_PROPERTY_KEY)) { + hiveVarProperties = extractHiveVariables( + (String) props.remove(HS2ConnectionFileParser.HIVE_VAR_PROPERTY_KEY), false); + } + StringBuilder urlSb = new StringBuilder(); + urlSb.append(urlPrefix.trim()); + urlSb.append(hosts.trim()); + urlSb.append(File.separator); + urlSb.append(defaultDB.trim()); + List<String> keys = new ArrayList<>(props.stringPropertyNames()); + // sorting the keys from the properties helps to create + // a deterministic url which is tested for various configuration in + // TestHS2ConnectionConfigFileManager + Collections.sort(keys); + for (String propertyName : keys) { + urlSb.append(";"); + urlSb.append(propertyName); + urlSb.append("="); + urlSb.append(props.getProperty(propertyName)); + } + if (!hiveConfProperties.isEmpty()) { + urlSb.append(hiveConfProperties.toString()); + } + if (!hiveVarProperties.isEmpty()) { + urlSb.append(hiveVarProperties.toString()); + } + return urlSb.toString(); + } + + private static String extractHiveVariables(String propertyValue, boolean isHiveConf) + throws BeelineHS2ConnectionFileParseException { + StringBuilder hivePropertiesList = new StringBuilder(); + String delimiter; + if (isHiveConf) { + delimiter = "?"; + } else { + delimiter = "#"; + } + hivePropertiesList.append(delimiter); + addPropertyValues(propertyValue, hivePropertiesList); + return hivePropertiesList.toString(); + } + + private static void addPropertyValues(String value, StringBuilder hivePropertiesList) + throws BeelineHS2ConnectionFileParseException { + // There could be multiple keyValuePairs separated by comma + String[] values = value.split(","); + boolean first = true; + for (String keyValuePair : values) { + String[] keyValue = keyValuePair.split("="); + if (keyValue.length != 2) { + throw new BeelineHS2ConnectionFileParseException("Unable to parse " + keyValuePair + + " in hs2 connection config file"); + } + if (!first) { + hivePropertiesList.append(";"); + } + first = false; + hivePropertiesList.append(keyValue[0].trim()); + hivePropertiesList.append("="); + hivePropertiesList.append(keyValue[1].trim()); + } + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/8029e11b/beeline/src/java/org/apache/hive/beeline/hs2connection/HiveSiteHS2ConnectionFileParser.java ---------------------------------------------------------------------- diff --git a/beeline/src/java/org/apache/hive/beeline/hs2connection/HiveSiteHS2ConnectionFileParser.java b/beeline/src/java/org/apache/hive/beeline/hs2connection/HiveSiteHS2ConnectionFileParser.java new file mode 100644 index 0000000..f1814fa --- /dev/null +++ b/beeline/src/java/org/apache/hive/beeline/hs2connection/HiveSiteHS2ConnectionFileParser.java @@ -0,0 +1,172 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline.hs2connection; + +import java.net.InetAddress; +import java.net.URL; +import java.net.UnknownHostException; +import java.util.Properties; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.ServerUtils; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.annotations.VisibleForTesting; + +/* + * Looks for a hive-site.xml from the classpath. If found this class parses the hive-site.xml + * to return a set of connection properties which can be used to construct the connection url + * for Beeline connection + */ +public class HiveSiteHS2ConnectionFileParser implements HS2ConnectionFileParser { + private Configuration conf; + private final URL hiveSiteURI; + private final static String TRUSTSTORE_PASS_PROP = "javax.net.ssl.trustStorePassword"; + private final static String TRUSTSTORE_PROP = "javax.net.ssl.trustStore"; + private static final Logger log = LoggerFactory.getLogger(HiveSiteHS2ConnectionFileParser.class); + + public HiveSiteHS2ConnectionFileParser() { + hiveSiteURI = HiveConf.getHiveSiteLocation(); + conf = new Configuration(); + if(hiveSiteURI == null) { + log.debug("hive-site.xml not found for constructing the connection URL"); + } else { + log.info("Using hive-site.xml at " + hiveSiteURI); + conf.addResource(hiveSiteURI); + } + } + + @VisibleForTesting + void setHiveConf(HiveConf hiveConf) { + this.conf = hiveConf; + } + + @Override + public Properties getConnectionProperties() throws BeelineHS2ConnectionFileParseException { + Properties props = new Properties(); + if(!configExists()) { + return props; + } + props.setProperty(HS2ConnectionFileParser.URL_PREFIX_PROPERTY_KEY, "jdbc:hive2://"); + addHosts(props); + addSSL(props); + addKerberos(props); + addHttp(props); + return props; + } + + private void addSSL(Properties props) { + if (!HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_SERVER2_USE_SSL)) { + return; + } else { + props.setProperty("ssl", "true"); + } + String truststore = System.getenv(TRUSTSTORE_PROP); + if (truststore != null && truststore.isEmpty()) { + props.setProperty("sslTruststore", truststore); + } + String trustStorePassword = System.getenv(TRUSTSTORE_PASS_PROP); + if (trustStorePassword != null && !trustStorePassword.isEmpty()) { + props.setProperty("trustStorePassword", trustStorePassword); + } + String saslQop = HiveConf.getVar(conf, ConfVars.HIVE_SERVER2_THRIFT_SASL_QOP); + if (!"auth".equalsIgnoreCase(saslQop)) { + props.setProperty("sasl.qop", saslQop); + } + } + + private void addKerberos(Properties props) { + if ("KERBEROS".equals(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_SERVER2_AUTHENTICATION))) { + props.setProperty("principal", + HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL)); + } + } + + private void addHttp(Properties props) { + if ("http".equalsIgnoreCase(HiveConf.getVar(conf, ConfVars.HIVE_SERVER2_TRANSPORT_MODE))) { + props.setProperty("transportMode", "http"); + } else { + return; + } + props.setProperty("httpPath", HiveConf.getVar(conf, ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH)); + } + + private void addHosts(Properties props) throws BeelineHS2ConnectionFileParseException { + // if zk HA is enabled get hosts property + if (HiveConf.getBoolVar(conf, + HiveConf.ConfVars.HIVE_SERVER2_SUPPORT_DYNAMIC_SERVICE_DISCOVERY)) { + addZKServiceDiscoveryHosts(props); + } else { + addDefaultHS2Hosts(props); + } + } + + private void addZKServiceDiscoveryHosts(Properties props) + throws BeelineHS2ConnectionFileParseException { + props.setProperty("serviceDiscoveryMode", "zooKeeper"); + props.setProperty("zooKeeperNamespace", + HiveConf.getVar(conf, ConfVars.HIVE_SERVER2_ZOOKEEPER_NAMESPACE)); + props.setProperty("hosts", HiveConf.getVar(conf, ConfVars.HIVE_ZOOKEEPER_QUORUM)); + } + + private void addDefaultHS2Hosts(Properties props) throws BeelineHS2ConnectionFileParseException { + String hiveHost = System.getenv("HIVE_SERVER2_THRIFT_BIND_HOST"); + if (hiveHost == null) { + hiveHost = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST); + } + + InetAddress serverIPAddress; + try { + serverIPAddress = ServerUtils.getHostAddress(hiveHost); + } catch (UnknownHostException e) { + throw new BeelineHS2ConnectionFileParseException(e.getMessage(), e); + } + int portNum = getPortNum( + "http".equalsIgnoreCase(HiveConf.getVar(conf, ConfVars.HIVE_SERVER2_TRANSPORT_MODE))); + props.setProperty("hosts", serverIPAddress.getHostName() + ":" + portNum); + } + + private int getPortNum(boolean isHttp) { + String portString; + int portNum; + if (isHttp) { + portString = System.getenv("HIVE_SERVER2_THRIFT_HTTP_PORT"); + if (portString != null) { + portNum = Integer.parseInt(portString); + } else { + portNum = HiveConf.getIntVar(conf, ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT); + } + } else { + portString = System.getenv("HIVE_SERVER2_THRIFT_PORT"); + if (portString != null) { + portNum = Integer.parseInt(portString); + } else { + portNum = HiveConf.getIntVar(conf, ConfVars.HIVE_SERVER2_THRIFT_PORT); + } + } + return portNum; + } + + @Override + public boolean configExists() { + return (hiveSiteURI != null); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/8029e11b/beeline/src/java/org/apache/hive/beeline/hs2connection/UserHS2ConnectionFileParser.java ---------------------------------------------------------------------- diff --git a/beeline/src/java/org/apache/hive/beeline/hs2connection/UserHS2ConnectionFileParser.java b/beeline/src/java/org/apache/hive/beeline/hs2connection/UserHS2ConnectionFileParser.java new file mode 100644 index 0000000..93a6231 --- /dev/null +++ b/beeline/src/java/org/apache/hive/beeline/hs2connection/UserHS2ConnectionFileParser.java @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline.hs2connection; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.Map.Entry; +import java.util.Properties; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.annotations.VisibleForTesting; + +/** + * This class implements HS2ConnectionFileParser for the user-specific connection configuration + * file named beeline-hs2-connection.xml. The class looks for this file in + * ${user.home}/.beeline, ${HIVE_CONF_DIR} or /etc/conf/hive in that order and uses the first file + * found in the above locations. + */ +public class UserHS2ConnectionFileParser implements HS2ConnectionFileParser { + + public static final String DEFAULT_CONNECTION_CONFIG_FILE_NAME = "beeline-hs2-connection.xml"; + public static final String DEFAULT_BEELINE_USER_CONF_LOCATION = + System.getProperty("user.home") + File.separator + + (System.getProperty("os.name").toLowerCase().indexOf("windows") != -1 ? "" : ".") + + "beeline" + File.separator; + public static final String ETC_HIVE_CONF_LOCATION = + File.separator + "etc" + File.separator + "conf" + File.separator + "hive"; + + private final List<String> locations = new ArrayList<>(); + private static final Logger log = LoggerFactory.getLogger(UserHS2ConnectionFileParser.class); + + public UserHS2ConnectionFileParser() { + // file locations to be searched in the correct order + locations.add(DEFAULT_BEELINE_USER_CONF_LOCATION + DEFAULT_CONNECTION_CONFIG_FILE_NAME); + if (System.getenv("HIVE_CONF_DIR") != null) { + locations.add( + System.getenv("HIVE_CONF_DIR") + File.separator + DEFAULT_CONNECTION_CONFIG_FILE_NAME); + } + locations.add(ETC_HIVE_CONF_LOCATION + DEFAULT_CONNECTION_CONFIG_FILE_NAME); + } + + @VisibleForTesting + UserHS2ConnectionFileParser(List<String> testLocations) { + if(testLocations == null) { + return; + } + locations.addAll(testLocations); + } + + @Override + public Properties getConnectionProperties() throws BeelineHS2ConnectionFileParseException { + Properties props = new Properties(); + String fileLocation = getFileLocation(); + if (fileLocation == null) { + log.debug("User connection configuration file not found"); + return props; + } + log.info("Using connection configuration file at " + fileLocation); + props.setProperty(HS2ConnectionFileParser.URL_PREFIX_PROPERTY_KEY, "jdbc:hive2://"); + // load the properties from config file + Configuration conf = new Configuration(false); + conf.addResource(new Path(new File(fileLocation).toURI())); + try { + for (Entry<String, String> kv : conf) { + String key = kv.getKey(); + if (key.startsWith(BEELINE_CONNECTION_PROPERTY_PREFIX)) { + props.setProperty(key.substring(BEELINE_CONNECTION_PROPERTY_PREFIX.length()), + kv.getValue()); + } else { + log.warn("Ignoring " + key + " since it does not start with " + + BEELINE_CONNECTION_PROPERTY_PREFIX); + } + } + } catch (Exception ex) { + throw new BeelineHS2ConnectionFileParseException(ex.getMessage(), ex); + } + + return props; + } + + @Override + public boolean configExists() { + return (getFileLocation() != null); + } + /* + * This method looks in locations specified above and returns the first location where the file + * exists. If the file does not exist in any one of the locations it returns null + */ + String getFileLocation() { + for (String location : locations) { + if (new File(location).exists()) { + return location; + } + } + return null; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/8029e11b/beeline/src/test/org/apache/hive/beeline/TestBeeLineHistory.java ---------------------------------------------------------------------- diff --git a/beeline/src/test/org/apache/hive/beeline/TestBeeLineHistory.java b/beeline/src/test/org/apache/hive/beeline/TestBeeLineHistory.java index e861976..489db56 100644 --- a/beeline/src/test/org/apache/hive/beeline/TestBeeLineHistory.java +++ b/beeline/src/test/org/apache/hive/beeline/TestBeeLineHistory.java @@ -58,7 +58,7 @@ public class TestBeeLineHistory { BeeLine beeline = new BeeLine(); beeline.getOpts().setHistoryFile(fileName); beeline.setOutputStream(ops); - beeline.getConsoleReader(null); + beeline.initializeConsoleReader(null); beeline.dispatch("!history"); String output = os.toString("UTF-8"); int numHistories = output.split("\n").length; @@ -73,7 +73,7 @@ public class TestBeeLineHistory { BeeLine beeline = new BeeLine(); beeline.getOpts().setHistoryFile(fileName); beeline.setOutputStream(ops); - beeline.getConsoleReader(null); + beeline.initializeConsoleReader(null); beeline.dispatch("!history"); String output = os.toString("UTF-8"); String[] tmp = output.split("\n"); http://git-wip-us.apache.org/repos/asf/hive/blob/8029e11b/beeline/src/test/org/apache/hive/beeline/hs2connection/TestUserHS2ConnectionFileParser.java ---------------------------------------------------------------------- diff --git a/beeline/src/test/org/apache/hive/beeline/hs2connection/TestUserHS2ConnectionFileParser.java b/beeline/src/test/org/apache/hive/beeline/hs2connection/TestUserHS2ConnectionFileParser.java new file mode 100644 index 0000000..05b2af4 --- /dev/null +++ b/beeline/src/test/org/apache/hive/beeline/hs2connection/TestUserHS2ConnectionFileParser.java @@ -0,0 +1,211 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline.hs2connection; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hive.beeline.hs2connection.BeelineHS2ConnectionFileParseException; +import org.apache.hive.beeline.hs2connection.UserHS2ConnectionFileParser; +import org.apache.hive.beeline.hs2connection.HS2ConnectionFileUtils; +import org.apache.hive.common.util.HiveTestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +public class TestUserHS2ConnectionFileParser { + private final String LOCATION_1 = System.getProperty("java.io.tmpdir") + "loc1" + File.separator + + UserHS2ConnectionFileParser.DEFAULT_CONNECTION_CONFIG_FILE_NAME; + + private final String LOCATION_2 = System.getProperty("java.io.tmpdir") + "loc2" + File.separator + + UserHS2ConnectionFileParser.DEFAULT_CONNECTION_CONFIG_FILE_NAME; + + private final String LOCATION_3 = System.getProperty("java.io.tmpdir") + "loc3" + File.separator + + UserHS2ConnectionFileParser.DEFAULT_CONNECTION_CONFIG_FILE_NAME; + + List<String> testLocations = new ArrayList<>(); + + @After + public void cleanUp() { + try { + deleteFile(LOCATION_1); + deleteFile(LOCATION_2); + deleteFile(LOCATION_3); + } catch (Exception e) { + e.printStackTrace(); + } + testLocations.clear(); + } + + @Test + public void testParseNoAuthentication() throws BeelineHS2ConnectionFileParseException { + String url = getParsedUrlFromConfigFile("test-hs2-connection-config-noauth.xml"); + String expectedUrl = "jdbc:hive2://localhost:10000/default;user=hive"; + Assert.assertTrue("Expected " + expectedUrl + " got " + url, expectedUrl.equals(url)); + } + + @Test + public void testParseZookeeper() throws BeelineHS2ConnectionFileParseException { + String url = getParsedUrlFromConfigFile("test-hs2-connection-zookeeper-config.xml"); + String expectedUrl = + "jdbc:hive2://zk-node-1:10000,zk-node-2:10001,zk-node-3:10004/default;serviceDiscoveryMode=zookeeper;zooKeeperNamespace=hiveserver2"; + Assert.assertTrue("Expected " + expectedUrl + " got " + url, expectedUrl.equals(url)); + } + + @Test + public void testParseWithKerberosNoSSL() throws BeelineHS2ConnectionFileParseException { + String url = getParsedUrlFromConfigFile("test-hs2-conn-conf-kerberos-nossl.xml"); + String expectedUrl = + "jdbc:hive2://localhost:10000/default;principal=hive/[email protected];ssl=false"; + Assert.assertTrue("Expected " + expectedUrl + " got " + url, expectedUrl.equals(url)); + } + + @Test + public void testParseWithKerberosSSL() throws BeelineHS2ConnectionFileParseException { + String url = getParsedUrlFromConfigFile("test-hs2-conn-conf-kerberos-ssl.xml"); + String expectedUrl = + "jdbc:hive2://localhost:10000/default;principal=hive/[email protected];ssl=true;" + + "sslTrustStore=test/truststore;trustStorePassword=testTruststorePassword"; + Assert.assertTrue("Expected " + expectedUrl + " got " + url, expectedUrl.equals(url)); + } + + @Test + public void testParseWithSSLAndHttpMode() throws BeelineHS2ConnectionFileParseException { + String url = getParsedUrlFromConfigFile("test-hs2-conn-conf-kerberos-http.xml"); + String expectedUrl = + "jdbc:hive2://localhost:10000/default;httpPath=testHTTPPath;principal=hive/[email protected];" + + "ssl=true;sslTrustStore=test/truststore;transportMode=http;trustStorePassword=testTruststorePassword"; + Assert.assertTrue("Expected " + expectedUrl + " got " + url, expectedUrl.equals(url)); + } + + @Test + public void testUrlWithHiveConfValues() throws Exception { + String url = getParsedUrlFromConfigFile("test-hs2-connection-conf-list.xml"); + String expectedUrl = + "jdbc:hive2://localhost:10000/default;user=hive?hive.cli.print.current.db=false#testVarName1=value1"; + Assert.assertTrue("Expected " + expectedUrl + " got " + url, expectedUrl.equals(url)); + } + + @Test + public void testUrlWithMultipleHiveConfValues() throws Exception { + String url = getParsedUrlFromConfigFile("test-hs2-connection-multi-conf-list.xml"); + String expectedUrl = + "jdbc:hive2://localhost:10000/default;user=hive?hive.cli.print.current.db=true;" + + "hive.cli.print.header=true#testVarName1=value1;testVarName2=value2"; + Assert.assertTrue("Expected " + expectedUrl + " got " + url, expectedUrl.equals(url)); + } + + /* + * Tests if null value returned when file is not present in any of the lookup locations + */ + @Test + public void testNoLocationFoundCase() throws Exception { + testLocations.add(LOCATION_1); + testLocations.add(LOCATION_2); + testLocations.add(LOCATION_3); + UserHS2ConnectionFileParser testHS2ConfigManager = + new UserHS2ConnectionFileParser(testLocations); + Assert.assertTrue(testHS2ConfigManager.getConnectionProperties().isEmpty()); + } + + /* + * Tests if LOCATION_1 is returned when file is present in the first directory in lookup order + */ + @Test + public void testGetLocation1() throws Exception { + createNewFile(LOCATION_1); + testLocations.add(LOCATION_1); + testLocations.add(LOCATION_2); + testLocations.add(LOCATION_3); + UserHS2ConnectionFileParser testHS2ConfigManager = + new UserHS2ConnectionFileParser(testLocations); + Assert.assertTrue("File location " + LOCATION_1 + " was not returned", + LOCATION_1.equals(testHS2ConfigManager.getFileLocation())); + } + + /* + * Tests if LOCATION_3 is returned when the first file is found is later in lookup order + */ + @Test + public void testGetLocation3() throws Exception { + createNewFile(LOCATION_3); + testLocations.add(LOCATION_1); + testLocations.add(LOCATION_2); + testLocations.add(LOCATION_3); + UserHS2ConnectionFileParser testHS2ConfigManager = + new UserHS2ConnectionFileParser(testLocations); + Assert.assertTrue("File location " + LOCATION_3 + " was not returned", + LOCATION_3.equals(testHS2ConfigManager.getFileLocation())); + } + + /* + * Tests if it returns the first file present in the lookup order when files are present in the + * lookup order + */ + @Test + public void testGetLocationOrder() throws Exception { + createNewFile(LOCATION_2); + createNewFile(LOCATION_3); + testLocations.add(LOCATION_1); + testLocations.add(LOCATION_2); + testLocations.add(LOCATION_3); + UserHS2ConnectionFileParser testHS2ConfigManager = + new UserHS2ConnectionFileParser(testLocations); + Assert.assertTrue("File location " + LOCATION_2 + " was not returned", + LOCATION_2.equals(testHS2ConfigManager.getFileLocation())); + } + + private String getParsedUrlFromConfigFile(String filename) + throws BeelineHS2ConnectionFileParseException { + String path = HiveTestUtils.getFileFromClasspath(filename); + testLocations.add(path); + UserHS2ConnectionFileParser testHS2ConfigManager = + new UserHS2ConnectionFileParser(testLocations); + + String url = HS2ConnectionFileUtils.getUrl(testHS2ConfigManager.getConnectionProperties()); + return url; + } + + private void createNewFile(final String path) throws Exception { + File file = new File(path); + if (file.exists()) { + return; + } + String dir = path.substring(0, + path.indexOf(UserHS2ConnectionFileParser.DEFAULT_CONNECTION_CONFIG_FILE_NAME)); + if (!new File(dir).exists()) { + if (!new File(dir).mkdirs()) { + throw new Exception("Could not create directory " + dir); + } + } + if (!file.createNewFile()) { + throw new Exception("Could not create new file at " + path); + } + } + + private void deleteFile(final String path) throws Exception { + File file = new File(path); + if (file.exists()) { + if (!file.delete()) { + throw new Exception("Could not delete file at " + path); + } + } + } + +} http://git-wip-us.apache.org/repos/asf/hive/blob/8029e11b/beeline/src/test/resources/hive-site.xml ---------------------------------------------------------------------- diff --git a/beeline/src/test/resources/hive-site.xml b/beeline/src/test/resources/hive-site.xml index 5f310d6..fdda94b 100644 --- a/beeline/src/test/resources/hive-site.xml +++ b/beeline/src/test/resources/hive-site.xml @@ -45,4 +45,9 @@ <value>${test.tmp.dir}/warehouse</value> <description></description> </property> + <property> + <name>test.data.files</name> + <value>${hive.root}/data/files</value> + <description></description> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/hive/blob/8029e11b/beeline/src/test/resources/test-hs2-conn-conf-kerberos-http.xml ---------------------------------------------------------------------- diff --git a/beeline/src/test/resources/test-hs2-conn-conf-kerberos-http.xml b/beeline/src/test/resources/test-hs2-conn-conf-kerberos-http.xml new file mode 100644 index 0000000..75f4fb8 --- /dev/null +++ b/beeline/src/test/resources/test-hs2-conn-conf-kerberos-http.xml @@ -0,0 +1,48 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration> +<property> + <name>beeline.hs2.connection.hosts</name> + <value>localhost:10000</value> +</property> +<property> + <name>beeline.hs2.connection.principal</name> + <value>hive/[email protected]</value> +</property> +<property> + <name>beeline.hs2.connection.ssl</name> + <value>true</value> +</property> +<property> + <name>beeline.hs2.connection.sslTrustStore</name> + <value>test/truststore</value> +</property> +<property> + <name>beeline.hs2.connection.trustStorePassword</name> + <value>testTruststorePassword</value> +</property> +<property> + <name>beeline.hs2.connection.transportMode</name> + <value>http</value> +</property> +<property> + <name>beeline.hs2.connection.httpPath</name> + <value>testHTTPPath</value> +</property> +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/8029e11b/beeline/src/test/resources/test-hs2-conn-conf-kerberos-nossl.xml ---------------------------------------------------------------------- diff --git a/beeline/src/test/resources/test-hs2-conn-conf-kerberos-nossl.xml b/beeline/src/test/resources/test-hs2-conn-conf-kerberos-nossl.xml new file mode 100644 index 0000000..dbceee5 --- /dev/null +++ b/beeline/src/test/resources/test-hs2-conn-conf-kerberos-nossl.xml @@ -0,0 +1,32 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration> +<property> + <name>beeline.hs2.connection.hosts</name> + <value>localhost:10000</value> +</property> +<property> + <name>beeline.hs2.connection.principal</name> + <value>hive/[email protected]</value> +</property> +<property> + <name>beeline.hs2.connection.ssl</name> + <value>false</value> +</property> +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/8029e11b/beeline/src/test/resources/test-hs2-conn-conf-kerberos-ssl.xml ---------------------------------------------------------------------- diff --git a/beeline/src/test/resources/test-hs2-conn-conf-kerberos-ssl.xml b/beeline/src/test/resources/test-hs2-conn-conf-kerberos-ssl.xml new file mode 100644 index 0000000..7dce56c --- /dev/null +++ b/beeline/src/test/resources/test-hs2-conn-conf-kerberos-ssl.xml @@ -0,0 +1,40 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration> +<property> + <name>beeline.hs2.connection.hosts</name> + <value>localhost:10000</value> +</property> +<property> + <name>beeline.hs2.connection.principal</name> + <value>hive/[email protected]</value> +</property> +<property> + <name>beeline.hs2.connection.ssl</name> + <value>true</value> +</property> +<property> + <name>beeline.hs2.connection.sslTrustStore</name> + <value>test/truststore</value> +</property> +<property> + <name>beeline.hs2.connection.trustStorePassword</name> + <value>testTruststorePassword</value> +</property> +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/8029e11b/beeline/src/test/resources/test-hs2-connection-conf-list.xml ---------------------------------------------------------------------- diff --git a/beeline/src/test/resources/test-hs2-connection-conf-list.xml b/beeline/src/test/resources/test-hs2-connection-conf-list.xml new file mode 100644 index 0000000..6c022b1 --- /dev/null +++ b/beeline/src/test/resources/test-hs2-connection-conf-list.xml @@ -0,0 +1,36 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration> +<property> + <name>beeline.hs2.connection.hosts</name> + <value>localhost:10000</value> +</property> +<property> + <name>beeline.hs2.connection.user</name> + <value>hive</value> +</property> +<property> + <name>beeline.hs2.connection.hiveconf</name> + <value>hive.cli.print.current.db=false</value> +</property> +<property> + <name>beeline.hs2.connection.hivevar</name> + <value>testVarName1=value1</value> +</property> +</configuration> http://git-wip-us.apache.org/repos/asf/hive/blob/8029e11b/beeline/src/test/resources/test-hs2-connection-config-noauth.xml ---------------------------------------------------------------------- diff --git a/beeline/src/test/resources/test-hs2-connection-config-noauth.xml b/beeline/src/test/resources/test-hs2-connection-config-noauth.xml new file mode 100644 index 0000000..7a858e0 --- /dev/null +++ b/beeline/src/test/resources/test-hs2-connection-config-noauth.xml @@ -0,0 +1,28 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration> +<property> + <name>beeline.hs2.connection.hosts</name> + <value>localhost:10000</value> +</property> +<property> + <name>beeline.hs2.connection.user</name> + <value>hive</value> +</property> +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/8029e11b/beeline/src/test/resources/test-hs2-connection-multi-conf-list.xml ---------------------------------------------------------------------- diff --git a/beeline/src/test/resources/test-hs2-connection-multi-conf-list.xml b/beeline/src/test/resources/test-hs2-connection-multi-conf-list.xml new file mode 100644 index 0000000..7faae44 --- /dev/null +++ b/beeline/src/test/resources/test-hs2-connection-multi-conf-list.xml @@ -0,0 +1,37 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration> +<property> + <name>beeline.hs2.connection.hosts</name> + <value>localhost:10000</value> +</property> +<property> + <name>beeline.hs2.connection.user</name> + <value>hive</value> +</property> +<property> + <name>beeline.hs2.connection.hiveconf</name> + <value>hive.cli.print.current.db=true, hive.cli.print.header=true</value> +</property> +<property> + <name>beeline.hs2.connection.hivevar</name> + <value>testVarName1=value1, testVarName2=value2</value> +</property> +</configuration> + http://git-wip-us.apache.org/repos/asf/hive/blob/8029e11b/beeline/src/test/resources/test-hs2-connection-zookeeper-config.xml ---------------------------------------------------------------------- diff --git a/beeline/src/test/resources/test-hs2-connection-zookeeper-config.xml b/beeline/src/test/resources/test-hs2-connection-zookeeper-config.xml new file mode 100644 index 0000000..aa95b76 --- /dev/null +++ b/beeline/src/test/resources/test-hs2-connection-zookeeper-config.xml @@ -0,0 +1,32 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration> +<property> + <name>beeline.hs2.connection.hosts</name> + <value>zk-node-1:10000,zk-node-2:10001,zk-node-3:10004</value> +</property> +<property> + <name>beeline.hs2.connection.serviceDiscoveryMode</name> + <value>zookeeper</value> +</property> +<property> + <name>beeline.hs2.connection.zooKeeperNamespace</name> + <value>hiveserver2</value> +</property> +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/8029e11b/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineConnectionUsingHiveSite.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineConnectionUsingHiveSite.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineConnectionUsingHiveSite.java new file mode 100644 index 0000000..fe77667 --- /dev/null +++ b/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineConnectionUsingHiveSite.java @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline.hs2connection; + +import java.io.File; + +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hive.beeline.hs2connection.HS2ConnectionFileParser; +import org.apache.hive.jdbc.miniHS2.MiniHS2; +import org.junit.Test; + +public class TestBeelineConnectionUsingHiveSite extends TestBeelineWithHS2ConnectionFile { + @Test + public void testBeelineConnectionHttp() throws Exception { + setupHs2(); + String path = createDefaultHs2ConnectionFile(); + testBeeLineConnection(path, new String[] { "-e", "show tables;" }, tableName); + } + + @Test + public void testBeelineConnectionSSL() throws Exception { + setupSSLHs2(); + String path = createDefaultHs2ConnectionFile(); + testBeeLineConnection(path, new String[] { "-e", "show tables;" }, tableName); + } + + @Test + public void testBeelineConnectionNoAuth() throws Exception { + setupNoAuthHs2(); + String path = createDefaultHs2ConnectionFile(); + testBeeLineConnection(path, new String[] { "-e", "show tables;" }, tableName); + } + + /* + * tests if the beeline behaves like default mode if there is no user-specific connection + * configuration file + */ + @Test + public void testBeelineWithNoConnectionFile() throws Exception { + setupNoAuthHs2(); + testBeeLineConnection(null, new String[] { "-e", "show tables;" }, "no current connection"); + } + + @Test + public void testBeelineUsingArgs() throws Exception { + setupNoAuthHs2(); + String url = miniHS2.getBaseJdbcURL() + "default"; + String args[] = new String[] { "-u", url, "-n", System.getProperty("user.name"), "-p", "foo", + "-e", "show tables;" }; + testBeeLineConnection(null, args, tableName); + } + + private void setupNoAuthHs2() throws Exception { + // use default configuration for no-auth mode + miniHS2.start(confOverlay); + createTable(); + } + + private void setupSSLHs2() throws Exception { + confOverlay.put(ConfVars.HIVE_SERVER2_USE_SSL.varname, "true"); + confOverlay.put(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH.varname, + dataFileDir + File.separator + LOCALHOST_KEY_STORE_NAME); + confOverlay.put(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname, + KEY_STORE_TRUST_STORE_PASSWORD); + miniHS2.start(confOverlay); + createTable(); + System.setProperty(JAVA_TRUST_STORE_PROP, dataFileDir + File.separator + TRUST_STORE_NAME); + System.setProperty(JAVA_TRUST_STORE_PASS_PROP, KEY_STORE_TRUST_STORE_PASSWORD); + } + + @Override + protected MiniHS2 getNewMiniHS2() throws Exception { + return new MiniHS2(hiveConf, MiniHS2.MiniClusterType.DFS_ONLY, true); + } + + private void setupHs2() throws Exception { + confOverlay.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, HS2_HTTP_MODE); + confOverlay.put(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH.varname, HS2_HTTP_ENDPOINT); + confOverlay.put(ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, "true"); + miniHS2.start(confOverlay); + createTable(); + } + + private String createDefaultHs2ConnectionFile() throws Exception { + Hs2ConnectionXmlConfigFileWriter writer = new Hs2ConnectionXmlConfigFileWriter(); + String baseJdbcURL = miniHS2.getBaseJdbcURL(); + System.out.println(baseJdbcURL); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "user", + System.getProperty("user.name")); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "password", "foo"); + writer.close(); + return writer.path(); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/8029e11b/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineWithHS2ConnectionFile.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineWithHS2ConnectionFile.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineWithHS2ConnectionFile.java new file mode 100644 index 0000000..32e9afd --- /dev/null +++ b/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineWithHS2ConnectionFile.java @@ -0,0 +1,214 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline.hs2connection; + +import static org.junit.Assert.assertFalse; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.PrintStream; +import java.io.PrintWriter; +import java.io.UnsupportedEncodingException; +import java.sql.DriverManager; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hive.beeline.BeeLine; +import org.apache.hive.beeline.hs2connection.HS2ConnectionFileParser; +import org.apache.hive.beeline.hs2connection.HiveSiteHS2ConnectionFileParser; +import org.apache.hive.beeline.hs2connection.UserHS2ConnectionFileParser; +import org.apache.hive.jdbc.miniHS2.MiniHS2; +import org.apache.hive.service.cli.CLIServiceClient; +import org.apache.hive.service.cli.HiveSQLException; +import org.apache.hive.service.cli.OperationHandle; +import org.apache.hive.service.cli.RowSet; +import org.apache.hive.service.cli.SessionHandle; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; + +public abstract class TestBeelineWithHS2ConnectionFile { + protected MiniHS2 miniHS2; + protected HiveConf hiveConf = new HiveConf(); + protected final String tableName = "testBeelineTable"; + protected String dataFileDir = hiveConf.get("test.data.files"); + protected static final String LOCALHOST_KEY_STORE_NAME = "keystore.jks"; + protected static final String TRUST_STORE_NAME = "truststore.jks"; + protected static final String KEY_STORE_TRUST_STORE_PASSWORD = "HiveJdbc"; + protected static final String HS2_HTTP_MODE = "http"; + protected static final String HS2_HTTP_ENDPOINT = "cliservice"; + private final String fileLocation = + System.getProperty("java.io.tmpdir") + "testHs2ConnectionConfig.xml"; + protected static final String JAVA_TRUST_STORE_PROP = "javax.net.ssl.trustStore"; + protected static final String JAVA_TRUST_STORE_PASS_PROP = "javax.net.ssl.trustStorePassword"; + + protected Map<String, String> confOverlay = new HashMap<>(); + + protected class TestBeeLine extends BeeLine { + UserHS2ConnectionFileParser testHs2ConfigFileManager; + ByteArrayOutputStream os; + + TestBeeLine(List<String> defaultHS2ConnectionFiles) { + testHs2ConfigFileManager = new UserHS2ConnectionFileParser(defaultHS2ConnectionFiles); + os = new ByteArrayOutputStream(); + PrintStream beelineOutputStream = new PrintStream(os); + setOutputStream(beelineOutputStream); + setErrorStream(beelineOutputStream); + } + + TestBeeLine() { + testHs2ConfigFileManager = new UserHS2ConnectionFileParser(null); + os = new ByteArrayOutputStream(); + PrintStream beelineOutputStream = new PrintStream(os); + setOutputStream(beelineOutputStream); + setErrorStream(beelineOutputStream); + } + + public String getOutput() throws UnsupportedEncodingException { + return os.toString("UTF8"); + } + + @Override + public HS2ConnectionFileParser getUserHS2ConnFileParser() { + return testHs2ConfigFileManager; + } + + @Override + public HS2ConnectionFileParser getHiveSiteHS2ConnectionFileParser() { + HiveSiteHS2ConnectionFileParser ret = new HiveSiteHS2ConnectionFileParser(); + ret.setHiveConf(miniHS2.getHiveConf()); + return ret; + } + } + + /* + * Wrapper class to write a HS2ConnectionConfig file + */ + protected class Hs2ConnectionXmlConfigFileWriter { + private final PrintWriter writer; + private final File file; + private final Configuration conf; + + protected Hs2ConnectionXmlConfigFileWriter() throws IOException { + file = new File(fileLocation); + conf = new Configuration(false); + try { + if (file.exists()) { + file.delete(); + } + file.createNewFile(); + writer = new PrintWriter(file.getAbsolutePath(), "UTF-8"); + } finally { + file.deleteOnExit(); + } + } + + protected void writeProperty(String key, String value) { + conf.set(key, value); + } + + protected String path() { + return file.getAbsolutePath(); + } + + protected void close() throws IOException { + try { + conf.writeXml(writer); + } finally { + writer.close(); + } + } + } + + @BeforeClass + public static void beforeTest() throws Exception { + Class.forName(MiniHS2.getJdbcDriverName()); + } + + @Before + public void before() throws Exception { + DriverManager.setLoginTimeout(0); + if (!System.getProperty("test.data.files", "").isEmpty()) { + dataFileDir = System.getProperty("test.data.files"); + } + dataFileDir = dataFileDir.replace('\\', '/').replace("c:", ""); + hiveConf = new HiveConf(); + miniHS2 = getNewMiniHS2(); + confOverlay = new HashMap<String, String>(); + confOverlay.put(ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); + confOverlay.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, "binary"); + } + + protected MiniHS2 getNewMiniHS2() throws Exception { + return new MiniHS2(hiveConf); + } + + @After + public void tearDown() throws Exception { + if (miniHS2 != null && miniHS2.isStarted()) { + miniHS2.stop(); + } + miniHS2 = null; + System.clearProperty(JAVA_TRUST_STORE_PROP); + System.clearProperty(JAVA_TRUST_STORE_PASS_PROP); + } + + protected void createTable() throws HiveSQLException { + CLIServiceClient serviceClient = miniHS2.getServiceClient(); + SessionHandle sessHandle = serviceClient.openSession("foo", "bar"); + serviceClient.executeStatement(sessHandle, "DROP TABLE IF EXISTS " + tableName, confOverlay); + serviceClient.executeStatement(sessHandle, "CREATE TABLE " + tableName + " (id INT)", + confOverlay); + OperationHandle opHandle = + serviceClient.executeStatement(sessHandle, "SHOW TABLES", confOverlay); + RowSet rowSet = serviceClient.fetchResults(opHandle); + assertFalse(rowSet.numRows() == 0); + } + + protected String testBeeLineConnection(String path, String[] beelineArgs, + String expectedOutput) throws IOException { + TestBeeLine beeLine = null; + try { + if(path != null) { + List<String> testLocations = new ArrayList<>(); + testLocations.add(path); + beeLine = new TestBeeLine(testLocations); + } else { + beeLine = new TestBeeLine(); + } + beeLine.begin(beelineArgs, null); + String output = beeLine.getOutput(); + System.out.println(output); + Assert.assertNotNull(output); + Assert.assertTrue("Output " + output + " does not contain " + expectedOutput, + output.toLowerCase().contains(expectedOutput.toLowerCase())); + return output; + } finally { + if (beeLine != null) { + beeLine.close(); + } + } + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/8029e11b/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineWithUserHs2ConnectionFile.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineWithUserHs2ConnectionFile.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineWithUserHs2ConnectionFile.java new file mode 100644 index 0000000..0f86186 --- /dev/null +++ b/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineWithUserHs2ConnectionFile.java @@ -0,0 +1,129 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline.hs2connection; + +import java.io.File; +import java.net.URI; + +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hive.beeline.hs2connection.HS2ConnectionFileParser; +import org.junit.Test; + +public class TestBeelineWithUserHs2ConnectionFile extends TestBeelineWithHS2ConnectionFile { + + @Test + public void testBeelineConnectionHttp() throws Exception { + setupHttpHs2(); + String path = createHttpHs2ConnectionFile(); + testBeeLineConnection(path, new String[] { "-e", "show tables;" }, tableName); + } + + private void setupHttpHs2() throws Exception { + confOverlay.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, HS2_HTTP_MODE); + confOverlay.put(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH.varname, HS2_HTTP_ENDPOINT); + confOverlay.put(ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, "true"); + miniHS2.start(confOverlay); + createTable(); + } + + private String createHttpHs2ConnectionFile() throws Exception { + Hs2ConnectionXmlConfigFileWriter writer = new Hs2ConnectionXmlConfigFileWriter(); + String baseJdbcURL = miniHS2.getBaseJdbcURL(); + + URI uri = new URI(baseJdbcURL.substring(5)); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "hosts", + uri.getHost() + ":" + uri.getPort()); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "user", + System.getProperty("user.name")); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "password", + "foo"); + writer.writeProperty( + HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "transportMode", + HS2_HTTP_MODE); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "httpPath", + HS2_HTTP_ENDPOINT); + writer.close(); + return writer.path(); + } + + @Test + public void testBeelineConnectionNoAuth() throws Exception { + setupNoAuthConfHS2(); + String path = createNoAuthHs2ConnectionFile(); + testBeeLineConnection(path, new String[] { "-e", "show tables;" }, tableName); + } + + private void setupNoAuthConfHS2() throws Exception { + // use default configuration for no-auth mode + miniHS2.start(confOverlay); + createTable(); + } + + private String createNoAuthHs2ConnectionFile() throws Exception { + Hs2ConnectionXmlConfigFileWriter writer = new Hs2ConnectionXmlConfigFileWriter(); + String baseJdbcURL = miniHS2.getBaseJdbcURL(); + URI uri = new URI(baseJdbcURL.substring(5)); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "hosts", + uri.getHost() + ":" + uri.getPort()); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "user", + System.getProperty("user.name")); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "password", + "foo"); + writer.close(); + return writer.path(); + } + + @Test + public void testBeelineConnectionSSL() throws Exception { + setupSslHs2(); + String path = createSSLHs2ConnectionFile(); + testBeeLineConnection(path, new String[] { "-e", "show tables;" }, tableName); + } + + private String createSSLHs2ConnectionFile() throws Exception { + Hs2ConnectionXmlConfigFileWriter writer = new Hs2ConnectionXmlConfigFileWriter(); + String baseJdbcURL = miniHS2.getBaseJdbcURL(); + URI uri = new URI(baseJdbcURL.substring(5)); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "hosts", + uri.getHost() + ":" + uri.getPort()); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "user", + System.getProperty("user.name")); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "password", + "foo"); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "ssl", + "true"); + writer.writeProperty( + HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "trustStorePassword", + KEY_STORE_TRUST_STORE_PASSWORD); + writer.writeProperty( + HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "sslTrustStore", + dataFileDir + File.separator + TRUST_STORE_NAME); + writer.close(); + return writer.path(); + } + + private void setupSslHs2() throws Exception { + confOverlay.put(ConfVars.HIVE_SERVER2_USE_SSL.varname, "true"); + confOverlay.put(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH.varname, + dataFileDir + File.separator + LOCALHOST_KEY_STORE_NAME); + confOverlay.put(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname, + KEY_STORE_TRUST_STORE_PASSWORD); + miniHS2.start(confOverlay); + createTable(); + } +}
