Repository: hive Updated Branches: refs/heads/master d1c98a969 -> 6891128ca
HIVE-12300 : deprecate MR in Hive 2.0 (Sergey Shelukhin, reviewed by Sushanth Sowmyan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6891128c Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6891128c Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6891128c Branch: refs/heads/master Commit: 6891128caabc04c084582f34a948e7c4cad1ebd6 Parents: d1c98a9 Author: Sergey Shelukhin <[email protected]> Authored: Thu Nov 19 15:02:29 2015 -0800 Committer: Sergey Shelukhin <[email protected]> Committed: Thu Nov 19 15:02:29 2015 -0800 ---------------------------------------------------------------------- .../java/org/apache/hive/beeline/BeeLine.java | 13 ++++++- .../org/apache/hadoop/hive/cli/CliDriver.java | 8 ++++ .../org/apache/hadoop/hive/conf/HiveConf.java | 29 ++++++++++++++- .../java/org/apache/hadoop/hive/ql/Driver.java | 20 +++++++++- .../ql/processors/CommandProcessorResponse.java | 15 ++++++-- .../hadoop/hive/ql/processors/SetProcessor.java | 39 ++++++++++++++------ .../hadoop/hive/ql/session/OperationLog.java | 10 +++++ .../cli/operation/HiveCommandOperation.java | 11 ++++++ 8 files changed, 127 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/6891128c/beeline/src/java/org/apache/hive/beeline/BeeLine.java ---------------------------------------------------------------------- diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLine.java b/beeline/src/java/org/apache/hive/beeline/BeeLine.java index 377703f..1e289ca 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLine.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLine.java @@ -109,6 +109,7 @@ import org.apache.hive.beeline.cli.CliOptionsProcessor; * </ul> * */ +@SuppressWarnings("static-access") public class BeeLine implements Closeable { private static final ResourceBundle resourceBundle = ResourceBundle.getBundle(BeeLine.class.getSimpleName()); @@ -657,7 +658,7 @@ public class BeeLine implements Closeable { Properties confProps = commandLine.getOptionProperties("hiveconf"); for (String propKey : confProps.stringPropertyNames()) { - getOpts().getHiveConfVariables().put(propKey, confProps.getProperty(propKey)); + setHiveConfVar(propKey, confProps.getProperty(propKey)); } Properties hiveVars = commandLine.getOptionProperties("define"); @@ -739,7 +740,7 @@ public class BeeLine implements Closeable { Properties hiveConfs = cl.getOptionProperties("hiveconf"); for (String key : hiveConfs.stringPropertyNames()) { - getOpts().getHiveConfVariables().put(key, hiveConfs.getProperty(key)); + setHiveConfVar(key, hiveConfs.getProperty(key)); } driver = cl.getOptionValue("d"); @@ -794,6 +795,14 @@ public class BeeLine implements Closeable { return code; } + + private void setHiveConfVar(String key, String val) { + getOpts().getHiveConfVariables().put(key, val); + if (HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname.equals(key) && "mr".equals(val)) { + info(HiveConf.generateMrDeprecationWarning()); + } + } + private String constructCmd(String url, String user, String pass, String driver, boolean stripPasswd) { String com = "!connect " + url + " " http://git-wip-us.apache.org/repos/asf/hive/blob/6891128c/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java ---------------------------------------------------------------------- diff --git a/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java b/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java index b359850..e04f247 100644 --- a/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java +++ b/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java @@ -290,6 +290,11 @@ public class CliDriver { ss.out.println("Query returned non-zero code: " + res.getResponseCode() + ", cause: " + res.getErrorMessage()); } + if (res.getConsoleMessages() != null) { + for (String consoleMsg : res.getConsoleMessages()) { + console.printInfo(consoleMsg); + } + } ret = res.getResponseCode(); } } @@ -748,6 +753,9 @@ public class CliDriver { System.err.println("Could not open input file for reading. (" + e.getMessage() + ")"); return 3; } + if ("mr".equals(HiveConf.getVar(conf, ConfVars.HIVE_EXECUTION_ENGINE))) { + console.printInfo(HiveConf.generateMrDeprecationWarning()); + } setupConsoleReader(); http://git-wip-us.apache.org/repos/asf/hive/blob/6891128c/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 2185f85..0afb964 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2164,7 +2164,9 @@ public class HiveConf extends Configuration { "Whether to show the unquoted partition names in query results."), HIVE_EXECUTION_ENGINE("hive.execution.engine", "mr", new StringSet("mr", "tez", "spark"), - "Chooses execution engine. Options are: mr (Map reduce, default), tez (hadoop 2 only), spark"), + "Chooses execution engine. Options are: mr (Map reduce, default), tez, spark. While MR\n" + + "remains the default engine for historical reasons, it is itself a historical engine\n" + + "and is deprecated in Hive 2 line. It may be removed without further warning."), HIVE_EXECUTION_MODE("hive.execution.mode", "container", new StringSet("container", "llap"), "Chooses whether query fragments will run in container or in llap"), @@ -2571,6 +2573,13 @@ public class HiveConf extends Configuration { return defaultExpr; } + private Set<String> getValidStringValues() { + if (validator == null || !(validator instanceof StringSet)) { + throw new RuntimeException(varname + " does not specify a list of valid values"); + } + return ((StringSet)validator).getExpected(); + } + enum VarType { STRING { @Override @@ -3394,4 +3403,22 @@ public class HiveConf extends Configuration { public static void setLoadHiveServer2Config(boolean loadHiveServer2Config) { HiveConf.loadHiveServer2Config = loadHiveServer2Config; } + + public static String getNonMrEngines() { + String result = ""; + for (String s : ConfVars.HIVE_EXECUTION_ENGINE.getValidStringValues()) { + if ("mr".equals(s)) continue; + if (!result.isEmpty()) { + result += ", "; + } + result += s; + } + return result; + } + + public static String generateMrDeprecationWarning() { + return "Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. " + + "Consider using a different execution engine (i.e. " + HiveConf.getNonMrEngines() + + ") or using Hive 1.X releases."; + } } http://git-wip-us.apache.org/repos/asf/hive/blob/6891128c/ql/src/java/org/apache/hadoop/hive/ql/Driver.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index fc5a951..fbde9eb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -114,6 +114,7 @@ import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObje import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject.HivePrivObjectActionType; import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject.HivePrivilegeObjectType; import org.apache.hadoop.hive.ql.session.OperationLog; +import org.apache.hadoop.hive.ql.session.OperationLog.LoggingLevel; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; import org.apache.hadoop.hive.serde2.ByteStream; @@ -1505,10 +1506,12 @@ public class Driver implements CommandProcessor { } } - int jobs = Utilities.getMRTasks(plan.getRootTasks()).size() + int mrJobs = Utilities.getMRTasks(plan.getRootTasks()).size(); + int jobs = mrJobs + Utilities.getTezTasks(plan.getRootTasks()).size() + Utilities.getSparkTasks(plan.getRootTasks()).size(); if (jobs > 0) { + logMrWarning(mrJobs); console.printInfo("Query ID = " + plan.getQueryId()); console.printInfo("Total jobs = " + jobs); } @@ -1730,6 +1733,21 @@ public class Driver implements CommandProcessor { return (0); } + private void logMrWarning(int mrJobs) { + if (mrJobs <= 0 || !("mr".equals(HiveConf.getVar(conf, ConfVars.HIVE_EXECUTION_ENGINE)))) { + return; + } + String warning = HiveConf.generateMrDeprecationWarning(); + LOG.warn(warning); + warning = "WARNING: " + warning; + console.printInfo(warning); + // Propagate warning to beeline via operation log. + OperationLog ol = OperationLog.getCurrentOperationLog(); + if (ol != null) { + ol.writeOperationLog(LoggingLevel.EXECUTION, warning + "\n"); + } + } + private void setErrorMsgAndDetail(int exitVal, Throwable downstreamError, Task tsk) { this.downstreamError = downstreamError; errorMessage = "FAILED: Execution Error, return code " + exitVal + " from " + tsk.getClass().getName(); http://git-wip-us.apache.org/repos/asf/hive/blob/6891128c/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorResponse.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorResponse.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorResponse.java index 21b7457..d4f35f1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorResponse.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorResponse.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.ql.processors; +import java.util.List; + import org.apache.hadoop.hive.metastore.api.Schema; import org.apache.hadoop.hive.ql.ErrorMsg; @@ -36,6 +38,7 @@ public class CommandProcessorResponse { private final Schema resSchema; private final Throwable exception; + private final List<String> consoleMessages; public CommandProcessorResponse(int responseCode) { this(responseCode, null, null, null, null); @@ -45,6 +48,10 @@ public class CommandProcessorResponse { this(responseCode, errorMessage, SQLState, null, null); } + public CommandProcessorResponse(int responseCode, List<String> consoleMessages) { + this(responseCode, null, null, null, null, -1, consoleMessages); + } + public CommandProcessorResponse(int responseCode, String errorMessage, String SQLState, Throwable exception) { this(responseCode, errorMessage, SQLState, null, exception); } @@ -54,7 +61,7 @@ public class CommandProcessorResponse { } public CommandProcessorResponse(int responseCode, ErrorMsg canonicalErrMsg, Throwable t, String ... msgArgs) { this(responseCode, canonicalErrMsg.format(msgArgs), - canonicalErrMsg.getSQLState(), null, t, canonicalErrMsg.getErrorCode()); + canonicalErrMsg.getSQLState(), null, t, canonicalErrMsg.getErrorCode(), null); } /** @@ -71,16 +78,17 @@ public class CommandProcessorResponse { public CommandProcessorResponse(int responseCode, String errorMessage, String SQLState, Schema schema, Throwable exception) { - this(responseCode, errorMessage, SQLState, schema, exception, -1); + this(responseCode, errorMessage, SQLState, schema, exception, -1, null); } public CommandProcessorResponse(int responseCode, String errorMessage, String SQLState, - Schema schema, Throwable exception, int hiveErrorCode) { + Schema schema, Throwable exception, int hiveErrorCode, List<String> consoleMessages) { this.responseCode = responseCode; this.errorMessage = errorMessage; this.SQLState = SQLState; this.resSchema = schema; this.exception = exception; this.hiveErrorCode = hiveErrorCode; + this.consoleMessages = consoleMessages; } public int getResponseCode() { return responseCode; } @@ -89,6 +97,7 @@ public class CommandProcessorResponse { public Schema getSchema() { return resSchema; } public Throwable getException() { return exception; } public int getErrorCode() { return hiveErrorCode; } + public List<String> getConsoleMessages() { return consoleMessages; } public String toString() { return "(" + responseCode + "," + errorMessage + "," + (hiveErrorCode > 0 ? hiveErrorCode + "," : "" ) + http://git-wip-us.apache.org/repos/asf/hive/blob/6891128c/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java index 9389759..9a3ba04 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java @@ -37,12 +37,17 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Schema; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.session.SessionState; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; /** * SetProcessor. * */ public class SetProcessor implements CommandProcessor { + private static final Logger LOG = LoggerFactory.getLogger(SetProcessor.class); private static final String prefix = "set: "; @@ -110,22 +115,24 @@ public class SetProcessor implements CommandProcessor { public CommandProcessorResponse executeSetVariable(String varname, String varvalue) { try { - return new CommandProcessorResponse(setVariable(varname, varvalue)); + return setVariable(varname, varvalue); } catch (Exception e) { return new CommandProcessorResponse(1, e.getMessage(), "42000", e instanceof IllegalArgumentException ? null : e); } } - public static int setVariable(String varname, String varvalue) throws Exception { + public static CommandProcessorResponse setVariable( + String varname, String varvalue) throws Exception { SessionState ss = SessionState.get(); if (varvalue.contains("\n")){ ss.err.println("Warning: Value had a \\n character in it."); } varname = varname.trim(); + String nonErrorMessage = null; if (varname.startsWith(ENV_PREFIX)){ ss.err.println("env:* variables can not be set."); - return 1; + return new CommandProcessorResponse(1); // Should we propagate the error message properly? } else if (varname.startsWith(SYSTEM_PREFIX)){ String propName = varname.substring(SYSTEM_PREFIX.length()); System.getProperties() @@ -137,7 +144,7 @@ public class SetProcessor implements CommandProcessor { }).substitute(ss.getConf(), varvalue)); } else if (varname.startsWith(HIVECONF_PREFIX)){ String propName = varname.substring(HIVECONF_PREFIX.length()); - setConf(varname, propName, varvalue, false); + nonErrorMessage = setConf(varname, propName, varvalue, false); } else if (varname.startsWith(HIVEVAR_PREFIX)) { String propName = varname.substring(HIVEVAR_PREFIX.length()); ss.getHiveVariables().put(propName, new VariableSubstitution(new HiveVariableSource() { @@ -156,17 +163,21 @@ public class SetProcessor implements CommandProcessor { } }).substitute(ss.getConf(), varvalue)); } else { - setConf(varname, varname, varvalue, true); + nonErrorMessage = setConf(varname, varname, varvalue, true); if (varname.equals(HiveConf.ConfVars.HIVE_SESSION_HISTORY_ENABLED.toString())) { SessionState.get().updateHistory(Boolean.parseBoolean(varvalue), ss); } } - return 0; + return nonErrorMessage == null ? new CommandProcessorResponse(0) + : new CommandProcessorResponse(0, Lists.newArrayList(nonErrorMessage)); } - // returns non-null string for validation fail - private static void setConf(String varname, String key, String varvalue, boolean register) + /** + * @return A console message that is not strong enough to fail the command (e.g. deprecation). + */ + private static String setConf(String varname, String key, String varvalue, boolean register) throws IllegalArgumentException { + String result = null; HiveConf conf = SessionState.get().getConf(); String value = new VariableSubstitution(new HiveVariableSource() { @Override @@ -196,13 +207,19 @@ public class SetProcessor implements CommandProcessor { } } conf.verifyAndSet(key, value); - if (HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname.equals(key) - && !"spark".equals(value)) { - SessionState.get().closeSparkSession(); + if (HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname.equals(key)) { + if (!"spark".equals(value)) { + SessionState.get().closeSparkSession(); + } + if ("mr".equals(value)) { + result = HiveConf.generateMrDeprecationWarning(); + LOG.warn(result); + } } if (register) { SessionState.get().getOverriddenConfigurations().put(key, value); } + return result; } private SortedMap<String,String> propertiesToSortedMap(Properties p){ http://git-wip-us.apache.org/repos/asf/hive/blob/6891128c/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java b/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java index fb7dda5..2ecdde9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java @@ -103,6 +103,16 @@ public class OperationLog { } /** + * Write operation execution logs into log file + * @param operationLogMessage one line of log emitted from log4j + */ + public void writeOperationLog(LoggingLevel level, String operationLogMessage) { + if (opLoggingLevel.compareTo(level) < 0) return; + logFile.write(operationLogMessage); + } + + + /** * Read operation execution logs from log file * @param isFetchFirst true if the Enum FetchOrientation value is Fetch_First * @param maxRows the max number of fetched lines from log http://git-wip-us.apache.org/repos/asf/hive/blob/6891128c/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java b/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java index 22c55f1..04dc6e3 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java +++ b/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java @@ -34,7 +34,9 @@ import org.apache.commons.lang3.CharEncoding; import org.apache.hadoop.hive.metastore.api.Schema; import org.apache.hadoop.hive.ql.processors.CommandProcessor; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; +import org.apache.hadoop.hive.ql.session.OperationLog; import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.ql.session.OperationLog.LoggingLevel; import org.apache.hive.service.ServiceUtils; import org.apache.hive.service.cli.FetchOrientation; import org.apache.hive.service.cli.HiveSQLException; @@ -123,6 +125,15 @@ public class HiveCommandOperation extends ExecuteStatementOperation { setHasResultSet(false); resultSchema = new TableSchema(); } + if (response.getConsoleMessages() != null) { + // Propagate processor messages (if any) to beeline or other client. + OperationLog ol = OperationLog.getCurrentOperationLog(); + if (ol != null) { + for (String consoleMsg : response.getConsoleMessages()) { + ol.writeOperationLog(LoggingLevel.EXECUTION, consoleMsg + "\n"); + } + } + } } catch (HiveSQLException e) { setState(OperationState.ERROR); throw e;
