HIVE-14418 : Hive config validation prevents unsetting the settings (Sergey Shelukhin, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0705323d Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0705323d Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0705323d Branch: refs/heads/hive-14535 Commit: 0705323db28edf13777d29d3a0add48f19936db0 Parents: b80bcd0 Author: Sergey Shelukhin <[email protected]> Authored: Mon Aug 29 15:36:39 2016 -0700 Committer: Sergey Shelukhin <[email protected]> Committed: Mon Aug 29 15:36:39 2016 -0700 ---------------------------------------------------------------------- .../hive/ql/processors/ResetProcessor.java | 109 +++++++++++++++++-- .../hadoop/hive/ql/processors/SetProcessor.java | 3 +- ql/src/test/queries/clientpositive/reset_conf.q | 18 +++ .../test/queries/clientpositive/set_metaconf.q | 4 + .../results/clientpositive/reset_conf.q.out | 32 ++++++ .../results/clientpositive/set_metaconf.q.out | 2 + 6 files changed, 158 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/0705323d/ql/src/java/org/apache/hadoop/hive/ql/processors/ResetProcessor.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/ResetProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/ResetProcessor.java index e67422b..bbd4501 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/processors/ResetProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/ResetProcessor.java @@ -18,41 +18,134 @@ package org.apache.hadoop.hive.ql.processors; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveVariableSource; +import org.apache.hadoop.hive.conf.SystemVariables; +import org.apache.hadoop.hive.conf.VariableSubstitution; import org.apache.hadoop.hive.ql.CommandNeedRetryException; +import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType; import org.apache.hadoop.hive.ql.session.SessionState; +import com.google.common.collect.Lists; + public class ResetProcessor implements CommandProcessor { @Override public void init() { } + private final static String DEFAULT_ARG = "-d"; + @Override public CommandProcessorResponse run(String command) throws CommandNeedRetryException { SessionState ss = SessionState.get(); CommandProcessorResponse authErrResp = CommandUtil.authorizeCommand(ss, HiveOperationType.RESET, Arrays.asList(command)); - if(authErrResp != null){ + if (authErrResp != null) { // there was an authorization issue return authErrResp; } - - if (ss.getOverriddenConfigurations().isEmpty()) { + command = command.trim(); + if (StringUtils.isBlank(command)) { + resetOverridesOnly(ss); return new CommandProcessorResponse(0); } + String[] parts = command.split("\\s+"); + boolean isDefault = false; + List<String> varnames = new ArrayList<>(parts.length); + for (String part : parts) { + if (part.isEmpty()) continue; + if (DEFAULT_ARG.equals(part)) { + isDefault = true; + } else { + varnames.add(part); + } + } + if (varnames.isEmpty()) { + return new CommandProcessorResponse(1, "No variable names specified", "42000"); + } + String message = ""; + for (String varname : varnames) { + if (isDefault) { + if (!message.isEmpty()) { + message += ", "; + } + message += varname; + resetToDefault(ss, varname); + } else { + resetOverrideOnly(ss, varname); + } + } + return new CommandProcessorResponse(0, isDefault + ? Lists.newArrayList("Resetting " + message + " to default values") : null); + } + + private void resetOverridesOnly(SessionState ss) { + if (ss.getOverriddenConfigurations().isEmpty()) return; HiveConf conf = new HiveConf(); for (String key : ss.getOverriddenConfigurations().keySet()) { - String value = conf.get(key); - if (value != null) { - ss.getConf().set(key, value); - } + setSessionVariableFromConf(ss, key, conf); } ss.getOverriddenConfigurations().clear(); - return new CommandProcessorResponse(0); + } + + private void resetOverrideOnly(SessionState ss, String varname) { + if (!ss.getOverriddenConfigurations().containsKey(varname)) return; + setSessionVariableFromConf(ss, varname, new HiveConf()); + ss.getOverriddenConfigurations().remove(varname); + } + + private void setSessionVariableFromConf(SessionState ss, String varname, + HiveConf conf) { + String value = conf.get(varname); + if (value != null) { + ss.getConf().set(varname, value); + } + } + + private CommandProcessorResponse resetToDefault(SessionState ss, String varname) { + varname = varname.trim(); + try { + String nonErrorMessage = null; + if (varname.startsWith(SystemVariables.HIVECONF_PREFIX)){ + String propName = varname.substring(SystemVariables.HIVECONF_PREFIX.length()); + nonErrorMessage = SetProcessor.setConf( + varname, propName, getConfVar(propName).getDefaultValue(), false); + } else if (varname.startsWith(SystemVariables.METACONF_PREFIX)) { + String propName = varname.substring(SystemVariables.METACONF_PREFIX.length()); + HiveConf.ConfVars confVars = getConfVar(propName); + Hive.get(ss.getConf()).setMetaConf(propName, new VariableSubstitution(new HiveVariableSource() { + @Override + public Map<String, String> getHiveVariable() { + return SessionState.get().getHiveVariables(); + } + }).substitute(ss.getConf(), confVars.getDefaultValue())); + } else { + String defaultVal = getConfVar(varname).getDefaultValue(); + nonErrorMessage = SetProcessor.setConf(varname, varname, defaultVal, true); + if (varname.equals(HiveConf.ConfVars.HIVE_SESSION_HISTORY_ENABLED.toString())) { + SessionState.get().updateHistory(Boolean.parseBoolean(defaultVal), ss); + } + } + return nonErrorMessage == null ? new CommandProcessorResponse(0) + : new CommandProcessorResponse(0, Lists.newArrayList(nonErrorMessage)); + } catch (Exception e) { + return new CommandProcessorResponse(1, e.getMessage(), "42000", + e instanceof IllegalArgumentException ? null : e); + } + } + + private static HiveConf.ConfVars getConfVar(String propName) { + HiveConf.ConfVars confVars = HiveConf.getConfVars(propName); + if (confVars == null) throw new IllegalArgumentException(propName + " not found"); + return confVars; } } http://git-wip-us.apache.org/repos/asf/hive/blob/0705323d/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java index 2e13dab..c9d06ba 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java @@ -24,7 +24,6 @@ import static org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe.defaultN import static org.apache.hadoop.hive.conf.SystemVariables.*; -import java.util.HashSet; import java.util.Map; import java.util.Properties; import java.util.Set; @@ -179,7 +178,7 @@ public class SetProcessor implements CommandProcessor { /** * @return A console message that is not strong enough to fail the command (e.g. deprecation). */ - private static String setConf(String varname, String key, String varvalue, boolean register) + static String setConf(String varname, String key, String varvalue, boolean register) throws IllegalArgumentException { String result = null; HiveConf conf = SessionState.get().getConf(); http://git-wip-us.apache.org/repos/asf/hive/blob/0705323d/ql/src/test/queries/clientpositive/reset_conf.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/reset_conf.q b/ql/src/test/queries/clientpositive/reset_conf.q index 8ddde23..8420d02 100644 --- a/ql/src/test/queries/clientpositive/reset_conf.q +++ b/ql/src/test/queries/clientpositive/reset_conf.q @@ -9,3 +9,21 @@ reset; set hive.skewjoin.key; set hive.skewjoin.mapjoin.min.split; + +set hive.skewjoin.key=300000; +set hive.skewjoin.mapjoin.min.split=256000000; +select 'After setting hive.skewjoin.key and hive.skewjoin.mapjoin.min.split'; +set hive.skewjoin.key; + +reset -d hive.skewjoin.key; +select 'After resetting hive.skewjoin.key to default'; +set hive.skewjoin.key; +set hive.skewjoin.mapjoin.min.split; + +set hive.skewjoin.key=300000; + +reset -d hive.skewjoin.key hive.skewjoin.mapjoin.min.split; +select 'After resetting both to default'; +set hive.skewjoin.key; +set hive.skewjoin.mapjoin.min.split; + http://git-wip-us.apache.org/repos/asf/hive/blob/0705323d/ql/src/test/queries/clientpositive/set_metaconf.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/set_metaconf.q b/ql/src/test/queries/clientpositive/set_metaconf.q index a679489..6186122 100644 --- a/ql/src/test/queries/clientpositive/set_metaconf.q +++ b/ql/src/test/queries/clientpositive/set_metaconf.q @@ -4,3 +4,7 @@ set metaconf:hive.metastore.try.direct.sql; set metaconf:hive.metastore.try.direct.sql=false; set metaconf:hive.metastore.try.direct.sql; set hive.metastore.try.direct.sql; + +reset -d metaconf:hive.metastore.try.direct.sql; +set metaconf:hive.metastore.try.direct.sql; +set hive.metastore.try.direct.sql; http://git-wip-us.apache.org/repos/asf/hive/blob/0705323d/ql/src/test/results/clientpositive/reset_conf.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/reset_conf.q.out b/ql/src/test/results/clientpositive/reset_conf.q.out index e4e15be..12f2555 100644 --- a/ql/src/test/results/clientpositive/reset_conf.q.out +++ b/ql/src/test/results/clientpositive/reset_conf.q.out @@ -4,3 +4,35 @@ hive.skewjoin.key=300000 hive.skewjoin.mapjoin.min.split=256000000 hive.skewjoin.key=100000 hive.skewjoin.mapjoin.min.split=33554432 +PREHOOK: query: select 'After setting hive.skewjoin.key and hive.skewjoin.mapjoin.min.split' +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select 'After setting hive.skewjoin.key and hive.skewjoin.mapjoin.min.split' +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +After setting hive.skewjoin.key and hive.skewjoin.mapjoin.min.split +hive.skewjoin.key=300000 +PREHOOK: query: select 'After resetting hive.skewjoin.key to default' +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select 'After resetting hive.skewjoin.key to default' +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +After resetting hive.skewjoin.key to default +hive.skewjoin.key=100000 +hive.skewjoin.mapjoin.min.split=256000000 +PREHOOK: query: select 'After resetting both to default' +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select 'After resetting both to default' +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +After resetting both to default +hive.skewjoin.key=100000 +hive.skewjoin.mapjoin.min.split=33554432 http://git-wip-us.apache.org/repos/asf/hive/blob/0705323d/ql/src/test/results/clientpositive/set_metaconf.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/set_metaconf.q.out b/ql/src/test/results/clientpositive/set_metaconf.q.out index ec33e59..41b8957 100644 --- a/ql/src/test/results/clientpositive/set_metaconf.q.out +++ b/ql/src/test/results/clientpositive/set_metaconf.q.out @@ -1,3 +1,5 @@ metaconf:hive.metastore.try.direct.sql=true metaconf:hive.metastore.try.direct.sql=false hive.metastore.try.direct.sql=true +metaconf:hive.metastore.try.direct.sql=true +hive.metastore.try.direct.sql=true
