This is an automated email from the ASF dual-hosted git repository.
pravin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 9eeab401734 HIVE-27169: New Locked List to prevent configuration
change at runtime without throwing error (#4731) (Raghav Aggarwal, reviewed by
Okumin, Pravin Kumar Sinha)
9eeab401734 is described below
commit 9eeab40173479c74b6fbf6657c3472b81ce4efcd
Author: Raghav Aggarwal <[email protected]>
AuthorDate: Mon Oct 16 21:03:51 2023 +0530
HIVE-27169: New Locked List to prevent configuration change at runtime
without throwing error (#4731) (Raghav Aggarwal, reviewed by Okumin, Pravin
Kumar Sinha)
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 33 ++++++++++++++++++
.../org/apache/hadoop/hive/conf/HiveConfUtil.java | 16 +++++++++
.../org/apache/hadoop/hive/conf/TestHiveConf.java | 15 +++++++++
.../hadoop/hive/ql/processors/SetProcessor.java | 5 +++
.../hadoop/hive/ql/session/SessionState.java | 39 ++++++++++++++++++++++
5 files changed, 108 insertions(+)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 290cd8a4efa..adc6503debe 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -100,6 +100,7 @@ public class HiveConf extends Configuration {
private static final Map<String, ConfVars> metaConfs = new HashMap<String,
ConfVars>();
private final List<String> restrictList = new ArrayList<String>();
private final Set<String> hiddenSet = new HashSet<String>();
+ private final Set<String> lockedSet = new HashSet<>();
private final List<String> rscList = new ArrayList<>();
private Pattern modWhiteListPattern = null;
@@ -850,6 +851,10 @@ public class HiveConf extends Configuration {
HIVEIGNOREMAPJOINHINT("hive.ignore.mapjoin.hint", true, "Ignore the
mapjoin hint"),
+ HIVE_CONF_LOCKED_LIST("hive.conf.locked.list", "", "Comma separated " +
+ "list of configuration options which are locked and can not be
changed at runtime. Warning is logged and the " +
+ "change is ignored when user try to set these configs during
runtime"),
+
HIVE_FILE_MAX_FOOTER("hive.file.max.footer", 100,
"maximum number of lines for footer user can define for a table file"),
@@ -6010,6 +6015,9 @@ public class HiveConf extends Configuration {
throw new IllegalArgumentException("Cannot modify " + name + " at
runtime. It is in the list"
+ " of parameters that can't be modified at runtime or is prefixed
by a restricted variable");
}
+ if (isLockedConfig(name)) {
+ return;
+ }
String oldValue = name != null ? get(name) : null;
if (name == null || value == null || !value.equals(oldValue)) {
// When either name or value is null, the set method below will fail,
@@ -6022,6 +6030,10 @@ public class HiveConf extends Configuration {
return Iterables.any(hiddenSet, hiddenVar -> name.startsWith(hiddenVar));
}
+ public boolean isLockedConfig(String name) {
+ return Iterables.any(lockedSet, lockedVar -> name != null &&
name.equalsIgnoreCase(lockedVar));
+ }
+
public static boolean isEncodedPar(String name) {
for (ConfVars confVar : HiveConf.ENCODED_CONF) {
ConfVars confVar1 = confVar;
@@ -6427,6 +6439,7 @@ public class HiveConf extends Configuration {
origProp = (Properties)other.origProp.clone();
restrictList.addAll(other.restrictList);
hiddenSet.addAll(other.hiddenSet);
+ lockedSet.addAll(other.lockedSet);
modWhiteListPattern = other.modWhiteListPattern;
}
@@ -6560,6 +6573,9 @@ public class HiveConf extends Configuration {
setupRestrictList();
hiddenSet.clear();
hiddenSet.addAll(HiveConfUtil.getHiddenSet(this));
+
+ lockedSet.clear();
+ lockedSet.addAll(HiveConfUtil.getLockedSet(this));
}
/**
@@ -6938,6 +6954,22 @@ public class HiveConf extends Configuration {
setupRestrictList();
}
+ public void addToLockedSet(String lockedListStr) {
+ String oldList = this.getVar(ConfVars.HIVE_CONF_LOCKED_LIST);
+ if (oldList == null || oldList.isEmpty()) {
+ this.setVar(ConfVars.HIVE_CONF_LOCKED_LIST, lockedListStr);
+ } else {
+ this.setVar(ConfVars.HIVE_CONF_LOCKED_LIST, oldList + "," +
lockedListStr);
+ }
+ String modifiedLockedSet = this.getVar(ConfVars.HIVE_CONF_LOCKED_LIST);
+ lockedSet.clear();
+ if (modifiedLockedSet != null) {
+ for (String entry : modifiedLockedSet.split(",")) {
+ lockedSet.add(entry.trim());
+ }
+ }
+ }
+
/**
* Set white list of parameters that are allowed to be modified
*
@@ -6975,6 +7007,7 @@ public class HiveConf extends Configuration {
restrictList.add(ConfVars.HIVE_CONF_RESTRICTED_LIST.varname);
restrictList.add(ConfVars.HIVE_CONF_HIDDEN_LIST.varname);
restrictList.add(ConfVars.HIVE_CONF_INTERNAL_VARIABLE_LIST.varname);
+ restrictList.add(ConfVars.HIVE_CONF_LOCKED_LIST.varname);
}
/**
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java
index 179ee83b109..d48c884ae7c 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java
@@ -96,6 +96,22 @@ public class HiveConfUtil {
return hiddenSet;
}
+ /**
+ * Getting the set of locked configurations
+ * @param configuration The original configuration
+ * @return The list of the configuration values to be locked
+ */
+ public static Set<String> getLockedSet(Configuration configuration) {
+ Set<String> lockedSet = new HashSet<>();
+ String lockedListStr = HiveConf.getVar(configuration,
ConfVars.HIVE_CONF_LOCKED_LIST);
+ if (lockedListStr != null) {
+ for (String entry : lockedListStr.split(",")) {
+ lockedSet.add(entry.trim());
+ }
+ }
+ return lockedSet;
+ }
+
/**
* Strips hidden config entries from configuration
* @param conf The configuration to strip from
diff --git a/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
b/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
index decba6dbea0..bff79a98faa 100644
--- a/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
+++ b/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
@@ -176,6 +176,21 @@ public class TestHiveConf {
}
}
+ @Test
+ public void testLockedConfig() throws Exception {
+ HiveConf conf = new HiveConf();
+ // Set the default value of the config
+ conf.setVar(ConfVars.HIVE_EXECUTION_ENGINE, "mr");
+ String defaultVal = conf.get(ConfVars.HIVE_EXECUTION_ENGINE.varname);
+ // Update the lockedSet variable
+ conf.addToLockedSet(ConfVars.HIVE_EXECUTION_ENGINE.varname);
+ // Update the value of sample/test config
+ conf.verifyAndSet(ConfVars.HIVE_EXECUTION_ENGINE.varname, "tez");
+ String modifiedVal = conf.get(ConfVars.HIVE_EXECUTION_ENGINE.varname);
+ // Check if the value is changed.
+ Assert.assertEquals(defaultVal, modifiedVal);
+ }
+
@Test
public void testEncodingDecoding() throws UnsupportedEncodingException {
HiveConf conf = new HiveConf();
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
b/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
index 9c89c3bd3ae..fbaabe06f25 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
@@ -52,6 +52,7 @@ import com.google.common.collect.Sets;
*/
public class SetProcessor implements CommandProcessor {
private static final Logger LOG =
LoggerFactory.getLogger(SetProcessor.class);
+ private static final SessionState.LogHelper console =
SessionState.getConsole();
private static final String prefix = "set: ";
private static final Set<String> removedConfigs =
@@ -255,6 +256,10 @@ public class SetProcessor implements CommandProcessor {
}
}
conf.verifyAndSet(key, value);
+ if (conf.isLockedConfig(key)) {
+ console.printWarn("Cannot modify " + key + " at runtime. "
+ + "It is in the list of locked configurations that can't be
modified at runtime");
+ }
if (HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname.equals(key)) {
if ("mr".equals(value)) {
result = HiveConf.generateMrDeprecationWarning();
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 9614d95e395..cb7ed48a58a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -1321,6 +1321,45 @@ public class SessionState implements ISessionAuthState{
LOG.info(info + StringUtils.defaultString(detail));
}
+ /**
+ * Logs warn into the log file, and if the LogHelper is not silent then
into the HiveServer2 or
+ * HiveCli info stream too.
+ * BeeLine uses the operation log file to show the logs to the user, so
depending on the
+ * BeeLine settings it could be shown to the user.
+ * @param warn The log message
+ */
+ public void printWarn(String warn) {
+ printWarn(warn, null);
+ }
+
+ /**
+ * Logs warn into the log file, and if the LogHelper is not silent then
into the HiveServer2 or
+ * HiveCli info stream too. Handles an extra detail which will not be
printed if null.
+ * BeeLine uses the operation log file to show the logs to the user, so
depending on the
+ * BeeLine settings it could be shown to the user.
+ * @param warn The log message
+ * @param detail Extra detail to log which will be not printed if null
+ */
+ public void printWarn(String warn, String detail) {
+ printWarn(warn, detail, getIsSilent());
+ }
+
+ /**
+ * Logs warn into the log file, and if not silent then into the
HiveServer2 or HiveCli info
+ * stream too. Handles an extra detail which will not be printed if null.
+ * BeeLine uses the operation log file to show the logs to the user, so
depending on the
+ * BeeLine settings it could be shown to the user.
+ * @param warn The log message
+ * @param detail Extra detail to log which will be not printed if null
+ * @param isSilent If true then the message will not be printed to the
info stream
+ */
+ public void printWarn(String warn, String detail, boolean isSilent) {
+ if (!isSilent) {
+ getInfoStream().println(warn);
+ }
+ LOG.warn(warn + StringUtils.defaultString(detail));
+ }
+
/**
* Logs an error into the log file, and into the HiveServer2 or HiveCli
error stream too.
* BeeLine uses the operation log file to show the logs to the user, so
depending on the
