This is an automated email from the ASF dual-hosted git repository.
okumin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 330edeba887 HIVE-29848: Restrict the config parameter override in
compaction jobs (#6928)
330edeba887 is described below
commit 330edeba88739e5407d851290ffa4234454a495b
Author: Marta Kuczora <[email protected]>
AuthorDate: Thu Nov 5 06:04:32 2026 +0100
HIVE-29848: Restrict the config parameter override in compaction jobs
(#6928)
* HIVE-29848: Restrict the config parameter override in compaction jobs
* Apply batched suggestions from code review
Co-authored-by: Shohei Okumiya <[email protected]>
* Apply batched suggestions from code review
Co-authored-by: Shohei Okumiya <[email protected]>
* Did some tuning on the allow and deny list and improved the test
- Extended the allowlist with hive.iceberg.* as those configs should be
allowed to be settable
- Extended the denylist with credential, password, secret and token keywords
- Added test for iceberg configs
- Changed the test case for deny list test to contain example for each
keyword in the denylist. Also set the allowlist to .* in this test.
* Address review comment and add one more test case
---------
Co-authored-by: Shohei Okumiya <[email protected]>
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 54 ++--
.../org/apache/hadoop/hive/conf/HiveConfUtil.java | 34 +++
.../hive/ql/txn/compactor/TestCompactorBase.java | 1 +
.../txn/compactor/CompactorPropertyValidator.java | 153 ++++++++++
.../hive/ql/txn/compactor/CompactorUtil.java | 4 +-
.../hadoop/hive/ql/txn/compactor/MRCompactor.java | 12 +-
.../compactor/TestCompactorPropertyValidator.java | 339 +++++++++++++++++++++
.../TestMRCompactorJobQueueConfiguration.java | 96 ++++++
.../hive/ql/txn/compactor/TestStatsUpdater.java | 22 +-
9 files changed, 680 insertions(+), 35 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index bf6ac91e14d..c9132737542 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -5520,6 +5520,40 @@ public static enum ConfVars {
HIVE_MM_AVOID_GLOBSTATUS_ON_S3("hive.mm.avoid.s3.globstatus", true,
"Whether to use listFiles (optimized on S3) instead of globStatus when
on S3."),
+ HIVE_COMPACTOR_CONF_ALLOWLIST("hive.compactor.conf.allowlist",
+ "^mapreduce\\.(map|reduce|task\\.io)\\." +
+ "|^mapreduce\\.job\\.heap\\.memory-mb\\.ratio$" +
+ "|^yarn\\.app\\.mapreduce\\.am\\.resource\\." +
+ "|^hive\\.(compactor|mr\\.compactor)\\." +
+
"|^hive\\.(compute|exec|iceberg|merge|mm|optimize|orc|stats|tez|vectorized)\\."
+
+ "|^orc\\." +
+
"|^tez\\.(am\\.counters|am\\.resource|counters|grouping|task|runtime)\\." +
+
"|^(mapreduce\\.job\\.queuename|mapred\\.job\\.queue\\.name|tez\\.queue\\.name)$",
+ "A Java regex to define the configuration parameters which can be
overridden " +
+ "by table properties on a compaction job. A configuration parameter
can only be overridden " +
+ "if the parameter name matches this regex. To allow additional
parameters, please use the " +
+ "hive.compactor.conf.allowlist.append config, which takes exact
parameter names."),
+
+
HIVE_COMPACTOR_CONF_ALLOWLIST_APPEND("hive.compactor.conf.allowlist.append", "",
+ "Comma separated list of individual configuration parameters to add
to " +
+ "hive.compactor.conf.allowlist, matched by exact name. " +
+ "These parameters are also checked against the
hive.compactor.conf.denylist."),
+
+ HIVE_COMPACTOR_CONF_DENYLIST("hive.compactor.conf.denylist",
+
"^(fs|hive\\.security|hadoop\\.security)\\.|^tmp(jars|files|archives)$" +
+
"|(^|\\.)([^.]*opts|env|classloader|classpath|jars?|files|archives|hooks?|class(es)?|[^.]*class"
+
+
"|impl|factory|keytab|principal|credential[^.]*|[^.]*password[^.]*|[^.]*secret[^.]*|[^.]*token[^.]*"
+
+ "|log4j[^.]*|provider|encrypt|mask|plugin" +
+
"|whitelist|logger|listeners?|scratchdir|stagingdir|tmp\\.dir|temp\\.dir|input\\.dir|plan|trust"
+
+
"|params|codecs?|submitviachild|via\\.child|user\\.access|wrapper|script|ssl|tls"
+
+ "|kryo|publisher|aggregator|table\\.props|is\\.compressed)(\\.|$)",
+ "A Java regex, matched case insensitively against the parameters
that " +
+ "hive.compactor.conf.allowlist admits. Some config parameters may
hold security risk, therefore " +
+ "restricting the list of parameters is necessary. " +
+ "Denied properties are skipped with a warning and do not fail the
compaction." +
+ "If this is set to an empty value, only hive.conf.restricted.list,
hive.conf.hidden.list and " +
+ "hive.conf.locked.list are checked on top of the allow list."),
+
// If a parameter is added to the restricted list, add a test in
TestRestrictedList.Java
HIVE_CONF_RESTRICTED_LIST("hive.conf.restricted.list",
"hive.security.authenticator.manager,hive.security.authorization.manager," +
@@ -7073,26 +7107,8 @@ public void setModifiableWhiteListRegex(String
paramNameRegex) {
* including HIVE_CONF_RESTRICTED_LIST itself
*/
private void setupRestrictList() {
- String restrictListStr = this.getVar(ConfVars.HIVE_CONF_RESTRICTED_LIST);
restrictList.clear();
- if (restrictListStr != null) {
- for (String entry : restrictListStr.split(",")) {
- restrictList.add(entry.trim());
- }
- }
-
- String internalVariableListStr =
this.getVar(ConfVars.HIVE_CONF_INTERNAL_VARIABLE_LIST);
- if (internalVariableListStr != null) {
- for (String entry : internalVariableListStr.split(",")) {
- restrictList.add(entry.trim());
- }
- }
-
- restrictList.add(ConfVars.HIVE_IN_TEST.varname);
- restrictList.add(ConfVars.HIVE_CONF_RESTRICTED_LIST.varname);
- restrictList.add(ConfVars.HIVE_CONF_HIDDEN_LIST.varname);
- restrictList.add(ConfVars.HIVE_CONF_INTERNAL_VARIABLE_LIST.varname);
- restrictList.add(ConfVars.HIVE_CONF_LOCKED_LIST.varname);
+ restrictList.addAll(HiveConfUtil.getRestrictedSet(this));
}
/**
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java
index 85596156a0c..24fcf63c316 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java
@@ -97,6 +97,40 @@ public static Set<String> getHiddenSet(Configuration
configuration) {
return hiddenSet;
}
+ /**
+ * Getting the set of the configurations which are immutable at runtime.
Unlike
+ * {@link HiveConf#verifyAndSet(String, String)}, this works on any {@link
Configuration} and not
+ * only on a {@link HiveConf}, which makes it usable on a JobConf.
+ * @param configuration The original configuration
+ * @return The list of the configuration prefixes which must not be modified
at runtime
+ */
+ public static Set<String> getRestrictedSet(Configuration configuration) {
+ Set<String> restrictedSet = new HashSet<String>();
+ String restrictedListStr = HiveConf.getVar(configuration,
HiveConf.ConfVars.HIVE_CONF_RESTRICTED_LIST);
+ if (restrictedListStr != null) {
+ for (String entry : restrictedListStr.split(",")) {
+ restrictedSet.add(entry.trim());
+ }
+ }
+
+ String internalVariableListStr = HiveConf.getVar(configuration,
ConfVars.HIVE_CONF_INTERNAL_VARIABLE_LIST);
+ if (internalVariableListStr != null) {
+ for (String entry : internalVariableListStr.split(",")) {
+ restrictedSet.add(entry.trim());
+ }
+ }
+
+ restrictedSet.add(ConfVars.HIVE_IN_TEST.varname);
+ restrictedSet.add(ConfVars.HIVE_CONF_RESTRICTED_LIST.varname);
+ restrictedSet.add(ConfVars.HIVE_CONF_HIDDEN_LIST.varname);
+ restrictedSet.add(ConfVars.HIVE_CONF_INTERNAL_VARIABLE_LIST.varname);
+ restrictedSet.add(ConfVars.HIVE_CONF_LOCKED_LIST.varname);
+ restrictedSet.add(ConfVars.HIVE_COMPACTOR_CONF_ALLOWLIST.varname);
+ restrictedSet.add(ConfVars.HIVE_COMPACTOR_CONF_ALLOWLIST_APPEND.varname);
+ restrictedSet.add(ConfVars.HIVE_COMPACTOR_CONF_DENYLIST.varname);
+ return restrictedSet;
+ }
+
/**
* Getting the set of locked configurations
* @param configuration The original configuration
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactorBase.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactorBase.java
index d0607c6c5a2..09ddf70714d 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactorBase.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactorBase.java
@@ -96,6 +96,7 @@ public void setup() throws Exception {
MetastoreConf.setBoolVar(hiveConf,
MetastoreConf.ConfVars.COMPACTOR_CLEANER_ON, true);
MetastoreConf.setBoolVar(hiveConf,
MetastoreConf.ConfVars.COMPACTOR_CLEAN_ABORTS_USING_CLEANER, true);
HiveConf.setTimeVar(hiveConf, HIVE_COMPACTOR_CLEANER_RETENTION_TIME, 0,
TimeUnit.SECONDS);
+ HiveConf.setVar(hiveConf, HiveConf.ConfVars.HIVE_COMPACTOR_CONF_DENYLIST,
"(?!)");
TestTxnDbUtil.setConfValues(hiveConf);
TestTxnDbUtil.cleanDb(hiveConf);
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorPropertyValidator.java
b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorPropertyValidator.java
new file mode 100644
index 00000000000..7cbb4fcd57e
--- /dev/null
+++
b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorPropertyValidator.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hive.ql.txn.compactor;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConfUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+/**
+ * The allow list bounding which configuration parameters a table property may
override on a compaction
+ * job, read from {@link HiveConf.ConfVars#HIVE_COMPACTOR_CONF_ALLOWLIST}.
This config limits the scope
+ * of the overridable config parameters, because most of the parameters have
no effect on a compaction job.
+ * With this limitation, the regex defined by the deny list can be kept simple
and short.
+ * <p>
+ * The config parameters are first evaluated against the allow list config. If
they match, then they will be
+ * evaluated against the deny list, read from {@link
HiveConf.ConfVars#HIVE_COMPACTOR_CONF_DENYLIST}.
+ * The purpose of the deny list is to filter for parameters which could lead
to security gaps.
+ * <p>
+ * The admins can extend the allow list through {@link
HiveConf.ConfVars#HIVE_COMPACTOR_CONF_ALLOWLIST_APPEND}.
+ * The config parameters listed here are also evaluated against the deny list
to prevent accidentally allowing
+ * configs which hold security risk.
+ * <p>
+ * As last step, all config parameters are checked against the already
existing lists such as
+ * {@link HiveConf.ConfVars#HIVE_CONF_RESTRICTED_LIST}, {@link
HiveConf.ConfVars#HIVE_CONF_HIDDEN_LIST}
+ * and {@link HiveConf.ConfVars#HIVE_CONF_LOCKED_LIST}
+ * This applies for the parameters defined through {@link
HiveConf.ConfVars#HIVE_COMPACTOR_CONF_ALLOWLIST_APPEND}
+ * as well.
+ */
+final class CompactorPropertyValidator {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(CompactorPropertyValidator.class);
+ private static final Map<String, Pattern> ALLOW_PATTERN_CACHE = new
ConcurrentHashMap<>();
+ private static final Map<String, Pattern> DENY_PATTERN_CACHE = new
ConcurrentHashMap<>();
+
+ private static final Pattern MATCHES_NOTHING = Pattern.compile("(?!)");
+ private static final Pattern DENY_EVERYTHING = Pattern.compile(".*");
+
+ private CompactorPropertyValidator() {
+ }
+
+ /**
+ * Decides whether a parameter is within the set a table property may
override.
+ *
+ * @param conf configuration the allow list settings are read from
+ * @param parameterName the config parameter name with its "compactor." or
"tblprops." prefix already removed
+ */
+ static boolean isParameterAllowed(Configuration conf, String parameterName) {
+ String key = parameterName.trim();
+ if (StringUtils.isBlank(key)) {
+ return false;
+ }
+
+ Pattern allowPattern =
+ getAllowPattern(HiveConf.getVar(conf,
HiveConf.ConfVars.HIVE_COMPACTOR_CONF_ALLOWLIST), MATCHES_NOTHING,
+ HiveConf.ConfVars.HIVE_COMPACTOR_CONF_ALLOWLIST.varname);
+
+ if ((allowPattern != null && allowPattern.matcher(key).find()) ||
isOnAllowListAppend(conf, key)) {
+ Pattern denyPattern =
+ getDenyPattern(HiveConf.getVar(conf,
HiveConf.ConfVars.HIVE_COMPACTOR_CONF_DENYLIST), DENY_EVERYTHING,
+ HiveConf.ConfVars.HIVE_COMPACTOR_CONF_DENYLIST.varname);
+ if (denyPattern != null && denyPattern.matcher(key).find()) {
+ LOG.warn("Ignoring compaction property '{}', as it is not permitted to
override by table properties per {}.",
+ key, HiveConf.ConfVars.HIVE_COMPACTOR_CONF_DENYLIST.varname);
+ return false;
+ }
+
+ if (matchesAnyPrefix(HiveConfUtil.getHiddenSet(conf), key) ||
matchesAnyPrefix(HiveConfUtil.getLockedSet(conf),
+ key) || matchesAnyPrefix(HiveConfUtil.getRestrictedSet(conf), key)) {
+ LOG.warn("Ignoring compaction property '{}' as it is immutable at
runtime per {}, {} or {}.", key,
+ HiveConf.ConfVars.HIVE_CONF_RESTRICTED_LIST.varname,
HiveConf.ConfVars.HIVE_CONF_HIDDEN_LIST.varname,
+ HiveConf.ConfVars.HIVE_CONF_LOCKED_LIST.varname);
+ return false;
+ }
+ return true;
+ } else {
+ LOG.warn("Ignoring compaction property '{}', as it is not allowed by {}
or {}.", key,
+ HiveConf.ConfVars.HIVE_COMPACTOR_CONF_ALLOWLIST.varname,
+ HiveConf.ConfVars.HIVE_COMPACTOR_CONF_ALLOWLIST_APPEND.varname);
+ return false;
+ }
+ }
+
+ private static boolean isOnAllowListAppend(Configuration conf, String key) {
+ String configured = HiveConf.getVar(conf,
HiveConf.ConfVars.HIVE_COMPACTOR_CONF_ALLOWLIST_APPEND);
+ if (StringUtils.isBlank(configured)) {
+ return false;
+ }
+ for (String exception : configured.split(",")) {
+ if (key.equalsIgnoreCase(exception.trim())) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static Pattern getAllowPattern(String regexConfig, Pattern
fallbackRegex, String configName) {
+ return getPattern(regexConfig, fallbackRegex, configName,
ALLOW_PATTERN_CACHE);
+ }
+
+ private static Pattern getDenyPattern(String regexConfig, Pattern
fallbackRegex, String configName) {
+ return getPattern(regexConfig, fallbackRegex, configName,
DENY_PATTERN_CACHE);
+ }
+
+ private static Pattern getPattern(String regexConfig, Pattern fallbackRegex,
String configName,
+ Map<String, Pattern> patternCache) {
+
+ String regex = StringUtils.trimToEmpty(regexConfig);
+
+ if (regex.isEmpty()) {
+ return null;
+ }
+ return patternCache.computeIfAbsent(regex, r -> {
+ try {
+ return Pattern.compile(r, Pattern.CASE_INSENSITIVE);
+ } catch (PatternSyntaxException e) {
+ LOG.error("Invalid regex '{}' is set in '{}'. No property will be
applied until it is fixed.", r, configName,
+ e);
+ return fallbackRegex;
+ }
+ });
+ }
+
+ private static boolean matchesAnyPrefix(Set<String> prefixes, String key) {
+ return prefixes.stream().anyMatch(prefix -> !prefix.isEmpty() &&
key.startsWith(prefix));
+ }
+}
+
+
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorUtil.java
b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorUtil.java
index c858382666c..4862bfd6575 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorUtil.java
@@ -598,7 +598,9 @@ public static void overrideConfProps(
.filter(entry -> entry.getKey().startsWith(COMPACTOR_PREFIX))
.forEach(entry -> {
String property =
entry.getKey().substring(COMPACTOR_PREFIX.length());
- conf.set(property, entry.getValue());
+ if (CompactorPropertyValidator.isParameterAllowed(conf,
property)) {
+ conf.set(property, entry.getValue());
+ }
});
}
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MRCompactor.java
b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MRCompactor.java
index 6d895a033de..e722973dc97 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MRCompactor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MRCompactor.java
@@ -209,8 +209,10 @@ private void overrideTblProps(JobConf job, Map<String,
String> tblproperties, St
// mingle existing tblproperties with those specified on the ALTER TABLE
command
for (String key : stringableMap.keySet()) {
if (key.startsWith(TBLPROPS_PREFIX)) {
- String propKey = key.substring(9); // 9 is the length of "tblprops.".
We only keep the rest
- tblproperties.put(propKey, stringableMap.get(key));
+ String propKey = key.substring(TBLPROPS_PREFIX.length());
+ if (CompactorPropertyValidator.isParameterAllowed(job, propKey)) {
+ tblproperties.put(propKey, stringableMap.get(key));
+ }
}
}
// re-set TABLE_PROPS with reloaded tblproperties
@@ -226,8 +228,10 @@ private void overrideTblProps(JobConf job, Map<String,
String> tblproperties, St
private void overrideMRProps(JobConf job, Map<String, String> properties) {
for (String key : properties.keySet()) {
if (key.startsWith(COMPACTOR_PREFIX)) {
- String mrKey = key.substring(10); // 10 is the length of "compactor."
We only keep the rest.
- job.set(mrKey, properties.get(key));
+ String mrKey = key.substring(COMPACTOR_PREFIX.length());
+ if (CompactorPropertyValidator.isParameterAllowed(job, mrKey)) {
+ job.set(mrKey, properties.get(key));
+ }
}
}
}
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestCompactorPropertyValidator.java
b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestCompactorPropertyValidator.java
new file mode 100644
index 00000000000..d71a5a9b192
--- /dev/null
+++
b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestCompactorPropertyValidator.java
@@ -0,0 +1,339 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hive.ql.txn.compactor;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class TestCompactorPropertyValidator {
+
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "mapreduce.map.memory.mb",
+ "mapreduce.reduce.memory.mb",
+ "mapreduce.map.cpu.vcores",
+ "mapreduce.reduce.cpu.vcores",
+ "hive.tez.container.size",
+ "tez.task.resource.memory.mb",
+ "mapreduce.job.heap.memory-mb.ratio",
+ "mapreduce.task.io.sort.factor",
+ "mapreduce.task.io.sort.mb",
+ "yarn.app.mapreduce.am.resource.cpu-vcores",
+ "yarn.app.mapreduce.am.resource.mb",
+ "tez.am.resource.memory.mb",
+ "tez.am.resource.cpu.vcores",
+ "tez.grouping.min-size",
+ "tez.grouping.max-size",
+ "tez.grouping.split-waves"})
+ void testResourceConfigs(String key) {
+ assertTrue(allowed(new HiveConf(), key), key + " has to be settable in
compaction job config.");
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "hive.compactor.gather.stats",
+ "hive.mr.compactor.gather.stats",
+ "hive.compactor.compact.insert.only",
+ "hive.compactor.crud.query.based",
+ "hive.compactor.max.num.delta",
+ "hive.compactor.aborted.txn.time.threshold",
+ "hive.compactor.abortedtxn.threshold",
+ "hive.compactor.active.delta.dir.threshold",
+ "hive.compactor.delta.num.threshold",
+ "hive.compactor.obsolete.delta.dir.threshold",
+ "hive.compactor.delta.pct.threshold",
+ "hive.compactor.small.delta.dir.threshold"})
+ void testCompactorConfigs(String key) {
+ assertTrue(allowed(new HiveConf(), key), key + " has to be settable in
compaction job config.");
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "mapreduce.job.queuename",
+ "mapred.job.queue.name",
+ "tez.queue.name",
+ "hive.compactor.job.queue",
+ "hive.compactor.request.queue"})
+ void testQueueConfigs(String key) {
+ assertTrue(allowed(new HiveConf(), key), key + " has to be settable in
compaction job config.");
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "orc.compress",
+ "orc.stripe.size",
+ "orc.bloom.filter.columns",
+ "hive.exec.orc.default.buffer.size"})
+ void testOrcConfigs(String key) {
+ assertTrue(allowed(new HiveConf(), key), key + " has to be settable in
compaction job config.");
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "hive.iceberg.allow.datafiles.in.table.location.only",
+ "hive.iceberg.expire.snapshot.numthreads",
+ "hive.iceberg.stats.collect.partlevel",
+ "hive.iceberg.stats.max.snapshot.lookback",
+ "hive.iceberg.stats.source"})
+ void testHiveIcebergConfigs(String key) {
+ assertTrue(allowed(new HiveConf(), key), key + " has to be settable in
compaction job config.");
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "hive.iceberg.mask.default.location",
+ "iceberg.mr.split.size",
+ "iceberg.catalog.mycat.catalog-impl",
+ "iceberg.catalog.mycat.credential",
+ "iceberg.table-default.write.data.path",
+ "iceberg.vended.storage.credentials"})
+ void testIcebergConfigsRejected(String key) {
+ assertFalse(allowed(new HiveConf(), key),
+ key + " has to be rejected by hive.compactor.conf.allowlist or
hive.compactor.conf.denylist");
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "fs.defaultFS",
+ "fs.s3a.access.key",
+ "fs.AbstractFileSystem.abfs.impl",
+ "hadoop.security.authentication",
+ "hadoop.security.auth_to_local",
+ "hive.security.authenticator.manager",
+ "mapreduce.map.java.opts",
+ "mapred.child.java.opts",
+ "hive.tez.java.opts",
+ "mapreduce.map.env",
+ "mapreduce.reduce.env",
+ "mapreduce.job.classloader",
+ "mapreduce.application.classpath",
+ "yarn.application.classpath",
+ "hive.exec.pre.hooks",
+ "hive.exec.post.hooks",
+ "hive.exec.failure.hooks",
+ "hive.exec.driver.run.hooks",
+ "hive.exec.query.redactor.hooks",
+ "hive.optimize.cte.suggester.class",
+ "hive.tez.mapreduce.output.committer.class",
+ "tez.task.scale.memory.allocator.class",
+ "tez.task.scale.task.memory.allocator.class",
+ "hive.log4j.file",
+ "hive.exec.log4j.file",
+ "hive.added.archives.path",
+ "hadoop.token.files",
+ "hive.aux.jars.path",
+ "hive.added.jars.path",
+ "hive.metastore.event.message.factory",
+ "javax.jdo.PersistenceManagerFactoryClass",
+ "hadoop.rpc.socket.factory.class.default",
+ "hive.metastore.rawstore.impl",
+ "hadoop.http.authentication.kerberos.keytab",
+ "hadoop.zk.kerberos.keytab",
+ "hadoop.zk.kerberos.principal",
+ "hadoop.http.authentication.kerberos.principal",
+ "iceberg.catalog.mycat.credential",
+ "iceberg.vended.storage.credentials",
+ "hive.some.credentialfile",
+ "iceberg.catalog.mycat.token",
+ "hadoop.security.token.service.use_ip",
+ "hive.some.delegation-token.file",
+ "iceberg.catalog.mycat.s3.secret-access-key",
+ "hive.some.client.secret",
+ "javax.jdo.option.ConnectionPassword",
+ "hive.server2.keystore.password",
+ "hive.some.ssl.truststore-password",
+ "hive.server2.job.credential.provider.path",
+ "orc.key.provider",
+ "orc.encrypt",
+ "orc.mask",
+ "hive.qtest.additional.partial.mask.pattern",
+ "hive.llap.plugin.acl",
+ "mapreduce.job.reduce.shuffle.consumer.plugin.class",
+ "hive.exim.uri.scheme.whitelist",
+ "hive.vectorized.adaptor.custom.udf.whitelist",
+ "hive.exec.perf.logger",
+ "hive.metastore.end.function.listeners",
+ "hive.exec.scratchdir",
+ "hive.exec.local.scratchdir",
+ "hive.exec.stagingdir",
+ "hadoop.tmp.dir",
+ "hive.compactor.input.tmp.dir",
+ "hive.compactor.input.dir",
+ "hive.compactor.input.temp.dir",
+ "hive.exec.plan",
+ "hive.zookeeper.ssl.truststore.location",
+ "mapreduce.task.profile.params",
+ "io.compression.codecs",
+ "hive.exec.submitviachild",
+ "hive.exec.submit.local.task.via.child",
+ "hive.tez.hs2.user.access",
+ "hive.llap.io.lrfu.bp.wrapper.size",
+ "hive.exec.script.wrapper",
+ "hive.exec.script.maxerrsize",
+ "hive.script.operator.env.blacklist",
+ "hadoop.ssl.enabled.protocols",
+ "hadoop.tls.enabled",
+ "hive.stats.default.publisher",
+ "hive.stats.default.aggregator",
+ "orc.kryo.sarg",
+ "hive.compactor.table.props",
+ "hive.compactor.is.compressed",
+ "hadoop.htrace.span.receiver.classes",
+ "hive.intermediate.compression.codec",
+ "tmpjars",
+ "tmpjars",
+ "tmparchives"})
+ void testParametersRejectedByDenyList(String key) {
+ HiveConf conf = new HiveConf();
+ conf.setVar(HiveConf.ConfVars.HIVE_COMPACTOR_CONF_ALLOWLIST, ".*");
+ assertFalse(allowed(new HiveConf(), key), key + " has to be rejected by
hive.compactor.conf.denylist");
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "mapreduce.map.java.opts ",
+ " mapreduce.map.java.opts",
+ "mapreduce.map.env\t",
+ "tez.task.launch.cmd-opts\n",
+ "hive.exec.pre.hooks\r"})
+ void testParametersWithWhitespaceRejected(String key) {
+ assertFalse(allowed(new HiveConf(), key), "'" + key + "' must be
rejected.");
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "hive.security.authenticator.manager",
+ "hadoop.security.authenticator",
+ "hive.iceberg.allow.datafiles.in.table.location.only",
+ "hive.zookeeper.ssl.keystore.location",
+ "adls.sas-token.key",
+ "dfs.adls.oauth2.credential"})
+ void testParametersRejectedRestrictedAndHiddenList(String key) {
+ HiveConf conf = new HiveConf();
+ conf.setVar(HiveConf.ConfVars.HIVE_COMPACTOR_CONF_ALLOWLIST, ".*");
+ conf.setVar(HiveConf.ConfVars.HIVE_COMPACTOR_CONF_DENYLIST, "(?!)");
+ conf.setVar(HiveConf.ConfVars.HIVE_CONF_RESTRICTED_LIST,
"hive.security.authenticator.manager,hadoop.security.");
+ conf.setVar(HiveConf.ConfVars.HIVE_CONF_HIDDEN_LIST,
"adls.sas-token.,dfs.adls.oauth2.credential");
+ conf.setVar(HiveConf.ConfVars.HIVE_CONF_LOCKED_LIST,
+ "hive.iceberg.allow.datafiles.in.table.location.only,hive.zookeeper.");
+ assertFalse(allowed(conf, key), key +
+ " has to be rejected by hive.conf.restricted.list and
hive.conf.hidden.list");
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "mapreduce.map.memory.mb ",
+ " mapreduce.reduce.memory.mb",
+ "\tmapreduce.map.cpu.vcores",
+ "mapreduce.reduce.cpu.vcores\t",
+ "hive.tez.container.size\n",
+ "tez.task.resource.memory.mb\r"})
+ void testParametersWithWhitespaceAllowed(String key) {
+ assertTrue(allowed(new HiveConf(), key), key + " has to be settable in
compaction job config.");
+ }
+
+ @Test
+ void testAddConfigToAllowListAppend() {
+ HiveConf conf = new HiveConf();
+ assertFalse(allowed(conf, "hive.some.random.config"), "Not on the allow
list by default");
+ assertFalse(allowed(conf, "hive.madeup.config"), "Not on the allow list by
default");
+ assertFalse(allowed(conf, "hive.mapjoin.full.outer"), "Not on the allow
list by default");
+ assertFalse(allowed(conf, "hive.metastore.dml.events"), "Not on the allow
list by default");
+ assertFalse(allowed(conf, "yarn.app.mapreduce.am.hard-kill-timeout-ms"),
"Not on the allow list by default");
+ conf.setVar(HiveConf.ConfVars.HIVE_COMPACTOR_CONF_ALLOWLIST_APPEND,
+ "hive.some.random.config,
hive.madeup.config,hive.mapjoin.full.outer,hive.metastore.dml.events,yarn.app.mapreduce.am.hard-kill-timeout-ms");
+ assertTrue(allowed(conf, "hive.some.random.config"), "Admitted by exact
name");
+ assertTrue(allowed(conf, "hive.madeup.config"), "Whitespace around entries
is trimmed");
+ assertFalse(allowed(conf, "hive.some.random.otherconfig"), "Not admitted
by exact name");
+ assertTrue(allowed(conf, "mapreduce.map.memory.mb"), "The allow list regex
still applies");
+ assertTrue(allowed(conf, "hive.mapjoin.full.outer"), "Admitted by exact
name");
+ assertTrue(allowed(conf, "hive.metastore.dml.events"), "Admitted by exact
name");
+ assertTrue(allowed(conf, "yarn.app.mapreduce.am.hard-kill-timeout-ms"),
"Admitted by exact name");
+ }
+
+ @Test
+ void testConfigsAllowedListAppendCannotOverrideTheDenyList() {
+ HiveConf conf = new HiveConf();
+ conf.setVar(HiveConf.ConfVars.HIVE_COMPACTOR_CONF_ALLOWLIST_APPEND,
+
"mapreduce.map.java.opts,hive.exec.pre.hooks,metastore.event.listeners,"
+ + "hadoop.zk.ssl.keystore.password,hive.compactor.conf.denylist,"
+ + "hive.compactor.conf.allowlist");
+ assertFalse(allowed(conf, "mapreduce.map.java.opts"), "JVM options stay
denied");
+ assertFalse(allowed(conf, "hive.exec.pre.hooks"), "Hooks stay denied");
+ assertFalse(allowed(conf, "metastore.event.listeners"), "Metastore
listeners stay denied");
+ assertFalse(allowed(conf, "hadoop.zk.ssl.keystore.password"), "Credentials
stay denied");
+ assertFalse(allowed(conf, "hive.compactor.conf.denylist"), "Policy
parameters stay denied");
+ assertFalse(allowed(conf, "hive.compactor.conf.allowlist"), "The allow
list cannot admit itself");
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "hive.compactor.conf.allowlist",
+ "hive.compactor.conf.allowlist.append",
+ "hive.compactor.conf.denylist",
+ "hive.conf.restricted.list",
+ "hive.conf.hidden.list",
+ "hive.conf.locked.list"})
+ void testPolicyParametersAreNotSettable(String key) {
+ assertFalse(allowed(new HiveConf(), key), key + " controls the policy and
must not be settable");
+ }
+
+ /**
+ * An unusable allow list admits nothing rather than everything.
+ */
+ @Test
+ void testInvalidRegexInAllowList() {
+ HiveConf conf = new HiveConf();
+ conf.setVar(HiveConf.ConfVars.HIVE_COMPACTOR_CONF_ALLOWLIST, "([unclosed");
+ assertFalse(allowed(conf, "mapreduce.map.memory.mb"), "Even ordinary
tuning is rejected");
+ conf.setVar(HiveConf.ConfVars.HIVE_COMPACTOR_CONF_ALLOWLIST_APPEND,
"mapreduce.map.memory.mb");
+ assertTrue(allowed(conf, "mapreduce.map.memory.mb"),
+ "The append list is independent of the regex, so it still works");
+ }
+
+ /**
+ * An unusable deny list allows nothing rather than everything.
+ */
+ @Test
+ void testInvalidRegexInDenyList() {
+ HiveConf conf = new HiveConf();
+ conf.setVar(HiveConf.ConfVars.HIVE_COMPACTOR_CONF_DENYLIST, "([unclosed");
+ assertFalse(allowed(conf, "mapreduce.map.memory.mb"), "Even ordinary
tuning is rejected");
+ conf.setVar(HiveConf.ConfVars.HIVE_COMPACTOR_CONF_ALLOWLIST_APPEND,
"mapreduce.map.memory.mb");
+ assertFalse(allowed(conf, "mapreduce.map.memory.mb"),
+ "Even if the parameter is added to
hive.compactor.conf.allowlist.append, it is still rejected.");
+ }
+
+ @Test
+ void testEmptyKeyIsRejected() {
+ HiveConf conf = new HiveConf();
+ assertFalse(allowed(conf, ""), "Empty key is rejected");
+ assertFalse(allowed(conf, " "), "Empty key is rejected");
+ }
+
+ private static boolean allowed(HiveConf conf, String key) {
+ return CompactorPropertyValidator.isParameterAllowed(conf, key);
+ }
+
+}
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestMRCompactorJobQueueConfiguration.java
b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestMRCompactorJobQueueConfiguration.java
index a0dbc6d2874..c7df2e0ff1a 100644
---
a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestMRCompactorJobQueueConfiguration.java
+++
b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestMRCompactorJobQueueConfiguration.java
@@ -28,6 +28,7 @@
import org.apache.hadoop.hive.metastore.txn.entities.CompactionInfo;
import org.apache.hadoop.hive.metastore.utils.StringableMap;
import org.apache.hadoop.mapred.JobConf;
+import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
@@ -42,6 +43,8 @@
import java.util.stream.Stream;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
/**
* Tests for {@link MRCompactor#createBaseJobConf(HiveConf, String, Table,
StorageDescriptor, ValidWriteIdList, CompactionInfo)}.
@@ -62,6 +65,99 @@ void testCreateBaseJobConfHasCorrectJobQueue(ConfSetup
input) {
assertEquals(input.expectedQueue, c.getQueueName(), "Test failed for the
following input:" + input);
}
+ /**
+ * A table property must not be able to inject JVM arguments into the
compaction map containers,
+ * since the compaction job may run as a privileged user (see
metastore.compactor.run.as.user)
+ * while the property itself is settable by anyone owning the table.
+ */
+ @Test
+ void testCreateBaseJobConfIgnoresDeniedMRPropertyFromTableProperties() {
+ JobConf c =
createBaseJobConf(Collections.singletonMap("compactor.mapreduce.map.java.opts",
+ "-javaagent:/tmp/evil.jar"), Collections.emptyMap(),
Collections.emptyMap());
+ assertNull(c.get("mapreduce.map.java.opts"), "JVM options must not be
settable via tblproperties");
+ }
+
+ /**
+ * Same as above, but through the ALTER TABLE ... COMPACT ... WITH OVERWRITE
TBLPROPERTIES route,
+ * which reaches the same sink via a different argument.
+ */
+ @Test
+ void testCreateBaseJobConfIgnoresDeniedMRPropertyFromCompactionProperties() {
+ JobConf c = createBaseJobConf(Collections.emptyMap(),
Collections.singletonMap("compactor.tmpjars",
+ "hdfs:///tmp/evil.jar"), Collections.emptyMap());
+ assertNull(c.get("tmpjars"), "Classpath additions must not be settable via
ALTER TABLE COMPACT");
+ }
+
+ /**
+ * Keys on hive.conf.restricted.list are immutable at runtime and rejected by
+ * {@link HiveConf#verifyAndSet(String, String)}. The compactor writes
through the unchecked
+ * setter, so it must apply the same policy itself.
+ */
+ @Test
+ void testCreateBaseJobConfIgnoresRestrictedListProperty() {
+ JobConf c = createBaseJobConf(
+
Collections.singletonMap("compactor.hive.security.authenticator.manager",
"com.attacker.Authenticator"),
+ Collections.emptyMap(),
Collections.singletonMap(HiveConf.ConfVars.HIVE_CONF_RESTRICTED_LIST.varname,
+ "hive.security.authenticator.manager"));
+ assertNotEquals("com.attacker.Authenticator",
c.get("hive.security.authenticator.manager"),
+ "Restricted configs must not be overridable via tblproperties");
+ }
+
+ /** Values on hive.conf.hidden.list are credentials; they must not be
redirectable either. */
+ @Test
+ void testCreateBaseJobConfIgnoresHiddenListProperty() {
+ JobConf c =
createBaseJobConf(Collections.singletonMap("compactor.fs.s3a.secret.key",
"attacker-key"),
+ Collections.emptyMap(), Collections.emptyMap());
+ assertNull(c.get("fs.s3a.secret.key"), "Hidden configs must not be
settable via tblproperties");
+ }
+
+ /** Guards against the fix being over-broad: documented resource tuning must
keep working. */
+ @Test
+ void testCreateBaseJobConfStillAppliesLegitimateOverrides() {
+ Map<String, String> tableProperties = new HashMap<>();
+ tableProperties.put("compactor.mapreduce.map.memory.mb", "2048");
+ tableProperties.put("compactor.hive.compactor.gather.stats", "false");
+ JobConf c = createBaseJobConf(tableProperties, Collections.emptyMap(),
Collections.emptyMap());
+ assertEquals("2048", c.get("mapreduce.map.memory.mb"));
+ assertEquals("false", c.get("hive.compactor.gather.stats"));
+ }
+
+ /**
+ * The "tblprops." prefix of ALTER TABLE ... COMPACT does not reach the job
configuration, but it
+ * does reach the table properties handed to the ORC record writers, which
select a key provider,
+ * an encryption or masking policy, and a filter plugin from that map.
+ */
+ @Test
+ void testCreateBaseJobConfIgnoresDeniedTblProperty() {
+ JobConf c = createBaseJobConf(Collections.emptyMap(),
+ Collections.singletonMap("tblprops.orc.key.provider",
"kms://attacker"), Collections.emptyMap());
+ Map<String, String> tableProps = new
StringableMap(c.get("hive.compactor.table.props"));
+ assertNull(tableProps.get("orc.key.provider"),
+ "Encryption key provider must not be settable via ALTER TABLE
COMPACT");
+ }
+
+ /** Guards against over-blocking: ordinary ORC tuning through "tblprops."
must still work. */
+ @Test
+ void testCreateBaseJobConfStillAppliesLegitimateTblProperty() {
+ JobConf c = createBaseJobConf(Collections.emptyMap(),
Collections.singletonMap("tblprops.orc.compress", "ZLIB"),
+ Collections.emptyMap());
+ Map<String, String> tableProps = new
StringableMap(c.get("hive.compactor.table.props"));
+ assertEquals("ZLIB", tableProps.get("orc.compress"));
+ }
+
+
+ private static JobConf createBaseJobConf(Map<String, String> tableProperties,
+ Map<String, String> compactionProperties, Map<String, String>
confProperties) {
+ Table tbl = createPersonTable();
+ tbl.setParameters(new HashMap<>(tableProperties));
+ MRCompactor compactor = new MRCompactor(null);
+ CompactionInfo ci = new CompactionInfo(tbl.getDbName(),
tbl.getTableName(), null, CompactionType.MAJOR);
+ ci.properties = new StringableMap(new
HashMap<>(compactionProperties)).toString();
+ HiveConf conf = new HiveConf();
+ confProperties.forEach(conf::set);
+ return compactor.createBaseJobConf(conf, "test-job", tbl, tbl.getSd(), new
ValidReaderWriteIdList(), ci);
+ }
+
private static Stream<ConfSetup> generateBaseJobConfSetup() {
List<ConfSetup> inputs = new ArrayList<>();
String mrProperty = "mapreduce.job.queuename";
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestStatsUpdater.java
b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestStatsUpdater.java
index b84b4c00638..3416df72c7a 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestStatsUpdater.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestStatsUpdater.java
@@ -36,13 +36,13 @@ public void
testSetUpDriverSessionAddsCompactorPropertiesFromTableProperties() {
HiveConf conf = new HiveConf();
Map<String, String> tableProperties = new HashMap<String, String>() {{
- put(COMPACTOR_PREFIX + "test.property", "test-value");
+ put(COMPACTOR_PREFIX + "hive.compactor.stats.test.property",
"test-value");
}};
Map<String, String> ciProperties = new HashMap<>();
HiveConf statsUpdaterConf = statsUpdater.setUpDriverSession(conf,
"testQueue", tableProperties, ciProperties);
- assertThat(statsUpdaterConf.get("test.property"), is("test-value"));
+ assertThat(statsUpdaterConf.get("hive.compactor.stats.test.property"),
is("test-value"));
}
@Test
@@ -52,12 +52,12 @@ public void
testSetUpDriverSessionAddsCompactorPropertiesFromCompactionInfo() {
HiveConf conf = new HiveConf();
Map<String, String> tableProperties = new HashMap<>();
Map<String, String> ciProperties = new HashMap<String, String>() {{
- put(COMPACTOR_PREFIX + "test.property", "test-value");
+ put(COMPACTOR_PREFIX + "hive.compactor.stats.test.property",
"test-value");
}};
HiveConf statsUpdaterConf = statsUpdater.setUpDriverSession(conf,
"testQueue", tableProperties, ciProperties);
- assertThat(statsUpdaterConf.get("test.property"), is("test-value"));
+ assertThat(statsUpdaterConf.get("hive.compactor.stats.test.property"),
is("test-value"));
}
@Test
@@ -65,15 +65,15 @@ public void
testSetUpDriverSessionOverridesExistingCompactorPropertiesWithTableP
StatsUpdater statsUpdater = new StatsUpdater();
HiveConf conf = new HiveConf();
- conf.set(COMPACTOR_PREFIX + "test.property", "test-value");
+ conf.set(COMPACTOR_PREFIX + "hive.compactor.stats.test.property",
"test-value");
Map<String, String> tableProperties = new HashMap<String, String>() {{
- put(COMPACTOR_PREFIX + "test.property", "changed-in-table-properties");
+ put(COMPACTOR_PREFIX + "hive.compactor.stats.test.property",
"changed-in-table-properties");
}};
HiveConf statsUpdaterConf = statsUpdater.setUpDriverSession(conf,
"testQueue", tableProperties, emptyMap());
- assertThat(statsUpdaterConf.get("test.property"),
is("changed-in-table-properties"));
+ assertThat(statsUpdaterConf.get("hive.compactor.stats.test.property"),
is("changed-in-table-properties"));
}
@Test
@@ -81,18 +81,18 @@ public void
testSetUpDriverSessionOverridesExistingCompactorPropertiesWithProper
StatsUpdater statsUpdater = new StatsUpdater();
HiveConf conf = new HiveConf();
- conf.set(COMPACTOR_PREFIX + "test.property", "test-value");
+ conf.set(COMPACTOR_PREFIX + "hive.compactor.stats.test.property",
"test-value");
Map<String, String> tableProperties = new HashMap<String, String>() {{
- put(COMPACTOR_PREFIX + "test.property", "changed-in-table-properties");
+ put(COMPACTOR_PREFIX + "hive.compactor.stats.test.property",
"changed-in-table-properties");
}};
Map<String, String> ciProperties = new HashMap<String, String>() {{
- put(COMPACTOR_PREFIX + "test.property", "changed-in-compaction-info");
+ put(COMPACTOR_PREFIX + "hive.compactor.stats.test.property",
"changed-in-compaction-info");
}};
HiveConf statsUpdaterConf = statsUpdater.setUpDriverSession(conf,
"testQueue", tableProperties, ciProperties);
- assertThat(statsUpdaterConf.get("test.property"),
is("changed-in-compaction-info"));
+ assertThat(statsUpdaterConf.get("hive.compactor.stats.test.property"),
is("changed-in-compaction-info"));
}
}