Repository: hive Updated Branches: refs/heads/master 01ed46b4b -> 4e415609c
HIVE-21030 : Add credential store env properties redaction in JobConf (Denys Kuzmenko reviewed by Vihang Karajgaonkar) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4e415609 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4e415609 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4e415609 Branch: refs/heads/master Commit: 4e415609ce333fd17c1dd5d4bf44ca9a3897ec42 Parents: 01ed46b Author: denys kuzmenko <dkuzme...@cloudera.com> Authored: Fri Dec 14 13:29:03 2018 -0800 Committer: Vihang Karajgaonkar <vihan...@apache.org> Committed: Fri Dec 14 13:29:41 2018 -0800 ---------------------------------------------------------------------- .../apache/hadoop/hive/conf/HiveConfUtil.java | 35 ++++++++++++++----- .../ql/exec/TestHiveCredentialProviders.java | 36 ++++++++++++++++++++ 2 files changed, 62 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/4e415609/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java index 2ad5f9e..ae6fa43 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java @@ -24,12 +24,14 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.classification.InterfaceAudience.Private; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hive.common.util.HiveStringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashSet; @@ -38,6 +40,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.StringTokenizer; +import java.util.stream.Stream; /** * Hive Configuration utils @@ -182,23 +185,37 @@ public class HiveConfUtil { String jobKeyStoreLocation = jobConf.get(HiveConf.ConfVars.HIVE_SERVER2_JOB_CREDENTIAL_PROVIDER_PATH.varname); String oldKeyStoreLocation = jobConf.get(Constants.HADOOP_CREDENTIAL_PROVIDER_PATH_CONFIG); + if (StringUtils.isNotBlank(jobKeyStoreLocation)) { jobConf.set(Constants.HADOOP_CREDENTIAL_PROVIDER_PATH_CONFIG, jobKeyStoreLocation); LOG.debug("Setting job conf credstore location to " + jobKeyStoreLocation + " previous location was " + oldKeyStoreLocation); } - String credStorepassword = getJobCredentialProviderPassword(jobConf); - if (credStorepassword != null) { - // if the execution engine is MR set the map/reduce env with the credential store password + String credstorePassword = getJobCredentialProviderPassword(jobConf); + if (credstorePassword != null) { String execEngine = jobConf.get(ConfVars.HIVE_EXECUTION_ENGINE.varname); + if ("mr".equalsIgnoreCase(execEngine)) { - addKeyValuePair(jobConf, JobConf.MAPRED_MAP_TASK_ENV, - Constants.HADOOP_CREDENTIAL_PASSWORD_ENVVAR, credStorepassword); - addKeyValuePair(jobConf, JobConf.MAPRED_REDUCE_TASK_ENV, - Constants.HADOOP_CREDENTIAL_PASSWORD_ENVVAR, credStorepassword); - addKeyValuePair(jobConf, "yarn.app.mapreduce.am.admin.user.env", - Constants.HADOOP_CREDENTIAL_PASSWORD_ENVVAR, credStorepassword); + // if the execution engine is MR set the map/reduce env with the credential store password + + Collection<String> redactedProperties = + jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES); + + Stream.of( + JobConf.MAPRED_MAP_TASK_ENV, + JobConf.MAPRED_REDUCE_TASK_ENV, + "yarn.app.mapreduce.am.admin.user.env") + + .forEach(property -> { + addKeyValuePair(jobConf, property, + Constants.HADOOP_CREDENTIAL_PASSWORD_ENVVAR, credstorePassword); + redactedProperties.add(property); + }); + + // Hide sensitive configuration values from MR HistoryUI by telling MR to redact the following list. + jobConf.set(MRJobConfig.MR_JOB_REDACTED_PROPERTIES, + StringUtils.join(redactedProperties, ",")); } } } http://git-wip-us.apache.org/repos/asf/hive/blob/4e415609/ql/src/test/org/apache/hadoop/hive/ql/exec/TestHiveCredentialProviders.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestHiveCredentialProviders.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestHiveCredentialProviders.java index 62eb9e4..4f49190 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestHiveCredentialProviders.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestHiveCredentialProviders.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.ql.exec; import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -27,6 +29,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConfUtil; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapreduce.MRJobConfig; + import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -44,6 +48,10 @@ public class TestHiveCredentialProviders { private static final String HADOOP_CREDSTORE_LOCATION = "localjceks://file/user/hive/localcreds.jceks"; + private static final Collection<String> REDACTED_PROPERTIES = Arrays.asList( + JobConf.MAPRED_MAP_TASK_ENV, + JobConf.MAPRED_REDUCE_TASK_ENV); + private Configuration jobConf; /* @@ -93,6 +101,9 @@ public class TestHiveCredentialProviders { // make sure REDUCE task environment points to HIVE_JOB_CREDSTORE_PASSWORD Assert.assertEquals(HIVE_JOB_CREDSTORE_PASSWORD_ENVVAR_VAL, getValueFromJobConf( jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV), HADOOP_CREDENTIAL_PASSWORD_ENVVAR)); + + Assert.assertTrue(jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES) + .containsAll(REDACTED_PROPERTIES)); } /* @@ -114,6 +125,9 @@ public class TestHiveCredentialProviders { // make sure REDUCE task environment points to HADOOP_CREDSTORE_PASSWORD Assert.assertEquals(HADOOP_CREDSTORE_PASSWORD_ENVVAR_VAL, getValueFromJobConf( jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV), HADOOP_CREDENTIAL_PASSWORD_ENVVAR)); + + Assert.assertTrue(jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES) + .containsAll(REDACTED_PROPERTIES)); } /* @@ -131,6 +145,10 @@ public class TestHiveCredentialProviders { Assert.assertNull(getValueFromJobConf(jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV), HADOOP_CREDENTIAL_PASSWORD_ENVVAR)); + + REDACTED_PROPERTIES.forEach(property -> Assert.assertFalse( + jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES) + .contains(property))); } /* @@ -150,6 +168,9 @@ public class TestHiveCredentialProviders { Assert.assertEquals(HADOOP_CREDSTORE_PASSWORD_ENVVAR_VAL, getValueFromJobConf( jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV), HADOOP_CREDENTIAL_PASSWORD_ENVVAR)); + + Assert.assertTrue(jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES) + .containsAll(REDACTED_PROPERTIES)); } /* @@ -166,6 +187,10 @@ public class TestHiveCredentialProviders { Assert.assertNull(jobConf.get(JobConf.MAPRED_MAP_TASK_ENV)); Assert.assertNull(jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV)); + REDACTED_PROPERTIES.forEach(property -> Assert.assertFalse( + jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES) + .contains(property))); + resetConfig(); setupConfigs(true, false, false, false); @@ -174,6 +199,10 @@ public class TestHiveCredentialProviders { jobConf.get(HADOOP_CREDENTIAL_PROVIDER_PATH_CONFIG)); Assert.assertNull(jobConf.get(JobConf.MAPRED_MAP_TASK_ENV)); Assert.assertNull(jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV)); + + REDACTED_PROPERTIES.forEach(property -> Assert.assertFalse( + jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES) + .contains(property))); } /* @@ -193,6 +222,9 @@ public class TestHiveCredentialProviders { assertEquals(HADOOP_CREDSTORE_PASSWORD_ENVVAR_VAL, getValueFromJobConf( jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV), HADOOP_CREDENTIAL_PASSWORD_ENVVAR)); + + Assert.assertTrue(jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES) + .containsAll(REDACTED_PROPERTIES)); } /* @@ -210,6 +242,10 @@ public class TestHiveCredentialProviders { assertNull(getValueFromJobConf(jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV), HADOOP_CREDENTIAL_PASSWORD_ENVVAR)); + + REDACTED_PROPERTIES.forEach(property -> Assert.assertFalse( + jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES) + .contains(property))); } /*
