Repository: hadoop
Updated Branches:
refs/heads/branch-2.8 b76f6e5de -> 8b6eeb259
HADOOP-13494. ReconfigurableBase can log sensitive information. Contributed by
Sean Mackrory.
(cherry picked from commit 4b689e7a758a55cec2ca8398727feefc8ac21bfd)
(cherry picked from commit c91e6f28283d1f8ad4504a2338a01e284ced6b24)
Conflicts:
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8b6eeb25
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8b6eeb25
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8b6eeb25
Branch: refs/heads/branch-2.8
Commit: 8b6eeb259578cd0f4e8fd7227f5c186122e3feae
Parents: b76f6e5
Author: Andrew Wang <[email protected]>
Authored: Tue Aug 16 15:01:18 2016 -0700
Committer: Andrew Wang <[email protected]>
Committed: Tue Aug 16 15:14:06 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/conf/ConfigRedactor.java | 84 ++++++++++++++++++++
.../apache/hadoop/conf/ReconfigurableBase.java | 13 ++-
.../fs/CommonConfigurationKeysPublic.java | 14 ++++
.../src/main/resources/core-default.xml | 10 +++
.../apache/hadoop/conf/TestConfigRedactor.java | 72 +++++++++++++++++
5 files changed, 190 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b6eeb25/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java
----------------------------------------------------------------------
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java
new file mode 100644
index 0000000..0ba756c
--- /dev/null
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.conf;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeys.*;
+
+/**
+ * Tool for redacting sensitive information when displaying config parameters.
+ *
+ * <p>Some config parameters contain sensitive information (for example, cloud
+ * storage keys). When these properties are displayed in plaintext, we should
+ * redactor their values as appropriate.
+ */
+public class ConfigRedactor {
+
+ private static final String REDACTED_TEXT = "<redacted>";
+
+ private List<Pattern> compiledPatterns;
+
+ public ConfigRedactor(Configuration conf) {
+ String sensitiveRegexList = conf.get(
+ HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS,
+ HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT);
+ List<String> sensitiveRegexes =
Arrays.asList(sensitiveRegexList.split(","));
+ compiledPatterns = new ArrayList<Pattern>();
+ for (String regex : sensitiveRegexes) {
+ Pattern p = Pattern.compile(regex);
+ compiledPatterns.add(p);
+ }
+ }
+
+ /**
+ * Given a key / value pair, decides whether or not to redact and returns
+ * either the original value or text indicating it has been redacted.
+ *
+ * @param key
+ * @param value
+ * @return Original value, or text indicating it has been redacted
+ */
+ public String redact(String key, String value) {
+ if (configIsSensitive(key)) {
+ return REDACTED_TEXT;
+ }
+ return value;
+ }
+
+ /**
+ * Matches given config key against patterns and determines whether or not
+ * it should be considered sensitive enough to redact in logs and other
+ * plaintext displays.
+ *
+ * @param key
+ * @return True if parameter is considered sensitive
+ */
+ private boolean configIsSensitive(String key) {
+ for (Pattern regex : compiledPatterns) {
+ if (regex.matcher(key).find()) {
+ return true;
+ }
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b6eeb25/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java
----------------------------------------------------------------------
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java
index 681ca2b..bdd006d 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java
@@ -117,17 +117,24 @@ public abstract class ReconfigurableBase
final Collection<PropertyChange> changes =
parent.getChangedProperties(newConf, oldConf);
Map<PropertyChange, Optional<String>> results = Maps.newHashMap();
+ ConfigRedactor oldRedactor = new ConfigRedactor(oldConf);
+ ConfigRedactor newRedactor = new ConfigRedactor(newConf);
for (PropertyChange change : changes) {
String errorMessage = null;
+ String oldValRedacted = oldRedactor.redact(change.prop, change.oldVal);
+ String newValRedacted = newRedactor.redact(change.prop, change.newVal);
if (!parent.isPropertyReconfigurable(change.prop)) {
LOG.info(String.format(
"Property %s is not configurable: old value: %s, new value: %s",
- change.prop, change.oldVal, change.newVal));
+ change.prop,
+ oldValRedacted,
+ newValRedacted));
continue;
}
LOG.info("Change property: " + change.prop + " from \""
- + ((change.oldVal == null) ? "<default>" : change.oldVal)
- + "\" to \"" + ((change.newVal == null) ? "<default>" :
change.newVal)
+ + ((change.oldVal == null) ? "<default>" : oldValRedacted)
+ + "\" to \""
+ + ((change.newVal == null) ? "<default>" : newValRedacted)
+ "\".");
try {
String effectiveValue =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b6eeb25/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index 772abb6..587e1e4 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -714,5 +714,19 @@ public class CommonConfigurationKeysPublic {
"hadoop.shell.safely.delete.limit.num.files";
public static final long HADOOP_SHELL_SAFELY_DELETE_LIMIT_NUM_FILES_DEFAULT =
100;
+
+ /**
+ * @see
+ * <a
href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
+ public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS =
+ "hadoop.security.sensitive-config-keys";
+ public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT =
+ "password$" + "," +
+ "fs.s3.*[Ss]ecret.?[Kk]ey" + "," +
+ "fs.azure\\.account.key.*" + "," +
+ "dfs.webhdfs.oauth2.[a-z]+.token" + "," +
+ HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b6eeb25/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git
a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 1bfb545..d093a93 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -431,6 +431,16 @@
</property>
<property>
+ <name>hadoop.security.sensitive-config-keys</name>
+
<value>password$,fs.s3.*[Ss]ecret.?[Kk]ey,fs.azure.account.key.*,dfs.webhdfs.oauth2.[a-z]+.token,hadoop.security.sensitive-config-keys</value>
+ <description>A comma-separated list of regular expressions to match against
+ configuration keys that should be redacted where appropriate, for
+ example, when logging modified properties during a reconfiguration,
+ private credentials should not be logged.
+ </description>
+</property>
+
+<property>
<name>hadoop.workaround.non.threadsafe.getpwuid</name>
<value>true</value>
<description>Some operating systems or authentication modules are known to
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b6eeb25/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
----------------------------------------------------------------------
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
new file mode 100644
index 0000000..81f8f71
--- /dev/null
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.conf;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Tests the tool (and the default expression) for deciding which config
+ * redact.
+ */
+public class TestConfigRedactor {
+ private static final String REDACTED_TEXT = "<redacted>";
+
+ private static final String ORIGINAL_VALUE = "Hello, World!";
+
+ @Test
+ public void redact() throws Exception {
+ Configuration conf = new Configuration();
+ ConfigRedactor redactor = new ConfigRedactor(conf);
+ String processedText;
+
+ List<String> sensitiveKeys = Arrays.asList(
+ "fs.s3a.secret.key",
+ "fs.s3n.awsSecretKey",
+ "fs.azure.account.key.abcdefg.blob.core.windows.net",
+ "dfs.webhdfs.oauth2.access.token",
+ "dfs.webhdfs.oauth2.refresh.token",
+ "ssl.server.keystore.keypassword",
+ "ssl.server.keystore.password",
+ "hadoop.security.sensitive-config-keys"
+ );
+ for (String key : sensitiveKeys) {
+ processedText = redactor.redact(key, ORIGINAL_VALUE);
+ Assert.assertEquals(
+ "Config parameter wasn't redacted and should be: " + key,
+ REDACTED_TEXT, processedText);
+ }
+
+ List<String> normalKeys = Arrays.asList(
+ "fs.defaultFS",
+ "dfs.replication",
+ "ssl.server.keystore.location",
+ "hadoop.security.credstore.java-keystore-provider.password-file"
+ );
+ for (String key : normalKeys) {
+ processedText = redactor.redact(key, ORIGINAL_VALUE);
+ Assert.assertEquals(
+ "Config parameter was redacted and shouldn't be: " + key,
+ ORIGINAL_VALUE, processedText);
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
