HDFS-11080. Update HttpFS to use ConfigRedactor. Contributed by Sean Mackrory.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7e521c5a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7e521c5a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7e521c5a Branch: refs/heads/YARN-5355 Commit: 7e521c5a49fbcf88285c102051ea2522edc847b9 Parents: dcc07ad Author: Andrew Wang <w...@apache.org> Authored: Wed Nov 2 19:11:05 2016 -0700 Committer: Andrew Wang <w...@apache.org> Committed: Wed Nov 2 19:11:05 2016 -0700 ---------------------------------------------------------------------- .../hadoop/fs/CommonConfigurationKeysPublic.java | 2 ++ .../src/main/resources/core-default.xml | 2 +- .../org/apache/hadoop/conf/TestConfigRedactor.java | 2 ++ .../java/org/apache/hadoop/lib/server/Server.java | 15 ++++++--------- 4 files changed, 11 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e521c5a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index b5b107c..f23dd51 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -770,7 +770,9 @@ public class CommonConfigurationKeysPublic { public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS = "hadoop.security.sensitive-config-keys"; public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT = + "secret$" + "," + "password$" + "," + + "ssl.keystore.pass$" + "," + "fs.s3.*[Ss]ecret.?[Kk]ey" + "," + "fs.azure\\.account.key.*" + "," + "dfs.webhdfs.oauth2.[a-z]+.token" + "," + http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e521c5a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 1e15b8e..327acfa 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -504,7 +504,7 @@ <property> <name>hadoop.security.sensitive-config-keys</name> - <value>password$,fs.s3.*[Ss]ecret.?[Kk]ey,fs.azure.account.key.*,dfs.webhdfs.oauth2.[a-z]+.token,hadoop.security.sensitive-config-keys</value> + <value>secret$,password$,ssl.keystore.pass$,fs.s3.*[Ss]ecret.?[Kk]ey,fs.azure.account.key.*,dfs.webhdfs.oauth2.[a-z]+.token,hadoop.security.sensitive-config-keys</value> <description>A comma-separated list of regular expressions to match against configuration keys that should be redacted where appropriate, for example, when logging modified properties during a reconfiguration, http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e521c5a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java index 81f8f71..eedb9b2 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java @@ -47,6 +47,7 @@ public class TestConfigRedactor { "dfs.webhdfs.oauth2.refresh.token", "ssl.server.keystore.keypassword", "ssl.server.keystore.password", + "httpfs.ssl.keystore.pass", "hadoop.security.sensitive-config-keys" ); for (String key : sensitiveKeys) { @@ -60,6 +61,7 @@ public class TestConfigRedactor { "fs.defaultFS", "dfs.replication", "ssl.server.keystore.location", + "httpfs.config.dir", "hadoop.security.credstore.java-keystore-provider.password-file" ); for (String key : normalKeys) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e521c5a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/Server.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/Server.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/Server.java index 1a0f9ff..82be027 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/Server.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/Server.java @@ -19,6 +19,7 @@ package org.apache.hadoop.lib.server; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.ConfigRedactor; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.lib.util.Check; import org.apache.hadoop.lib.util.ConfigurationUtils; @@ -482,15 +483,13 @@ public class Server { } ConfigurationUtils.injectDefaults(defaultConf, config); - + ConfigRedactor redactor = new ConfigRedactor(config); for (String name : System.getProperties().stringPropertyNames()) { String value = System.getProperty(name); if (name.startsWith(getPrefix() + ".")) { config.set(name, value); - if (name.endsWith(".password") || name.endsWith(".secret")) { - value = "*MASKED*"; - } - log.info("System property sets {}: {}", name, value); + String redacted = redactor.redact(name, value); + log.info("System property sets {}: {}", name, redacted); } } @@ -499,10 +498,8 @@ public class Server { for (Map.Entry<String, String> entry : config) { String name = entry.getKey(); String value = config.get(entry.getKey()); - if (name.endsWith(".password") || name.endsWith(".secret")) { - value = "*MASKED*"; - } - log.debug(" {}: {}", entry.getKey(), value); + String redacted = redactor.redact(name, value); + log.debug(" {}: {}", entry.getKey(), redacted); } log.debug("------------------------------------------------------"); } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
