Repository: sqoop Updated Branches: refs/heads/sqoop2 1ea3db992 -> 77be2a8f9
SQOOP-2618: Sqoop2: Mask S3 sensitive properties in HDFS configuration (Abraham Fine via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/77be2a8f Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/77be2a8f Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/77be2a8f Branch: refs/heads/sqoop2 Commit: 77be2a8f9c8be384ae453b3f26296d30d4a40893 Parents: 1ea3db9 Author: Jarek Jarcec Cecho <[email protected]> Authored: Thu Oct 15 16:53:07 2015 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Thu Oct 15 16:53:07 2015 -0700 ---------------------------------------------------------------------- .../hdfs/configuration/LinkConfig.java | 3 +- .../sqoop/connector/hdfs/TestLinkConfig.java | 42 ++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/77be2a8f/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java ---------------------------------------------------------------------- diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java index 39f3752..5852e53 100644 --- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java +++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java @@ -36,7 +36,8 @@ public class LinkConfig { @Input(size = 255, validators = { @Validator(DirectoryExistsValidator.class)}) public String confDir; - @Input public Map<String, String> configOverrides; + @Input(sensitiveKeyPattern = "(?i)(.*(password|key|secret).*)") + public Map<String, String> configOverrides; public LinkConfig() { configOverrides = new HashMap<>(); http://git-wip-us.apache.org/repos/asf/sqoop/blob/77be2a8f/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestLinkConfig.java ---------------------------------------------------------------------- diff --git a/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestLinkConfig.java b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestLinkConfig.java index 05178ac..900e3db 100644 --- a/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestLinkConfig.java +++ b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestLinkConfig.java @@ -18,8 +18,15 @@ package org.apache.sqoop.connector.hdfs; import org.apache.sqoop.connector.hdfs.configuration.LinkConfig; +import org.apache.sqoop.connector.hdfs.configuration.LinkConfiguration; +import org.apache.sqoop.model.ConfigUtils; +import org.apache.sqoop.model.MConfig; +import org.apache.sqoop.model.MMapInput; import org.testng.annotations.Test; +import java.util.Map; + +import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; @@ -68,4 +75,39 @@ public class TestLinkConfig { assertFalse(validator.getStatus().canProceed(), uri); } } + + @Test + public void testSensitiveConfigOverridesKeys() { + String nonSensitiveKey = "public"; + String valueString = "value"; + String filler = "blah"; + + String[] sensitiveWords = new String[] {"password", "key", "secret"}; + + LinkConfiguration linkConfiguration = new LinkConfiguration(); + LinkConfig config = new LinkConfig(); + linkConfiguration.linkConfig = config; + config.configOverrides.put(nonSensitiveKey, valueString); + for (String sensitiveWord : sensitiveWords) { + for (String sensitiveWordWithCase : new String[] {sensitiveWord, sensitiveWord.toUpperCase()}) { + config.configOverrides.put(filler + sensitiveWordWithCase + filler, valueString); + config.configOverrides.put(sensitiveWordWithCase + filler, valueString); + config.configOverrides.put(filler + sensitiveWordWithCase, valueString); + } + } + + MConfig mLinkConfig = ConfigUtils.toConfigs(linkConfiguration).get(0); + MMapInput mConfigOverrides = mLinkConfig.getMapInput("linkConfig.configOverrides"); + + Map<String, String> redactedMap = mConfigOverrides.getNonsenstiveValue(); + assertEquals(redactedMap.get(nonSensitiveKey), valueString); + + for (String sensitiveWord : sensitiveWords) { + for (String sensitiveWordWithCase : new String[] {sensitiveWord, sensitiveWord.toUpperCase()}) { + assertEquals(redactedMap.get(filler + sensitiveWordWithCase + filler), MMapInput.SENSITIVE_VALUE_PLACEHOLDER); + assertEquals(redactedMap.get(sensitiveWordWithCase + filler), MMapInput.SENSITIVE_VALUE_PLACEHOLDER); + assertEquals(redactedMap.get(filler + sensitiveWordWithCase), MMapInput.SENSITIVE_VALUE_PLACEHOLDER); + } + } + } }
