Repository: sqoop Updated Branches: refs/heads/sqoop2 bec7bd046 -> 2e74a9155
SQOOP-2549: Sqoop2: Allow connector developers to mask certain keys in Maps when sending them back to clients (Abraham Fine via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/2e74a915 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/2e74a915 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/2e74a915 Branch: refs/heads/sqoop2 Commit: 2e74a9155ded8d2d4bdab14adb6270edde224b8f Parents: bec7bd0 Author: Jarek Jarcec Cecho <[email protected]> Authored: Tue Sep 29 11:19:09 2015 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Tue Sep 29 11:19:09 2015 -0700 ---------------------------------------------------------------------- .../sqoop/json/util/ConfigInputConstants.java | 1 + .../json/util/ConfigInputSerialization.java | 11 ++++- .../org/apache/sqoop/model/ConfigUtils.java | 3 +- .../main/java/org/apache/sqoop/model/Input.java | 12 +++++- .../java/org/apache/sqoop/model/MMapInput.java | 30 +++++++++++++- .../json/util/TestConfigSerialization.java | 30 ++++++++++++-- .../org/apache/sqoop/model/TestConfigUtils.java | 2 +- .../org/apache/sqoop/model/TestMConfig.java | 2 +- .../org/apache/sqoop/model/TestMConfigList.java | 2 +- .../java/org/apache/sqoop/model/TestMJob.java | 4 +- .../org/apache/sqoop/model/TestMMapInput.java | 42 +++++++++++++++----- docs/src/site/sphinx/ConnectorDevelopment.rst | 5 +++ .../common/CommonRepositoryHandler.java | 4 +- .../sqoop/repository/derby/DerbyTestCase.java | 20 +++++----- .../repository/mysql/MySqlTestCase.java | 4 +- .../postgresql/PostgresqlTestCase.java | 4 +- 16 files changed, 136 insertions(+), 40 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/common/src/main/java/org/apache/sqoop/json/util/ConfigInputConstants.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/json/util/ConfigInputConstants.java b/common/src/main/java/org/apache/sqoop/json/util/ConfigInputConstants.java index 21739da..b301563 100644 --- a/common/src/main/java/org/apache/sqoop/json/util/ConfigInputConstants.java +++ b/common/src/main/java/org/apache/sqoop/json/util/ConfigInputConstants.java @@ -35,6 +35,7 @@ public class ConfigInputConstants { public static final String CONFIG_INPUT_NAME = "name"; public static final String CONFIG_INPUT_TYPE = "type"; public static final String CONFIG_INPUT_SENSITIVE = "sensitive"; + public static final String CONFIG_INPUT_SENSITIVE_KEY_PATTERN = "sensitive-pattern"; public static final String CONFIG_INPUT_SIZE = "size"; public static final String CONFIG_INPUT_EDITABLE = "editable"; public static final String CONFIG_INPUT_OVERRIDES = "overrides"; http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/common/src/main/java/org/apache/sqoop/json/util/ConfigInputSerialization.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/json/util/ConfigInputSerialization.java b/common/src/main/java/org/apache/sqoop/json/util/ConfigInputSerialization.java index eb42da3..03e869c 100644 --- a/common/src/main/java/org/apache/sqoop/json/util/ConfigInputSerialization.java +++ b/common/src/main/java/org/apache/sqoop/json/util/ConfigInputSerialization.java @@ -106,7 +106,13 @@ public final class ConfigInputSerialization { // Skip if sensitive if (!mInput.isEmpty() && !(skipSensitive && mInput.isSensitive())) { if (mInput.getType() == MInputType.MAP) { - input.put(ConfigInputConstants.CONFIG_INPUT_VALUE, mInput.getValue()); + MMapInput mMapInput = (MMapInput)mInput; + input.put(ConfigInputConstants.CONFIG_INPUT_SENSITIVE_KEY_PATTERN, mMapInput.getSensitiveKeyPattern()); + if (skipSensitive) { + input.put(ConfigInputConstants.CONFIG_INPUT_VALUE, mMapInput.getNonsenstiveValue()); + } else { + input.put(ConfigInputConstants.CONFIG_INPUT_VALUE, mMapInput.getValue()); + } } else { input.put(ConfigInputConstants.CONFIG_INPUT_VALUE, mInput.getUrlSafeValueString()); } @@ -155,6 +161,7 @@ public final class ConfigInputSerialization { InputEditable.valueOf((String)input.get(ConfigInputConstants.CONFIG_INPUT_EDITABLE)) : InputEditable.USER_ONLY; String overrides = (String) input.get(ConfigInputConstants.CONFIG_INPUT_OVERRIDES); + String sensitveKeyPattern = (String) input.get(ConfigInputConstants.CONFIG_INPUT_SENSITIVE_KEY_PATTERN); MInput mInput = null; switch (type) { @@ -164,7 +171,7 @@ public final class ConfigInputSerialization { break; } case MAP: { - mInput = new MMapInput(name, sensitive.booleanValue(), editable, overrides); + mInput = new MMapInput(name, sensitive.booleanValue(), editable, overrides, sensitveKeyPattern); break; } case INTEGER: { http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/common/src/main/java/org/apache/sqoop/model/ConfigUtils.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/model/ConfigUtils.java b/common/src/main/java/org/apache/sqoop/model/ConfigUtils.java index 282a024..e5d4f80 100644 --- a/common/src/main/java/org/apache/sqoop/model/ConfigUtils.java +++ b/common/src/main/java/org/apache/sqoop/model/ConfigUtils.java @@ -144,6 +144,7 @@ public class ConfigUtils { short maxLen = inputAnnotation.size(); InputEditable editable = inputAnnotation.editable(); String overrides = inputAnnotation.overrides(); + String sensitiveKeyPattern = inputAnnotation.sensitiveKeyPattern(); Class<?> type = field.getType(); MInput input; @@ -158,7 +159,7 @@ public class ConfigUtils { if (type == String.class) { input = new MStringInput(inputName, sensitive, editable, overrides, maxLen); } else if (type.isAssignableFrom(Map.class)) { - input = new MMapInput(inputName, sensitive, editable, overrides); + input = new MMapInput(inputName, sensitive, editable, overrides, sensitiveKeyPattern); } else if (type == Integer.class) { input = new MIntegerInput(inputName, sensitive, editable, overrides); } else if (type == Long.class) { http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/common/src/main/java/org/apache/sqoop/model/Input.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/model/Input.java b/common/src/main/java/org/apache/sqoop/model/Input.java index 883abe6..fa7f6ac 100644 --- a/common/src/main/java/org/apache/sqoop/model/Input.java +++ b/common/src/main/java/org/apache/sqoop/model/Input.java @@ -17,6 +17,7 @@ */ package org.apache.sqoop.model; +import org.apache.commons.lang.StringUtils; import org.apache.sqoop.classification.InterfaceAudience; import org.apache.sqoop.classification.InterfaceStability; @@ -43,6 +44,15 @@ public @interface Input { boolean sensitive() default false; /** + * If this input is a map, keys matching this regular expression will be + * treated as sensitive. Sqoop will ensure that values associated with the + * sensitive keys will not be easily accessible. + * + * @return The regular expression that matches sensitive fields + */ + String sensitiveKeyPattern() default StringUtils.EMPTY; + + /** * Indicates the entity that can edit the input's values, all inputs are * created/deleted only by the connector code, other entities do not have * access to either create/delete an input @@ -63,7 +73,7 @@ public @interface Input { * separated list of other inputs in the config class. It validates the * attribute value obeys the expected conditions */ - String overrides() default ""; + String overrides() default StringUtils.EMPTY; /** * List of validators associated with this input. http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/common/src/main/java/org/apache/sqoop/model/MMapInput.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/model/MMapInput.java b/common/src/main/java/org/apache/sqoop/model/MMapInput.java index f2c1ed7..4e8df34 100644 --- a/common/src/main/java/org/apache/sqoop/model/MMapInput.java +++ b/common/src/main/java/org/apache/sqoop/model/MMapInput.java @@ -20,7 +20,9 @@ package org.apache.sqoop.model; import java.util.HashMap; import java.util.Map; import java.util.Set; +import java.util.regex.Pattern; +import org.apache.commons.lang.StringUtils; import org.apache.sqoop.classification.InterfaceAudience; import org.apache.sqoop.classification.InterfaceStability; import org.apache.sqoop.utils.UrlSafeUtils; @@ -29,8 +31,13 @@ import org.apache.sqoop.utils.UrlSafeUtils; @InterfaceStability.Unstable public final class MMapInput extends MInput<Map<String, String>> { - public MMapInput(String name, boolean sensitive, InputEditable editable, String overrides) { + public static final String SENSITIVE_VALUE_PLACEHOLDER = StringUtils.EMPTY; + + private final String sensitiveKeyPattern; + + public MMapInput(String name, boolean sensitive, InputEditable editable, String overrides, String sensitiveKeyPattern) { super(name, sensitive, editable, overrides); + this.sensitiveKeyPattern = sensitiveKeyPattern; } @Override @@ -117,7 +124,7 @@ public final class MMapInput extends MInput<Map<String, String>> { @Override public MMapInput clone(boolean cloneWithValue) { - MMapInput copy = new MMapInput(getName(), isSensitive(), getEditable(), getOverrides()); + MMapInput copy = new MMapInput(getName(), isSensitive(), getEditable(), getOverrides(), getSensitiveKeyPattern()); copy.setPersistenceId(getPersistenceId()); if(cloneWithValue && this.getValue() != null) { Map<String, String> copyMap = new HashMap<String, String>(); @@ -129,4 +136,23 @@ public final class MMapInput extends MInput<Map<String, String>> { } return copy; } + + public String getSensitiveKeyPattern() { + return sensitiveKeyPattern; + } + + public Map<String, String> getNonsenstiveValue() { + if (isEmpty()) return null; + + Map<String, String> nonsensitveValue = new HashMap<>(); + Pattern sensitivePattern = Pattern.compile(getSensitiveKeyPattern()); + for (Map.Entry<String, String> entry : getValue().entrySet()) { + if (sensitivePattern.matcher(entry.getKey()).matches()){ + nonsensitveValue.put(entry.getKey(), SENSITIVE_VALUE_PLACEHOLDER); + } else { + nonsensitveValue.put(entry.getKey(), entry.getValue()); + } + } + return nonsensitveValue; + } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/common/src/test/java/org/apache/sqoop/json/util/TestConfigSerialization.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/sqoop/json/util/TestConfigSerialization.java b/common/src/test/java/org/apache/sqoop/json/util/TestConfigSerialization.java index ab4f258..b188c4c 100644 --- a/common/src/test/java/org/apache/sqoop/json/util/TestConfigSerialization.java +++ b/common/src/test/java/org/apache/sqoop/json/util/TestConfigSerialization.java @@ -117,13 +117,37 @@ public class TestConfigSerialization { String serializedJson = jsonObject.toJSONString(); // Replace map value with a fake string to force exception - String badSerializedJson = serializedJson.replace("{\"A\":\"B\"}", "\"nonsensical string\""); + String badSerializedJson = serializedJson.replace("{\"A\":\"B\"}", + "\"nonsensical string\""); System.out.println(badSerializedJson); JSONObject retrievedJson = JSONUtils.parse(badSerializedJson); ConfigInputSerialization.restoreConfig(retrievedJson); } @Test + public void testMapDataTypeSensitiveKeys() { + MConfig config = getMapConfig(); + + // Inserted values + Map<String, String> map = new HashMap<String, String>(); + map.put("A", "B"); + config.getMapInput("Map").setValue(map); + + // Serialize + JSONObject jsonObject = ConfigInputSerialization.extractConfig(config, MConfigType.JOB, true); + String serializedJson = jsonObject.toJSONString(); + + // Map with sensitive values redacted + Map<String, String> sensitiveMap = new HashMap<String, String>(); + sensitiveMap.put("A", MMapInput.SENSITIVE_VALUE_PLACEHOLDER); + + // Deserialize + JSONObject retrievedJson = JSONUtils.parse(serializedJson); + MConfig retrieved = ConfigInputSerialization.restoreConfig(retrievedJson); + assertEquals(sensitiveMap, retrieved.getMapInput("Map").getValue()); + } + + @Test public void testInputEditableOptional() { // Inserted values Map<String, String> map = new HashMap<String, String>(); @@ -188,7 +212,7 @@ public class TestConfigSerialization { inputs = new LinkedList<MInput<?>>(); - input = new MMapInput("Map", false, InputEditable.ANY, StringUtils.EMPTY); + input = new MMapInput("Map", false, InputEditable.ANY, StringUtils.EMPTY, "A"); inputs.add(input); return new MConfig("c", inputs); @@ -208,7 +232,7 @@ public class TestConfigSerialization { input = new MStringInput("String", false, InputEditable.ANY, StringUtils.EMPTY, (short)30); inputs.add(input); - input = new MMapInput("Map", false, InputEditable.ANY, StringUtils.EMPTY); + input = new MMapInput("Map", false, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY); inputs.add(input); input = new MIntegerInput("Integer", false, InputEditable.ANY, StringUtils.EMPTY); http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/common/src/test/java/org/apache/sqoop/model/TestConfigUtils.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/sqoop/model/TestConfigUtils.java b/common/src/test/java/org/apache/sqoop/model/TestConfigUtils.java index 7052841..9ee0eec 100644 --- a/common/src/test/java/org/apache/sqoop/model/TestConfigUtils.java +++ b/common/src/test/java/org/apache/sqoop/model/TestConfigUtils.java @@ -245,7 +245,7 @@ public class TestConfigUtils { // Config C inputs = new LinkedList<MInput<?>>(); inputs.add(new MLongInput("cConfig.longValue", false, InputEditable.ANY, StringUtils.EMPTY)); - inputs.add(new MMapInput("cConfig.map", false, InputEditable.ANY, StringUtils.EMPTY)); + inputs.add(new MMapInput("cConfig.map", false, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY)); inputs.add(new MEnumInput("cConfig.enumeration", false, InputEditable.ANY, StringUtils.EMPTY, new String[] { "X", "Y" })); inputs.add(new MListInput("cConfig.list", false, InputEditable.ANY, StringUtils.EMPTY)); http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/common/src/test/java/org/apache/sqoop/model/TestMConfig.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/sqoop/model/TestMConfig.java b/common/src/test/java/org/apache/sqoop/model/TestMConfig.java index f18b069..00b4a46 100644 --- a/common/src/test/java/org/apache/sqoop/model/TestMConfig.java +++ b/common/src/test/java/org/apache/sqoop/model/TestMConfig.java @@ -69,7 +69,7 @@ public class TestMConfig { public void testGetInputs() { MIntegerInput intInput = new MIntegerInput("Config.A", false, InputEditable.ANY, StringUtils.EMPTY ); MLongInput longInput = new MLongInput("Config.A1", false, InputEditable.ANY, StringUtils.EMPTY ); - MMapInput mapInput = new MMapInput("Config.B", false, InputEditable.ANY, StringUtils.EMPTY ); + MMapInput mapInput = new MMapInput("Config.B", false, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY ); MStringInput stringInput = new MStringInput("Config.C", false, InputEditable.ANY, StringUtils.EMPTY, (short) 3); MEnumInput enumInput = new MEnumInput("Config.D", false, InputEditable.ANY, StringUtils.EMPTY, http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/common/src/test/java/org/apache/sqoop/model/TestMConfigList.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/sqoop/model/TestMConfigList.java b/common/src/test/java/org/apache/sqoop/model/TestMConfigList.java index d0847fe..0d2ff61 100644 --- a/common/src/test/java/org/apache/sqoop/model/TestMConfigList.java +++ b/common/src/test/java/org/apache/sqoop/model/TestMConfigList.java @@ -32,7 +32,7 @@ public class TestMConfigList { List<MConfig> configs = new LinkedList<MConfig>(); MIntegerInput intInput = new MIntegerInput("Config1.A", false, InputEditable.ANY, StringUtils.EMPTY); - MMapInput mapInput = new MMapInput("Config1.B", false, InputEditable.ANY, StringUtils.EMPTY); + MMapInput mapInput = new MMapInput("Config1.B", false, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY); List<MInput<?>> inputs = new ArrayList<MInput<?>>(); inputs.add(intInput); http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/common/src/test/java/org/apache/sqoop/model/TestMJob.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/sqoop/model/TestMJob.java b/common/src/test/java/org/apache/sqoop/model/TestMJob.java index e0ffdc2..6867ead 100644 --- a/common/src/test/java/org/apache/sqoop/model/TestMJob.java +++ b/common/src/test/java/org/apache/sqoop/model/TestMJob.java @@ -121,7 +121,7 @@ public class TestMJob { private MToConfig toConfig() { List<MConfig> configs = new ArrayList<MConfig>(); - MMapInput input = new MMapInput("MAP-INPUT", false, InputEditable.ANY, StringUtils.EMPTY); + MMapInput input = new MMapInput("MAP-INPUT", false, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY); List<MInput<?>> list = new ArrayList<MInput<?>>(); list.add(input); MConfig config = new MConfig("CONFIGTONAME", list); @@ -131,7 +131,7 @@ public class TestMJob { private MDriverConfig driverConfig() { List<MConfig> configs = new ArrayList<MConfig>(); - MMapInput input = new MMapInput("MAP-INPUT", false, InputEditable.ANY, StringUtils.EMPTY); + MMapInput input = new MMapInput("MAP-INPUT", false, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY); List<MInput<?>> list = new ArrayList<MInput<?>>(); list.add(input); MConfig config = new MConfig("CONFIGDRIVERNAME", list); http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/common/src/test/java/org/apache/sqoop/model/TestMMapInput.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/sqoop/model/TestMMapInput.java b/common/src/test/java/org/apache/sqoop/model/TestMMapInput.java index 118277e..6c2b7e6 100644 --- a/common/src/test/java/org/apache/sqoop/model/TestMMapInput.java +++ b/common/src/test/java/org/apache/sqoop/model/TestMMapInput.java @@ -38,7 +38,7 @@ public class TestMMapInput { */ @Test public void testInitialization() { - MMapInput input = new MMapInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY); + MMapInput input = new MMapInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY); assertEquals("sqoopsqoop", input.getName()); assertEquals(MInputType.MAP, input.getType()); } @@ -49,13 +49,13 @@ public class TestMMapInput { @Test public void testEquals() { // Positive test - MMapInput input1 = new MMapInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY); - MMapInput input2 = new MMapInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY); + MMapInput input1 = new MMapInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY); + MMapInput input2 = new MMapInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY); assertTrue(input1.equals(input2)); // Negative test - MMapInput input3 = new MMapInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY); - MMapInput input4 = new MMapInput("sqoopsqoop1", false, InputEditable.ANY, StringUtils.EMPTY); + MMapInput input3 = new MMapInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY); + MMapInput input4 = new MMapInput("sqoopsqoop1", false, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY); assertFalse(input3.equals(input4)); } @@ -64,7 +64,7 @@ public class TestMMapInput { */ @Test public void testValue() { - MMapInput input1 = new MMapInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY); + MMapInput input1 = new MMapInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY); Map<String, String> map1 = new HashMap<String, String>(); input1.setValue(map1); assertEquals(map1, input1.getValue()); @@ -77,7 +77,7 @@ public class TestMMapInput { */ @Test public void testUrlSafe() { - MMapInput input1 = new MMapInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY); + MMapInput input1 = new MMapInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY); Map<String, String> map1 = new HashMap<String, String>(); input1.setValue(map1); // Getting URL safe string @@ -98,7 +98,7 @@ public class TestMMapInput { */ @Test public void testNamedElement() { - MMapInput input1 = new MMapInput("sqoopsqoop", true, InputEditable.ANY, StringUtils.EMPTY); + MMapInput input1 = new MMapInput("sqoopsqoop", true, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY); assertEquals("sqoopsqoop.label", input1.getLabelKey()); assertEquals("sqoopsqoop.help", input1.getHelpKey()); } @@ -108,9 +108,31 @@ public class TestMMapInput { */ @Test public void testSensitivity() { - MMapInput input1 = new MMapInput("NAME", false, InputEditable.ANY, StringUtils.EMPTY ); - MMapInput input2 = new MMapInput("NAME", true, InputEditable.ANY, StringUtils.EMPTY ); + MMapInput input1 = new MMapInput("NAME", false, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY ); + MMapInput input2 = new MMapInput("NAME", true, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY ); assertFalse(input1.isSensitive()); assertTrue(input2.isSensitive()); } + + /** + * Test for sensitivity + */ + @Test + public void testSensitiveKeyPattern() { + Map<String, String> testValue = new HashMap<>(); + testValue.put("sqoop features", "awesome features"); + testValue.put("sqoop bugs", "horrible bugs"); + + MMapInput input1 = new MMapInput("NAME", false, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY ); + input1.setValue(testValue); + MMapInput input2 = new MMapInput("NAME", true, InputEditable.ANY, StringUtils.EMPTY, ".*bugs.*"); + input2.setValue(testValue); + + assertEquals(input1.getNonsenstiveValue(), testValue); + + Map<String, String> expectedNonsensitiveMap = new HashMap<>(); + expectedNonsensitiveMap.put("sqoop features", "awesome features"); + expectedNonsensitiveMap.put("sqoop bugs", MMapInput.SENSITIVE_VALUE_PLACEHOLDER); + assertEquals(input2.getNonsenstiveValue(), expectedNonsensitiveMap); + } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/docs/src/site/sphinx/ConnectorDevelopment.rst ---------------------------------------------------------------------- diff --git a/docs/src/site/sphinx/ConnectorDevelopment.rst b/docs/src/site/sphinx/ConnectorDevelopment.rst index b35c521..41389c8 100644 --- a/docs/src/site/sphinx/ConnectorDevelopment.rst +++ b/docs/src/site/sphinx/ConnectorDevelopment.rst @@ -401,6 +401,11 @@ Inputs associated with the link configuration include: +-----------------------------+---------+-----------------------------------------------------------------------+-------------------------------------------------+ | sensitive | Boolean |Describes if the input value should be hidden from display |@Input(sensitive = true) public String password | +-----------------------------+---------+-----------------------------------------------------------------------+-------------------------------------------------+ +| sensitiveKeyPattern | String |If the config paramteter is a map, this java regular expression |@Input(sensitiveKeyPattern = ".*sensitive") | +| | |(http://docs.oracle.com/javase/7/docs/api/java/util/regex/Pattern.html)|public Map<String, String> sensitiveMap | +| | |will be used to decide which keys are hidden from display. | | +| | | | | ++-----------------------------+---------+-----------------------------------------------------------------------+-------------------------------------------------+ | editable | Enum |Describes the roles that can edit the value of this input |@Input(editable = ANY) public String value | +-----------------------------+---------+-----------------------------------------------------------------------+-------------------------------------------------+ | overrides | String |Describes a list of other inputs this input can override in this config|@Input(overrides ="value") public String lvalue | http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java ---------------------------------------------------------------------- diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java index 46b5272..79b9e99 100644 --- a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java +++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java @@ -1998,7 +1998,7 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { input = new MStringInput(inputName, inputSensitivity, editableEnum, overrides, inputStrLength); break; case MAP: - input = new MMapInput(inputName, inputSensitivity, editableEnum, overrides); + input = new MMapInput(inputName, inputSensitivity, editableEnum, overrides, StringUtils.EMPTY); break; case BOOLEAN: input = new MBooleanInput(inputName, inputSensitivity, editableEnum, overrides); @@ -2176,7 +2176,7 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { inputStrLength); break; case MAP: - input = new MMapInput(inputName, inputSensitivity, editableEnum, overrides); + input = new MMapInput(inputName, inputSensitivity, editableEnum, overrides, StringUtils.EMPTY); break; case BOOLEAN: input = new MBooleanInput(inputName, inputSensitivity, editableEnum, overrides); http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java ---------------------------------------------------------------------- diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java index e41fe38..ed176fe 100644 --- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java +++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java @@ -1021,7 +1021,7 @@ abstract public class DerbyTestCase { List<MInput<?>> inputs = new LinkedList<MInput<?>>(); MInput input = new MStringInput(configName1 + ".I1", false, InputEditable.USER_ONLY, configName1 + ".I2", (short) 30); inputs.add(input); - input = new MMapInput(configName1 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName1 + ".I5"); + input = new MMapInput(configName1 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName1 + ".I5", StringUtils.EMPTY); inputs.add(input); input = new MIntegerInput(configName1 + ".I3", false, InputEditable.ANY, configName1 + ".I1"); inputs.add(input); @@ -1034,7 +1034,7 @@ abstract public class DerbyTestCase { inputs = new LinkedList<MInput<?>>(); input = new MStringInput(configName2 + ".I1", false, InputEditable.USER_ONLY, configName2 + ".I2", (short) 30); inputs.add(input); - input = new MMapInput(configName2 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName2 + ".I5"); + input = new MMapInput(configName2 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName2 + ".I5", StringUtils.EMPTY); inputs.add(input); input = new MIntegerInput(configName2 + ".I3", false, InputEditable.ANY, configName2 + ".I1"); inputs.add(input); @@ -1054,7 +1054,7 @@ abstract public class DerbyTestCase { // I1 overrides another user_only attribute, hence a bad config MInput input = new MStringInput(configName1 + ".I1", false, InputEditable.USER_ONLY, configName1 + ".I4", (short) 30); inputs.add(input); - input = new MMapInput(configName1 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName1 + ".I5"); + input = new MMapInput(configName1 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName1 + ".I5", StringUtils.EMPTY); inputs.add(input); input = new MIntegerInput(configName1 + ".I3", false, InputEditable.ANY, configName1 + ".I1"); inputs.add(input); @@ -1067,7 +1067,7 @@ abstract public class DerbyTestCase { inputs = new LinkedList<MInput<?>>(); input = new MStringInput(configName2 + ".I1", false, InputEditable.USER_ONLY, configName2 + ".I2", (short) 30); inputs.add(input); - input = new MMapInput(configName2 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName2 + ".I5"); + input = new MMapInput(configName2 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName2 + ".I5", StringUtils.EMPTY); inputs.add(input); input = new MIntegerInput(configName2 + ".I3", false, InputEditable.ANY, configName2 + ".I1"); inputs.add(input); @@ -1087,7 +1087,7 @@ abstract public class DerbyTestCase { // I1 overrides another user_only attribute, hence a bad config MInput input = new MStringInput(configName1 + ".I1", false, InputEditable.USER_ONLY, configName1 + ".I4", (short) 30); inputs.add(input); - input = new MMapInput(configName1 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName1 + ".I5"); + input = new MMapInput(configName1 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName1 + ".I5", StringUtils.EMPTY); inputs.add(input); input = new MIntegerInput(configName1 + ".I3", false, InputEditable.ANY, configName1 + ".I1"); inputs.add(input); @@ -1100,7 +1100,7 @@ abstract public class DerbyTestCase { inputs = new LinkedList<MInput<?>>(); input = new MStringInput(configName2 + ".I1", false, InputEditable.USER_ONLY, configName2 + ".I2", (short) 30); inputs.add(input); - input = new MMapInput(configName2 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName2 + ".I5"); + input = new MMapInput(configName2 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName2 + ".I5", StringUtils.EMPTY); inputs.add(input); input = new MIntegerInput(configName2 + ".I3", false, InputEditable.ANY, configName2 + ".I1"); inputs.add(input); @@ -1119,7 +1119,7 @@ abstract public class DerbyTestCase { List<MInput<?>> inputs = new LinkedList<MInput<?>>(); MInput input = new MStringInput(configName1 + ".I1", false, InputEditable.USER_ONLY, configName1 + ".I2", (short) 30); inputs.add(input); - input = new MMapInput(configName1 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName1 + ".I1," + configName1 + ".I3"); + input = new MMapInput(configName1 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName1 + ".I1," + configName1 + ".I3", StringUtils.EMPTY); inputs.add(input); input = new MIntegerInput(configName1 + ".I3", false, InputEditable.ANY, configName1 + ".I1"); inputs.add(input); @@ -1132,7 +1132,7 @@ abstract public class DerbyTestCase { inputs = new LinkedList<MInput<?>>(); input = new MStringInput(configName2 + ".I1", false, InputEditable.USER_ONLY, configName2 + ".I2", (short) 30); inputs.add(input); - input = new MMapInput(configName2 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName2 + ".I5"); + input = new MMapInput(configName2 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName2 + ".I5", StringUtils.EMPTY); inputs.add(input); input = new MIntegerInput(configName2 + ".I3", false, InputEditable.ANY, configName2 + ".I1"); inputs.add(input); @@ -1153,7 +1153,7 @@ abstract public class DerbyTestCase { // I2 overrides a nonexistant input MInput input = new MStringInput(configName1 + ".I1", false, InputEditable.USER_ONLY, configName1 + ".I2", (short) 30); inputs.add(input); - input = new MMapInput(configName1 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName1 + ".FOO"); + input = new MMapInput(configName1 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName1 + ".FOO", StringUtils.EMPTY); inputs.add(input); input = new MIntegerInput(configName1 + ".I3", false, InputEditable.ANY, configName1 + ".I1"); inputs.add(input); @@ -1166,7 +1166,7 @@ abstract public class DerbyTestCase { inputs = new LinkedList<MInput<?>>(); input = new MStringInput(configName2 + ".I1", false, InputEditable.USER_ONLY, configName2 + ".I2", (short) 30); inputs.add(input); - input = new MMapInput(configName2 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName2 + ".I5"); + input = new MMapInput(configName2 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName2 + ".I5", StringUtils.EMPTY); inputs.add(input); input = new MIntegerInput(configName2 + ".I3", false, InputEditable.ANY, configName2 + ".I1"); inputs.add(input); http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/repository/repository-mysql/src/test/java/org/apache/sqoop/integration/repository/mysql/MySqlTestCase.java ---------------------------------------------------------------------- diff --git a/repository/repository-mysql/src/test/java/org/apache/sqoop/integration/repository/mysql/MySqlTestCase.java b/repository/repository-mysql/src/test/java/org/apache/sqoop/integration/repository/mysql/MySqlTestCase.java index 0bb3c63..44f968a 100644 --- a/repository/repository-mysql/src/test/java/org/apache/sqoop/integration/repository/mysql/MySqlTestCase.java +++ b/repository/repository-mysql/src/test/java/org/apache/sqoop/integration/repository/mysql/MySqlTestCase.java @@ -170,14 +170,14 @@ abstract public class MySqlTestCase extends TestCase { MInput<?> input = new MStringInput("I1", false, InputEditable.ANY, StringUtils.EMPTY, (short) 30); inputs.add(input); - input = new MMapInput("I2", false, InputEditable.ANY, "I1"); + input = new MMapInput("I2", false, InputEditable.ANY, "I1", StringUtils.EMPTY); inputs.add(input); configs.add(new MConfig(configName1, inputs)); inputs = new LinkedList<MInput<?>>(); input = new MStringInput("I3", false, InputEditable.ANY, "I4", (short) 30); inputs.add(input); - input = new MMapInput("I4", false, InputEditable.ANY, "I3"); + input = new MMapInput("I4", false, InputEditable.ANY, "I3", StringUtils.EMPTY); inputs.add(input); configs.add(new MConfig(configName2, inputs)); http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestCase.java ---------------------------------------------------------------------- diff --git a/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestCase.java b/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestCase.java index 389ccec..faa0399 100644 --- a/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestCase.java +++ b/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestCase.java @@ -151,14 +151,14 @@ abstract public class PostgresqlTestCase { List<MInput<?>> inputs = new LinkedList<MInput<?>>(); MInput<?> input = new MStringInput("I1", false, InputEditable.ANY, StringUtils.EMPTY, (short) 30); inputs.add(input); - input = new MMapInput("I2", false, InputEditable.ANY, "I1"); + input = new MMapInput("I2", false, InputEditable.ANY, "I1", StringUtils.EMPTY); inputs.add(input); configs.add(new MConfig(configName1, inputs)); inputs = new LinkedList<MInput<?>>(); input = new MStringInput("I3", false, InputEditable.ANY, "I4", (short) 30); inputs.add(input); - input = new MMapInput("I4", false, InputEditable.ANY, "I3"); + input = new MMapInput("I4", false, InputEditable.ANY, "I3", StringUtils.EMPTY); inputs.add(input); configs.add(new MConfig(configName2, inputs));
