Repository: hadoop Updated Branches: refs/heads/trunk 24f75e097 -> 28790b81e
HADOOP-15295. Remove redundant logging related to tags from Configuration. Contributed by Ajay Kumar. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/28790b81 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/28790b81 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/28790b81 Branch: refs/heads/trunk Commit: 28790b81ecb719ac26a01ad0c400ee5f1c29ccf8 Parents: 24f75e0 Author: Anu Engineer <[email protected]> Authored: Fri Mar 23 21:17:18 2018 -0700 Committer: Anu Engineer <[email protected]> Committed: Fri Mar 23 21:17:18 2018 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/conf/Configuration.java | 41 ++++++++++---------- .../apache/hadoop/conf/TestConfiguration.java | 34 ++++------------ 2 files changed, 27 insertions(+), 48 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/28790b81/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index e0a5866..25fd656a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -229,7 +229,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>, private static final Logger LOG_DEPRECATION = LoggerFactory.getLogger( "org.apache.hadoop.conf.Configuration.deprecation"); - private static final Set<String> TAGS = new HashSet<>(); + private static final Set<String> TAGS = ConcurrentHashMap.newKeySet(); private boolean quietmode = true; @@ -2935,7 +2935,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>, resources.set(i, ret); } } - this.removeUndeclaredTags(properties); + this.addTags(properties); } private Resource loadResource(Properties properties, @@ -3183,29 +3183,28 @@ public class Configuration implements Iterable<Map.Entry<String,String>>, } /** - * Removes undeclared tags and related properties from propertyTagsMap. - * Its required because ordering of properties in xml config files is not - * guaranteed. + * Add tags defined in HADOOP_SYSTEM_TAGS, HADOOP_CUSTOM_TAGS. * @param prop */ - private void removeUndeclaredTags(Properties prop) { + public void addTags(Properties prop) { // Get all system tags - if (prop.containsKey(CommonConfigurationKeys.HADOOP_SYSTEM_TAGS)){ - String systemTags = prop.getProperty(CommonConfigurationKeys - .HADOOP_SYSTEM_TAGS); - Arrays.stream(systemTags.split(",")).forEach(tag -> TAGS.add(tag)); - } - // Get all custom tags - if (prop.containsKey(CommonConfigurationKeys.HADOOP_CUSTOM_TAGS)) { - String customTags = prop.getProperty(CommonConfigurationKeys - .HADOOP_CUSTOM_TAGS); - Arrays.stream(customTags.split(",")).forEach(tag -> TAGS.add(tag)); - } + try { + if (prop.containsKey(CommonConfigurationKeys.HADOOP_SYSTEM_TAGS)) { + String systemTags = prop.getProperty(CommonConfigurationKeys + .HADOOP_SYSTEM_TAGS); + Arrays.stream(systemTags.split(",")).forEach(tag -> TAGS.add(tag)); + } + // Get all custom tags + if (prop.containsKey(CommonConfigurationKeys.HADOOP_CUSTOM_TAGS)) { + String customTags = prop.getProperty(CommonConfigurationKeys + .HADOOP_CUSTOM_TAGS); + Arrays.stream(customTags.split(",")).forEach(tag -> TAGS.add(tag)); + } - Set undeclaredTags = propertyTagsMap.keySet(); - if (undeclaredTags.retainAll(TAGS)) { - LOG.info("Removed undeclared tags:"); + } catch (Exception ex) { + LOG.trace("Error adding tags in configuration", ex); } + } /** @@ -3219,8 +3218,8 @@ public class Configuration implements Iterable<Map.Entry<String,String>>, private void readTagFromConfig(String attributeValue, String confName, String confValue, List<String> confSource) { for (String tagStr : attributeValue.split(",")) { - tagStr = tagStr.trim(); try { + tagStr = tagStr.trim(); // Handle property with no/null value if (confValue == null) { confValue = ""; http://git-wip-us.apache.org/repos/asf/hadoop/blob/28790b81/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java index 8f54464..b0bb0d7 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java @@ -27,7 +27,6 @@ import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.io.OutputStreamWriter; -import java.io.PrintStream; import java.io.StringWriter; import java.net.InetAddress; import java.net.InetSocketAddress; @@ -2392,33 +2391,14 @@ public class TestConfiguration { @Test public void testInvalidTags() throws Exception { - PrintStream output = System.out; - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - System.setOut(new PrintStream(bytes)); - - out = new BufferedWriter(new FileWriter(CONFIG)); - startConfig(); - appendPropertyByTag("dfs.cblock.trace.io", "false", "MYOWNTAG,TAG2"); - endConfig(); - - Path fileResource = new Path(CONFIG); - conf.addResource(fileResource); - conf.getProps(); - - List<String> tagList = new ArrayList<>(); - tagList.add("REQUIRED"); - tagList.add("MYOWNTAG"); - tagList.add("TAG2"); + Path fileResource = new Path(CONFIG); + conf.addResource(fileResource); + conf.getProps(); - Properties properties = conf.getAllPropertiesByTags(tagList); - assertEq(0, properties.size()); - assertFalse(properties.containsKey("dfs.cblock.trace.io")); - assertFalse(bytes.toString().contains("Invalid tag ")); - assertFalse(bytes.toString().contains("Tag")); - } finally { - System.setOut(output); - } + assertFalse(conf.isPropertyTag("BADTAG")); + assertFalse(conf.isPropertyTag("CUSTOM_TAG")); + assertTrue(conf.isPropertyTag("DEBUG")); + assertTrue(conf.isPropertyTag("HDFS")); } /** --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
