Repository: hadoop Updated Branches: refs/heads/branch-2.9 90dc66553 -> 91a4fb54f
HADOOP-15973. Configuration: Included properties are not cached if resource is a stream. Contributed by Eric Payne (cherry picked from commit d62bfaf1a409c7cd034f9e76e97a61a77d517a25) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/91a4fb54 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/91a4fb54 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/91a4fb54 Branch: refs/heads/branch-2.9 Commit: 91a4fb54f13a29dc2af76fc7582ee79a063127ba Parents: 90dc665 Author: Jason Lowe <[email protected]> Authored: Thu Dec 20 10:02:18 2018 -0600 Committer: Jason Lowe <[email protected]> Committed: Thu Dec 20 10:03:01 2018 -0600 ---------------------------------------------------------------------- .../org/apache/hadoop/conf/Configuration.java | 6 +- .../apache/hadoop/conf/TestConfiguration.java | 99 ++++++++++++++++++++ 2 files changed, 103 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/91a4fb54/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index 235cc50..9ee1baf 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -2848,10 +2848,11 @@ public class Configuration implements Iterable<Map.Entry<String,String>>, // otherwise fallback to a file resource // xi:include are treated as inline and retain current source URL include = getResource(confInclude); + Properties tmpProps = new Properties(); if (include != null) { Resource classpathResource = new Resource(include, name, wrapper.isParserRestricted()); - loadResource(properties, classpathResource, quiet); + loadResource(tmpProps, classpathResource, quiet); } else { URL url; try { @@ -2873,8 +2874,9 @@ public class Configuration implements Iterable<Map.Entry<String,String>>, } Resource uriResource = new Resource(url, name, wrapper.isParserRestricted()); - loadResource(properties, uriResource, quiet); + loadResource(tmpProps, uriResource, quiet); } + toAddTo.putAll(tmpProps); break; case "fallback": fallbackEntered = true; http://git-wip-us.apache.org/repos/asf/hadoop/blob/91a4fb54/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java index 214be63..6fcff58 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java @@ -17,10 +17,12 @@ */ package org.apache.hadoop.conf; +import java.io.BufferedInputStream; import java.io.BufferedWriter; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.FileWriter; import java.io.IOException; @@ -773,6 +775,103 @@ public class TestConfiguration extends TestCase { tearDown(); } + // When a resource is parsed as an input stream the first time, included + // properties are saved within the config. However, the included properties + // are not cached in the resource object. So, if an additional resource is + // added after the config is parsed the first time, the config loses the + // prperties that were included from the first resource. + public void testIncludesFromInputStreamWhenResourceAdded() throws Exception { + tearDown(); + + // CONFIG includes CONFIG2. CONFIG2 includes CONFIG_FOR_ENUM + out=new BufferedWriter(new FileWriter(CONFIG_FOR_ENUM)); + startConfig(); + appendProperty("e", "SecondLevelInclude"); + appendProperty("f", "SecondLevelInclude"); + endConfig(); + + out=new BufferedWriter(new FileWriter(CONFIG2)); + startConfig(); + startInclude(CONFIG_FOR_ENUM); + endInclude(); + appendProperty("c","FirstLevelInclude"); + appendProperty("d","FirstLevelInclude"); + endConfig(); + + out=new BufferedWriter(new FileWriter(CONFIG)); + startConfig(); + startInclude(CONFIG2); + endInclude(); + appendProperty("a", "1"); + appendProperty("b", "2"); + endConfig(); + + // Add CONFIG as an InputStream resource. + File file = new File(CONFIG); + BufferedInputStream bis = + new BufferedInputStream(new FileInputStream(file)); + conf.addResource(bis); + + // The first time the conf is parsed, verify that all properties were read + // from all levels of includes. + assertEquals("1", conf.get("a")); + assertEquals("2", conf.get("b")); + assertEquals("FirstLevelInclude", conf.get("c")); + assertEquals("FirstLevelInclude", conf.get("d")); + assertEquals("SecondLevelInclude", conf.get("e")); + assertEquals("SecondLevelInclude", conf.get("f")); + + // Add another resource to the conf. + out=new BufferedWriter(new FileWriter(CONFIG_MULTI_BYTE)); + startConfig(); + appendProperty("g", "3"); + appendProperty("h", "4"); + endConfig(); + + Path fileResource = new Path(CONFIG_MULTI_BYTE); + conf.addResource(fileResource); + + // Verify that all properties were read from all levels of includes the + // second time the conf is parsed. + assertEquals("1", conf.get("a")); + assertEquals("2", conf.get("b")); + assertEquals("FirstLevelInclude", conf.get("c")); + assertEquals("FirstLevelInclude", conf.get("d")); + assertEquals("SecondLevelInclude", conf.get("e")); + assertEquals("SecondLevelInclude", conf.get("f")); + assertEquals("3", conf.get("g")); + assertEquals("4", conf.get("h")); + + tearDown(); + } + + public void testOrderOfDuplicatePropertiesWithInclude() throws Exception { + tearDown(); + + // Property "a" is set to different values inside and outside of includes. + out=new BufferedWriter(new FileWriter(CONFIG2)); + startConfig(); + appendProperty("a", "a-InsideInclude"); + appendProperty("b", "b-InsideInclude"); + endConfig(); + + out=new BufferedWriter(new FileWriter(CONFIG)); + startConfig(); + appendProperty("a","a-OutsideInclude"); + startInclude(CONFIG2); + endInclude(); + appendProperty("b","b-OutsideInclude"); + endConfig(); + + Path fileResource = new Path(CONFIG); + conf.addResource(fileResource); + + assertEquals("a-InsideInclude", conf.get("a")); + assertEquals("b-OutsideInclude", conf.get("b")); + + tearDown(); + } + public void testRelativeIncludes() throws Exception { tearDown(); String relConfig = new File("./tmp/test-config.xml").getAbsolutePath(); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
