Repository: hadoop Updated Branches: refs/heads/trunk dae505182 -> 268c29a5f
HADOOP-15331. Fix a race condition causing parsing error of java.io.BufferedInputStream in class org.apache.hadoop.conf.Configuration. Contributed by Miklos Szegedi. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/268c29a5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/268c29a5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/268c29a5 Branch: refs/heads/trunk Commit: 268c29a5f541449659f4b4ea1975c6f04c7b6a70 Parents: dae5051 Author: Yufei Gu <[email protected]> Authored: Thu Mar 22 11:04:37 2018 -0700 Committer: Yufei Gu <[email protected]> Committed: Thu Mar 22 11:04:37 2018 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/conf/Configuration.java | 5 +++- .../apache/hadoop/conf/TestConfiguration.java | 31 ++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/268c29a5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index 00b4702..e0a5866 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -816,8 +816,11 @@ public class Configuration implements Iterable<Map.Entry<String,String>>, */ @SuppressWarnings("unchecked") public Configuration(Configuration other) { - this.resources = (ArrayList<Resource>) other.resources.clone(); synchronized(other) { + // Make sure we clone a finalized state + // Resources like input streams can be processed only once + other.getProps(); + this.resources = (ArrayList<Resource>) other.resources.clone(); if (other.properties != null) { this.properties = (Properties)other.properties.clone(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/268c29a5/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java index f1d68cd..8f54464 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.conf; +import java.io.BufferedInputStream; import java.io.BufferedWriter; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -2419,4 +2420,34 @@ public class TestConfiguration { System.setOut(output); } } + + /** + * Test race conditions between clone() and getProps(). + * Test for race conditions in the way Hadoop handles the Configuration + * class. The scenario is the following. Let's assume that there are two + * threads sharing the same Configuration class. One adds some resources + * to the configuration, while the other one clones it. Resources are + * loaded lazily in a deferred call to loadResources(). If the cloning + * happens after adding the resources but before parsing them, some temporary + * resources like input stream pointers are cloned. Eventually both copies + * will load the same input stream resources. + * One parses the input stream XML and closes it updating it's own copy of + * the resource. The other one has another pointer to the same input stream. + * When it tries to load it, it will crash with a stream closed exception. + */ + @Test + public void testResourceRace() { + InputStream is = + new BufferedInputStream(new ByteArrayInputStream( + "<configuration></configuration>".getBytes())); + Configuration config = new Configuration(); + // Thread 1 + config.addResource(is); + // Thread 2 + Configuration confClone = new Configuration(conf); + // Thread 2 + confClone.get("firstParse"); + // Thread 1 + config.get("secondParse"); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
