Repository: hadoop Updated Branches: refs/heads/trunk 782971ae7 -> b05604811
MAPREDUCE-6022. map_input_file is missing from streaming job environment. Contributed by Jason Lowe. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b0560481 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b0560481 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b0560481 Branch: refs/heads/trunk Commit: b056048114bf4701ef9dd22486db937cb589e81b Parents: 782971a Author: Kihwal Lee <kih...@apache.org> Authored: Wed Oct 29 12:28:32 2014 -0500 Committer: Kihwal Lee <kih...@apache.org> Committed: Wed Oct 29 12:29:07 2014 -0500 ---------------------------------------------------------------------- .../org/apache/hadoop/conf/Configuration.java | 26 ++++++++++++++++++++ hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../org/apache/hadoop/streaming/PipeMapRed.java | 4 ++- .../java/org/apache/hadoop/streaming/TrApp.java | 3 +++ 4 files changed, 35 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0560481/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index 98fa4f6..a3fae19 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -559,6 +559,32 @@ public class Configuration implements Iterable<Map.Entry<String,String>>, } /** + * Sets all deprecated properties that are not currently set but have a + * corresponding new property that is set. Useful for iterating the + * properties when all deprecated properties for currently set properties + * need to be present. + */ + public void setDeprecatedProperties() { + DeprecationContext deprecations = deprecationContext.get(); + Properties props = getProps(); + Properties overlay = getOverlay(); + for (Map.Entry<String, DeprecatedKeyInfo> entry : + deprecations.getDeprecatedKeyMap().entrySet()) { + String depKey = entry.getKey(); + if (!overlay.contains(depKey)) { + for (String newKey : entry.getValue().newKeys) { + String val = overlay.getProperty(newKey); + if (val != null) { + props.setProperty(depKey, val); + overlay.setProperty(depKey, val); + break; + } + } + } + } + } + + /** * Checks for the presence of the property <code>name</code> in the * deprecation map. Returns the first of the list of new keys if present * in the deprecation map or the <code>name</code> itself. If the property http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0560481/hadoop-mapreduce-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index c0edb37..2c06aa3 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -450,6 +450,9 @@ Release 2.6.0 - UNRELEASED MAPREDUCE-6142. Fixed test failures in TestJobHistoryEventHandler and TestMRTimelineEventHandling. (Zhijie Shen via vinodkv) + MAPREDUCE-6022. map_input_file is missing from streaming job environment. + (jlowe via kihwal) + Release 2.5.1 - 2014-09-05 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0560481/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java index c8d099e..f47e756 100644 --- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java +++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java @@ -235,7 +235,9 @@ public abstract class PipeMapRed { } } - void addJobConfToEnvironment(JobConf conf, Properties env) { + void addJobConfToEnvironment(JobConf jobconf, Properties env) { + JobConf conf = new JobConf(jobconf); + conf.setDeprecatedProperties(); Iterator it = conf.iterator(); while (it.hasNext()) { Map.Entry en = (Map.Entry) it.next(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0560481/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java index e413261..d058332 100644 --- a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java +++ b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java @@ -57,6 +57,9 @@ public class TrApp // the FileSplit context properties are not available in local hadoop.. // so can't check them in this test. + // verify some deprecated properties appear for older stream jobs + expect("map_input_file", env.getProperty("mapreduce_map_input_file")); + expect("map_input_length", env.getProperty("mapreduce_map_input_length")); } // this runs in a subprocess; won't use JUnit's assertTrue()