MAPREDUCE-6105. nconsistent configuration in property mapreduce.reduce.shuffle.merge.percent. Contributed by Ray Chiang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/685dbafb Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/685dbafb Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/685dbafb Branch: refs/heads/YARN-2928 Commit: 685dbafbe2154e5bf4b638da0668ce32d8c879b0 Parents: ce5de93 Author: Harsh J <[email protected]> Authored: Tue Mar 17 01:17:34 2015 +0530 Committer: Harsh J <[email protected]> Committed: Tue Mar 17 02:28:09 2015 +0530 ---------------------------------------------------------------------- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java | 1 + .../apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java | 5 +++-- 3 files changed, 7 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/685dbafb/hadoop-mapreduce-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index d02d725..52880f6 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -253,6 +253,9 @@ Release 2.8.0 - UNRELEASED IMPROVEMENTS + MAPREDUCE-6105. Inconsistent configuration in property + mapreduce.reduce.shuffle.merge.percent. (Ray Chiang via harsh) + MAPREDUCE-4414. Add main methods to JobConf and YarnConfiguration, for debug purposes. (Plamen Jeliazkov via harsh) http://git-wip-us.apache.org/repos/asf/hadoop/blob/685dbafb/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 3aa304a..f0a6ddf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -305,6 +305,7 @@ public interface MRJobConfig { = "mapreduce.reduce.shuffle.memory.limit.percent"; public static final String SHUFFLE_MERGE_PERCENT = "mapreduce.reduce.shuffle.merge.percent"; + public static final float DEFAULT_SHUFFLE_MERGE_PERCENT = 0.66f; public static final String REDUCE_FAILURES_MAXPERCENT = "mapreduce.reduce.failures.maxpercent"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/685dbafb/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java index a4b1aa8..8bf17ef 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java @@ -191,8 +191,9 @@ public class MergeManagerImpl<K, V> implements MergeManager<K, V> { this.memToMemMergeOutputsThreshold = jobConf.getInt(MRJobConfig.REDUCE_MEMTOMEM_THRESHOLD, ioSortFactor); this.mergeThreshold = (long)(this.memoryLimit * - jobConf.getFloat(MRJobConfig.SHUFFLE_MERGE_PERCENT, - 0.90f)); + jobConf.getFloat( + MRJobConfig.SHUFFLE_MERGE_PERCENT, + MRJobConfig.DEFAULT_SHUFFLE_MERGE_PERCENT)); LOG.info("MergerManager: memoryLimit=" + memoryLimit + ", " + "maxSingleShuffleLimit=" + maxSingleShuffleLimit + ", " + "mergeThreshold=" + mergeThreshold + ", " +
