Repository: hadoop Updated Branches: refs/heads/branch-2 85bdc3a01 -> 3a4211ec2
MAPREDUCE-6169. MergeQueue should release reference to the current item from key and value at the end of the iteration to save memory. (Zhihai Xu via kasha) (cherry picked from commit 90194ca1cbd695d48c3705121c2ac9a8554578a2) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3a4211ec Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3a4211ec Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3a4211ec Branch: refs/heads/branch-2 Commit: 3a4211ec2ca28403dd985d01c7842036b74ee04c Parents: 85bdc3a Author: Karthik Kambatla <[email protected]> Authored: Thu Nov 20 15:36:57 2014 -0800 Committer: Karthik Kambatla <[email protected]> Committed: Thu Nov 20 15:37:18 2014 -0800 ---------------------------------------------------------------------- hadoop-mapreduce-project/CHANGES.txt | 4 ++++ .../src/main/java/org/apache/hadoop/mapred/Merger.java | 11 ++++++++++- .../apache/hadoop/mapreduce/task/reduce/TestMerger.java | 2 ++ 3 files changed, 16 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a4211ec/hadoop-mapreduce-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 0297d9a..c1c9c6c 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -10,6 +10,10 @@ Release 2.7.0 - UNRELEASED OPTIMIZATIONS + MAPREDUCE-6169. MergeQueue should release reference to the current item + from key and value at the end of the iteration to save memory. + (Zhihai Xu via kasha) + BUG FIXES MAPREDUCE-5918. LineRecordReader can return the same decompressor to http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a4211ec/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Merger.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Merger.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Merger.java index b44e742..fffa92a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Merger.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Merger.java @@ -528,9 +528,17 @@ public class Merger { } } + private void resetKeyValue() { + key = null; + value.reset(new byte[] {}, 0); + diskIFileValue.reset(new byte[] {}, 0); + } + public boolean next() throws IOException { - if (size() == 0) + if (size() == 0) { + resetKeyValue(); return false; + } if (minSegment != null) { //minSegment is non-null for all invocations of next except the first @@ -539,6 +547,7 @@ public class Merger { adjustPriorityQueue(minSegment); if (size() == 0) { minSegment = null; + resetKeyValue(); return false; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a4211ec/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMerger.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMerger.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMerger.java index 651dd38..6e3bedf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMerger.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMerger.java @@ -294,6 +294,8 @@ public class TestMerger { // Now there should be no more input Assert.assertFalse(mergeQueue.next()); Assert.assertEquals(1.0f, mergeQueue.getProgress().get(), epsilon); + Assert.assertTrue(mergeQueue.getKey() == null); + Assert.assertEquals(0, mergeQueue.getValue().getData().length); } private Progressable getReporter() {
