Repository: tez Updated Branches: refs/heads/branch-0.6 f36eeb673 -> c48bc6fb4
TEZ-2907. NPE in IFile.Reader.getLength during final merge operation (rbalamohan) (cherry picked from commit f3cea630e8329018a1e600859353576cb83cf77d) # Conflicts: # CHANGES.txt Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/c48bc6fb Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/c48bc6fb Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/c48bc6fb Branch: refs/heads/branch-0.6 Commit: c48bc6fb4c27b5c63016f637e46e919cbc7d20c7 Parents: f36eeb6 Author: Rajesh Balamohan <[email protected]> Authored: Thu Oct 22 13:59:05 2015 -0700 Committer: Rajesh Balamohan <[email protected]> Committed: Thu Oct 22 14:13:21 2015 -0700 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../library/common/shuffle/orderedgrouped/MergeManager.java | 6 ++++++ 2 files changed, 7 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/c48bc6fb/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 0356b75..a91aa7f 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -6,6 +6,7 @@ Release 0.6.3: Unreleased INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-2907. NPE in IFile.Reader.getLength during final merge operation TEZ-2850. Tez MergeManager OOM for small Map Outputs TEZ-2781. Fallback to send only TaskAttemptFailedEvent if taskFailed heartbeat fails TEZ-2855. Fix a potential NPE while routing VertexManager events. http://git-wip-us.apache.org/repos/asf/tez/blob/c48bc6fb/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java index efecb8a..f498787 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java @@ -812,12 +812,14 @@ public class MergeManager { class RawKVIteratorReader extends IFile.Reader { private final TezRawKeyValueIterator kvIter; + private final long size; public RawKVIteratorReader(TezRawKeyValueIterator kvIter, long size) throws IOException { super(null, size, null, spilledRecordsCounter, null, ifileReadAhead, ifileReadAheadLength, ifileBufferSize); this.kvIter = kvIter; + this.size = size; } @Override public KeyState readRawKey(DataInputBuffer key) throws IOException { @@ -845,6 +847,10 @@ public class MergeManager { public void close() throws IOException { kvIter.close(); } + + @Override public long getLength() { + return size; + } } private TezRawKeyValueIterator finalMerge(Configuration job, FileSystem fs,
