Repository: tez Updated Branches: refs/heads/branch-0.7 35bd5ab0b -> 0bb3b1d23
TEZ-2907. NPE in IFile.Reader.getLength during final merge operation (rbalamohan) (cherry picked from commit f3cea630e8329018a1e600859353576cb83cf77d) # Conflicts: # CHANGES.txt Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/0bb3b1d2 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/0bb3b1d2 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/0bb3b1d2 Branch: refs/heads/branch-0.7 Commit: 0bb3b1d2300f02b62a0858c09f2835ca9d0bbdbe Parents: 35bd5ab Author: Rajesh Balamohan <[email protected]> Authored: Thu Oct 22 13:59:05 2015 -0700 Committer: Rajesh Balamohan <[email protected]> Committed: Thu Oct 22 14:06:57 2015 -0700 ---------------------------------------------------------------------- CHANGES.txt | 2 ++ .../library/common/shuffle/orderedgrouped/MergeManager.java | 6 ++++++ 2 files changed, 8 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/0bb3b1d2/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 29e61bf..ba5b75d 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -6,6 +6,7 @@ Release 0.7.1: Unreleased INCOMPATIBLE CHANGES ALL CHANGES + TEZ-2907. NPE in IFile.Reader.getLength during final merge operation TEZ-2850. Tez MergeManager OOM for small Map Outputs TEZ-2886. Ability to merge AM credentials with DAG credentials. TEZ-2896. Fix thread names used during Input/Output initialization. @@ -295,6 +296,7 @@ Release 0.6.3: Unreleased INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-2907. NPE in IFile.Reader.getLength during final merge operation TEZ-2850. Tez MergeManager OOM for small Map Outputs TEZ-2781. Fallback to send only TaskAttemptFailedEvent if taskFailed heartbeat fails TEZ-2855. Fix a potential NPE while routing VertexManager events. http://git-wip-us.apache.org/repos/asf/tez/blob/0bb3b1d2/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java index e25c064..9516f27 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java @@ -846,12 +846,14 @@ public class MergeManager { class RawKVIteratorReader extends IFile.Reader { private final TezRawKeyValueIterator kvIter; + private final long size; public RawKVIteratorReader(TezRawKeyValueIterator kvIter, long size) throws IOException { super(null, size, null, spilledRecordsCounter, null, ifileReadAhead, ifileReadAheadLength, ifileBufferSize); this.kvIter = kvIter; + this.size = size; } @Override public KeyState readRawKey(DataInputBuffer key) throws IOException { @@ -879,6 +881,10 @@ public class MergeManager { public void close() throws IOException { kvIter.close(); } + + @Override public long getLength() { + return size; + } } private TezRawKeyValueIterator finalMerge(Configuration job, FileSystem fs,
