Repository: tez Updated Branches: refs/heads/master 01ad29e44 -> f3cea630e
TEZ-2907. NPE in IFile.Reader.getLength during final merge operation (rbalamohan) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/f3cea630 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/f3cea630 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/f3cea630 Branch: refs/heads/master Commit: f3cea630e8329018a1e600859353576cb83cf77d Parents: 01ad29e Author: Rajesh Balamohan <[email protected]> Authored: Thu Oct 22 13:59:05 2015 -0700 Committer: Rajesh Balamohan <[email protected]> Committed: Thu Oct 22 13:59:05 2015 -0700 ---------------------------------------------------------------------- CHANGES.txt | 3 +++ .../library/common/shuffle/orderedgrouped/MergeManager.java | 6 ++++++ 2 files changed, 9 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/f3cea630/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index aec51ed..d19df34 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -6,6 +6,7 @@ Release 0.8.2: Unreleased INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-2907. NPE in IFile.Reader.getLength during final merge operation TEZ-2903. Stop using proprietary APIs in RPCLoadGen. TEZ-2882. Consider improving fetch failure handling TEZ-2850. Tez MergeManager OOM for small Map Outputs @@ -220,6 +221,7 @@ Release 0.7.1: Unreleased INCOMPATIBLE CHANGES ALL CHANGES + TEZ-2907. NPE in IFile.Reader.getLength during final merge operation TEZ-2850. Tez MergeManager OOM for small Map Outputs TEZ-2886. Ability to merge AM credentials with DAG credentials. TEZ-2896. Fix thread names used during Input/Output initialization. @@ -504,6 +506,7 @@ Release 0.6.3: Unreleased INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-2907. NPE in IFile.Reader.getLength during final merge operation TEZ-2850. Tez MergeManager OOM for small Map Outputs TEZ-2781. Fallback to send only TaskAttemptFailedEvent if taskFailed heartbeat fails TEZ-2855. Fix a potential NPE while routing VertexManager events. http://git-wip-us.apache.org/repos/asf/tez/blob/f3cea630/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java index eb2cece..fb9b243 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java @@ -949,12 +949,14 @@ public class MergeManager implements FetchedInputAllocatorOrderedGrouped { class RawKVIteratorReader extends IFile.Reader { private final TezRawKeyValueIterator kvIter; + private final long size; public RawKVIteratorReader(TezRawKeyValueIterator kvIter, long size) throws IOException { super(null, size, null, spilledRecordsCounter, null, ifileReadAhead, ifileReadAheadLength, ifileBufferSize); this.kvIter = kvIter; + this.size = size; } @Override public KeyState readRawKey(DataInputBuffer key) throws IOException { @@ -982,6 +984,10 @@ public class MergeManager implements FetchedInputAllocatorOrderedGrouped { public void close() throws IOException { kvIter.close(); } + + @Override public long getLength() { + return size; + } } private TezRawKeyValueIterator finalMerge(Configuration job, FileSystem fs,
