Repository: tez Updated Branches: refs/heads/master 4a7719b0c -> a7f93ae1d
TEZ-3761. NPE in Fetcher under load (jeagles) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/a7f93ae1 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/a7f93ae1 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/a7f93ae1 Branch: refs/heads/master Commit: a7f93ae1da114fa03f7e500a12706e5737b189ee Parents: 4a7719b Author: Jonathan Eagles <[email protected]> Authored: Fri Jun 23 09:56:00 2017 -0500 Committer: Jonathan Eagles <[email protected]> Committed: Fri Jun 23 09:56:00 2017 -0500 ---------------------------------------------------------------------- .../tez/runtime/library/common/shuffle/Fetcher.java | 10 ++++++++++ 1 file changed, 10 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/a7f93ae1/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java index fefd53e..bf8c83b 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java @@ -853,7 +853,17 @@ public class Fetcher extends CallableWithNdc<FetchResult> { ShuffleHeader header = new ShuffleHeader(); header.readFields(input); pathComponent = header.getMapId(); + if (!pathComponent.startsWith(InputAttemptIdentifier.PATH_PREFIX)) { + throw new IllegalArgumentException("Invalid map id: " + header.getMapId() + ", expected to start with " + + InputAttemptIdentifier.PATH_PREFIX + ", partition: " + header.getPartition() + + " while fetching " + inputAttemptIdentifier); + } + srcAttemptId = pathToAttemptMap.get(new PathPartition(pathComponent, header.getPartition())); + if (srcAttemptId == null) { + throw new IllegalArgumentException("Source attempt not found for map id: " + header.getMapId() + + ", partition: " + header.getPartition() + " while fetching " + inputAttemptIdentifier); + } if (header.getCompressedLength() == 0) { // Empty partitions are already accounted for
