Repository: tez Updated Branches: refs/heads/branch-0.7 5ee6c30cc -> 49a38f012
TEZ-3368. NPE in DelayedContainerManager (Jason Lowe via jeagles) (cherry picked from commit 5bd5613fca24bb521f0d8a9dd2cace4f80c5e90a) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/49a38f01 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/49a38f01 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/49a38f01 Branch: refs/heads/branch-0.7 Commit: 49a38f0127dd4a5195eb643766975a82cf4fe40a Parents: 5ee6c30 Author: Jonathan Eagles <[email protected]> Authored: Fri Oct 7 11:32:17 2016 -0500 Committer: Jonathan Eagles <[email protected]> Committed: Fri Oct 7 12:19:36 2016 -0500 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../apache/tez/dag/app/rm/TezAMRMClientAsync.java | 17 +++++++---------- .../tez/dag/app/rm/YarnTaskSchedulerService.java | 9 +++++++++ 3 files changed, 17 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/49a38f01/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 1e93ce3..8ab6833 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -8,6 +8,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: TEZ-3460. Fix precommit release audit warning. + TEZ-3368. NPE in DelayedContainerManager TEZ-3440. Shuffling to memory can get out-of-sync when fetching multiple compressed map outputs TEZ-3426. Second AM attempt launched for session mode and recovery disabled for certain cases TEZ-2852. TestVertexImpl fails due to race in AsyncDispatcher. http://git-wip-us.apache.org/repos/asf/tez/blob/49a38f01/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsync.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsync.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsync.java index 7b3df84..40bc3aa 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsync.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsync.java @@ -20,8 +20,8 @@ package org.apache.tez.dag.app.rm; import java.util.Collection; import java.util.Collections; -import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.TreeMap; import java.util.concurrent.atomic.AtomicInteger; @@ -74,12 +74,10 @@ public class TezAMRMClientAsync<T extends ContainerRequest> extends AMRMClientAs } public synchronized Priority getTopPriority() { - Iterator<Priority> iter = - knownRequestsByPriority.descendingKeySet().iterator(); - if (!iter.hasNext()) { + if (knownRequestsByPriority.isEmpty()) { return null; } - return iter.next(); + return knownRequestsByPriority.lastKey(); } // Remove after YARN-1723 is fixed @@ -132,13 +130,12 @@ public class TezAMRMClientAsync<T extends ContainerRequest> extends AMRMClientAs String resourceName, Resource capability) { // Sort based on reverse order. By default, Priority ordering is based on // highest numeric value being considered to be lowest priority. - Iterator<Priority> iter = - knownRequestsByPriority.descendingKeySet().iterator(); - if (!iter.hasNext()) { + Map.Entry<Priority,LocalityRequestCounter> entry = knownRequestsByPriority.lastEntry(); + if (entry == null || entry.getValue() == null) { return Collections.emptyList(); } - Priority p = iter.next(); - LocalityRequestCounter lrc = knownRequestsByPriority.get(p); + Priority p = entry.getKey(); + LocalityRequestCounter lrc = entry.getValue(); if (lrc.localityRequests.get() == 0) { // Fallback to ANY if there are no pending requests that require // locality matching http://git-wip-us.apache.org/repos/asf/tez/blob/49a38f01/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java index f8e2b5d..e000c56 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java @@ -44,6 +44,7 @@ import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.math3.random.RandomDataGenerator; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -1969,6 +1970,14 @@ public class YarnTaskSchedulerService extends TaskSchedulerService @Override public void run() { + try { + mainLoop(); + } catch (Throwable e) { + ExitUtil.terminate(1, e); + } + } + + private void mainLoop() { while(running) { // Try assigning all containers if there's a request to do so. if (tryAssigningAll) {
