Repository: tez Updated Branches: refs/heads/master 149db1b48 -> 5bd5613fc
TEZ-3368. NPE in DelayedContainerManager (Jason Lowe via jeagles) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/5bd5613f Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/5bd5613f Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/5bd5613f Branch: refs/heads/master Commit: 5bd5613fca24bb521f0d8a9dd2cace4f80c5e90a Parents: 149db1b Author: Jonathan Eagles <[email protected]> Authored: Fri Oct 7 11:32:17 2016 -0500 Committer: Jonathan Eagles <[email protected]> Committed: Fri Oct 7 11:32:17 2016 -0500 ---------------------------------------------------------------------- CHANGES.txt | 3 +++ .../apache/tez/dag/app/rm/TezAMRMClientAsync.java | 17 +++++++---------- .../tez/dag/app/rm/YarnTaskSchedulerService.java | 9 +++++++++ 3 files changed, 19 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/5bd5613f/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index c85bc8e..4167747 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3368. NPE in DelayedContainerManager TEZ-3440. Shuffling to memory can get out-of-sync when fetching multiple compressed map outputs TEZ-3429. Set reconfigureDoneTime on VertexConfigurationDoneEvent properly. TEZ-3000. Fix TestContainerReuse. @@ -115,6 +116,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3368. NPE in DelayedContainerManager TEZ-3440. Shuffling to memory can get out-of-sync when fetching multiple compressed map outputs TEZ-3429. Set reconfigureDoneTime on VertexConfigurationDoneEvent properly. TEZ-3000. Fix TestContainerReuse. @@ -606,6 +608,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3368. NPE in DelayedContainerManager TEZ-3440. Shuffling to memory can get out-of-sync when fetching multiple compressed map outputs TEZ-3426. Second AM attempt launched for session mode and recovery disabled for certain cases TEZ-2852. TestVertexImpl fails due to race in AsyncDispatcher. http://git-wip-us.apache.org/repos/asf/tez/blob/5bd5613f/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsync.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsync.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsync.java index 7b3df84..40bc3aa 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsync.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsync.java @@ -20,8 +20,8 @@ package org.apache.tez.dag.app.rm; import java.util.Collection; import java.util.Collections; -import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.TreeMap; import java.util.concurrent.atomic.AtomicInteger; @@ -74,12 +74,10 @@ public class TezAMRMClientAsync<T extends ContainerRequest> extends AMRMClientAs } public synchronized Priority getTopPriority() { - Iterator<Priority> iter = - knownRequestsByPriority.descendingKeySet().iterator(); - if (!iter.hasNext()) { + if (knownRequestsByPriority.isEmpty()) { return null; } - return iter.next(); + return knownRequestsByPriority.lastKey(); } // Remove after YARN-1723 is fixed @@ -132,13 +130,12 @@ public class TezAMRMClientAsync<T extends ContainerRequest> extends AMRMClientAs String resourceName, Resource capability) { // Sort based on reverse order. By default, Priority ordering is based on // highest numeric value being considered to be lowest priority. - Iterator<Priority> iter = - knownRequestsByPriority.descendingKeySet().iterator(); - if (!iter.hasNext()) { + Map.Entry<Priority,LocalityRequestCounter> entry = knownRequestsByPriority.lastEntry(); + if (entry == null || entry.getValue() == null) { return Collections.emptyList(); } - Priority p = iter.next(); - LocalityRequestCounter lrc = knownRequestsByPriority.get(p); + Priority p = entry.getKey(); + LocalityRequestCounter lrc = entry.getValue(); if (lrc.localityRequests.get() == 0) { // Fallback to ANY if there are no pending requests that require // locality matching http://git-wip-us.apache.org/repos/asf/tez/blob/5bd5613f/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java index 6dfc7b9..5087d0d 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java @@ -47,6 +47,7 @@ import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.math3.random.RandomDataGenerator; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -1958,6 +1959,14 @@ public class YarnTaskSchedulerService extends TaskScheduler @Override public void run() { + try { + mainLoop(); + } catch (Throwable e) { + ExitUtil.terminate(1, e); + } + } + + private void mainLoop() { while(running) { // Try assigning all containers if there's a request to do so. if (tryAssigningAll) {
