Repository: tez Updated Branches: refs/heads/master 7236d1563 -> f86128a8d
TEZ-3834. TaskSchedulerManager NullPointerException during shutdown when failed to start. Contributed by Jonathan Eagles Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/f86128a8 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/f86128a8 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/f86128a8 Branch: refs/heads/master Commit: f86128a8dd8a7bbda6b2b61bcaee336b35c6100a Parents: 7236d15 Author: Jason Lowe <[email protected]> Authored: Tue Sep 19 13:17:11 2017 -0500 Committer: Jason Lowe <[email protected]> Committed: Tue Sep 19 13:17:11 2017 -0500 ---------------------------------------------------------------------- .../tez/dag/app/rm/TaskSchedulerManager.java | 24 ++++++++++++++------ .../dag/app/rm/TestTaskSchedulerManager.java | 17 ++++++++++++++ 2 files changed, 34 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/f86128a8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java index 640e8f6..7c1b926 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java @@ -684,12 +684,14 @@ public class TaskSchedulerManager extends AbstractService implements public void initiateStop() { for (int i = 0 ; i < taskSchedulers.length ; i++) { - try { - taskSchedulers[i].getTaskScheduler().initiateStop(); - } catch (Exception e) { - // Ignore for now as scheduler stop invoked on shutdown - LOG.error("Failed to do a clean initiateStop for Scheduler: " - + Utils.getTaskSchedulerIdentifierString(i, appContext), e); + if (taskSchedulers[i] != null) { + try { + taskSchedulers[i].getTaskScheduler().initiateStop(); + } catch (Exception e) { + // Ignore for now as scheduler stop invoked on shutdown + LOG.error("Failed to do a clean initiateStop for Scheduler: " + + Utils.getTaskSchedulerIdentifierString(i, appContext), e); + } } } } @@ -978,13 +980,21 @@ public class TaskSchedulerManager extends AbstractService implements } public boolean hasUnregistered() { + // Only return true if all task schedulers that were registered successfully unregister + if (taskSchedulers.length == 0) { + return false; + } boolean result = true; - for (int i = 0 ; i < taskSchedulers.length ; i++) { + for (int i = 0; i < taskSchedulers.length; i++) { // Explicitly not catching any exceptions around this API // No clear route to recover. Better to crash. + if (taskSchedulers[i] == null) { + return false; + } try { result = result & this.taskSchedulers[i].hasUnregistered(); } catch (Exception e) { + result = false; String msg = "Error in TaskScheduler when checking if a scheduler has unregistered" + ", scheduler=" + Utils.getTaskSchedulerIdentifierString(i, appContext); LOG.error(msg, e); http://git-wip-us.apache.org/repos/asf/tez/blob/f86128a8/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java index 43805f3..5df25de 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java @@ -570,6 +570,23 @@ public class TestTaskSchedulerManager { @SuppressWarnings("unchecked") @Test(timeout = 5000) + public void testShutdownBeforeStartTaskScheduler() { + Configuration conf = new TezConfiguration(); + AppContext appContext = mock(AppContext.class, RETURNS_DEEP_STUBS); + doReturn(conf).when(appContext).getAMConf(); + + List<NamedEntityDescriptor> list = new LinkedList<>(); + list.add(null); + + TaskSchedulerManager taskSchedulerManager = + new TaskSchedulerManager(appContext, null, null, + null, null, list, false,null); + assertFalse("Should not return true unless actually unregistered successfully", + taskSchedulerManager.hasUnregistered()); + } + + @SuppressWarnings("unchecked") + @Test(timeout = 5000) public void testReportFailureFromTaskScheduler() { String dagName = DAG_NAME; Configuration conf = new TezConfiguration();
