Repository: tez Updated Branches: refs/heads/master 191447e09 -> 44c660a34
TEZ-3166. Fix a few cases where counters aren't fully updated and sent for failed tasks. Contributed by Zhiyuan Yang. Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/44c660a3 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/44c660a3 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/44c660a3 Branch: refs/heads/master Commit: 44c660a343cbf1993a9988c545a6853b993f5b6a Parents: 191447e Author: Siddharth Seth <[email protected]> Authored: Thu Mar 17 17:38:41 2016 -0700 Committer: Siddharth Seth <[email protected]> Committed: Thu Mar 17 17:38:41 2016 -0700 ---------------------------------------------------------------------- CHANGES.txt | 2 ++ .../java/org/apache/tez/runtime/task/TaskRunner2Callable.java | 3 +++ .../test/java/org/apache/tez/runtime/task/TestTaskExecution2.java | 3 +++ 3 files changed, 8 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/44c660a3/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 5fd3856..f14a2fa 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES TEZ-3029. Add an onError method to service plugin contexts. ALL CHANGES: + TEZ-3166. Fix a few cases where counters aren't fully updated and sent for failed tasks. TEZ-2958. Recovered TA, whose commit cannot be recovered, should move to killed state TEZ-2936. Create ATS implementation that enables support for YARN-4265 (ATS v1.5) TEZ-3148. Invalid event TA_TEZ_EVENT_UPDATE on TaskAttempt. @@ -405,6 +406,7 @@ INCOMPATIBLE CHANGES TEZ-2949. Allow duplicate dag names within session for Tez. ALL CHANGES: + TEZ-3166. Fix a few cases where counters aren't fully updated and sent for failed tasks. TEZ-2958. Recovered TA, whose commit cannot be recovered, should move to killed state TEZ-3105. TezMxBeanResourceCalculator does not work on IBM JDK 7 or 8 causing Tez failures. TEZ-2863. Container, node, and logs not available in UI for tasks that fail to launch http://git-wip-us.apache.org/repos/asf/tez/blob/44c660a3/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskRunner2Callable.java ---------------------------------------------------------------------- diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskRunner2Callable.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskRunner2Callable.java index 40d4051..ffbed8c 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskRunner2Callable.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskRunner2Callable.java @@ -72,6 +72,7 @@ public class TaskRunner2Callable extends CallableWithNdc<TaskRunner2Callable.Tas } else { LOG.info("Stopped before running the processor taskAttemptId={}", task.getTaskAttemptID()); + task.setFrameworkCounters(); return new TaskRunner2CallableResult(null); } @@ -81,6 +82,7 @@ public class TaskRunner2Callable extends CallableWithNdc<TaskRunner2Callable.Tas task.setFrameworkCounters(); } else { LOG.info("Stopped before closing the processor, taskAttemptId={}", task.getTaskAttemptID()); + task.setFrameworkCounters(); return new TaskRunner2CallableResult(null); } LOG.info("Task completed, taskAttemptId={}, askedToStop={}", task.getTaskAttemptID(), stopRequested.get()); @@ -93,6 +95,7 @@ public class TaskRunner2Callable extends CallableWithNdc<TaskRunner2Callable.Tas if (t instanceof UndeclaredThrowableException) { t = t.getCause(); } + task.setFrameworkCounters(); return new TaskRunner2CallableResult(t); } finally { // If a stop was requested. Make sure the interrupt status is set during the cleanup. http://git-wip-us.apache.org/repos/asf/tez/blob/44c660a3/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution2.java ---------------------------------------------------------------------- diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution2.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution2.java index 7843754..18660f6 100644 --- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution2.java +++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution2.java @@ -225,6 +225,7 @@ public class TestTaskExecution2 { TezException.class.getName() + ": " + TezException.class.getSimpleName()); // Failure detected as a result of fall off from the run method. abort isn't required. assertFalse(TestProcessor.wasAborted()); + assertTrue(taskRunner.task.getCounters().countCounters() != 0); } finally { executor.shutdownNow(); } @@ -261,6 +262,7 @@ public class TestTaskExecution2 { + "Unable to load class: NotExitedProcessor"); // Failure detected as a result of fall off from the run method. abort isn't required. assertFalse(TestProcessor.wasAborted()); + assertTrue(taskRunner.task.getCounters().countCounters() != 0); } finally { executor.shutdownNow(); } @@ -298,6 +300,7 @@ public class TestTaskExecution2 { IOException.class.getName() + ": " + IOException.class.getSimpleName()); // Failure detected as a result of fall off from the run method. abort isn't required. assertFalse(TestProcessor.wasAborted()); + assertTrue(taskRunner.task.getCounters().countCounters() != 0); } finally { executor.shutdownNow(); }
