Repository: tez Updated Branches: refs/heads/branch-0.7 49a38f012 -> 2aa00bf91
TEZ-3335. DAG client thinks app is still running when app status is null (jeagles) (cherry picked from commit 539b8e710943b0b79362afc61e246bb83e862ac5) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/2aa00bf9 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/2aa00bf9 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/2aa00bf9 Branch: refs/heads/branch-0.7 Commit: 2aa00bf91c5e0f6b8b3c47028669fb2de3d219e1 Parents: 49a38f0 Author: Jonathan Eagles <[email protected]> Authored: Fri Oct 7 16:53:22 2016 -0500 Committer: Jonathan Eagles <[email protected]> Committed: Fri Oct 7 17:01:57 2016 -0500 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../org/apache/tez/client/TezYarnClient.java | 12 +++++++++- .../org/apache/tez/client/TestTezClient.java | 25 ++++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/2aa00bf9/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 8ab6833..8e3ffa9 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3335. DAG client thinks app is still running when app status is null TEZ-3460. Fix precommit release audit warning. TEZ-3368. NPE in DelayedContainerManager TEZ-3440. Shuffling to memory can get out-of-sync when fetching multiple compressed map outputs http://git-wip-us.apache.org/repos/asf/tez/blob/2aa00bf9/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java b/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java index 241e15c..3ac82ac 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java @@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.TezException; @@ -86,6 +87,15 @@ public class TezYarnClient extends FrameworkClient { @Override public ApplicationReport getApplicationReport(ApplicationId appId) throws YarnException, IOException { - return yarnClient.getApplicationReport(appId); + ApplicationReport report = yarnClient.getApplicationReport(appId); + if (report.getYarnApplicationState() == null) { + // The state can be null when the ResourceManager does not know about the app but the YARN + // application history server has an incomplete entry for it. Treat this scenario as if the + // application does not exist, since the final app status cannot be determined. This also + // matches the behavior for this scenario if the history server was not configured. + throw new ApplicationNotFoundException("YARN reports no state for application " + + appId); + } + return report; } } http://git-wip-us.apache.org/repos/asf/tez/blob/2aa00bf9/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java ---------------------------------------------------------------------- diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java index 08bb156..a052850 100644 --- a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java +++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java @@ -37,9 +37,11 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.google.protobuf.ServiceException; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; @@ -48,6 +50,8 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.tez.common.counters.LimitExceededException; import org.apache.tez.common.counters.Limits; @@ -131,6 +135,8 @@ public class TestTezClient { ApplicationId appId1 = ApplicationId.newInstance(0, 1); YarnClient yarnClient = mock(YarnClient.class, RETURNS_DEEP_STUBS); when(yarnClient.createApplication().getNewApplicationResponse().getApplicationId()).thenReturn(appId1); + when(yarnClient.getApplicationReport(appId1).getYarnApplicationState()).thenReturn(YarnApplicationState.NEW); + when(yarnClient.submitApplication(any(ApplicationSubmissionContext.class))).thenReturn(appId1); DAGClientAMProtocolBlockingPB sessionAmProxy = mock(DAGClientAMProtocolBlockingPB.class, RETURNS_DEEP_STUBS); when(sessionAmProxy.getAMStatus(any(RpcController.class), any(GetAMStatusRequestProto.class))) @@ -538,4 +544,23 @@ public class TestTezClient { } client.stop(); } + + @Test(timeout = 10000) + public void testMissingYarnAppStatus() throws Exception { + // verify an app not found exception is thrown when YARN reports a null app status + ApplicationId appId1 = ApplicationId.newInstance(0, 1); + ApplicationReport mockReport = mock(ApplicationReport.class); + when(mockReport.getApplicationId()).thenReturn(appId1); + when(mockReport.getYarnApplicationState()).thenReturn(null); + YarnClient yarnClient = mock(YarnClient.class, RETURNS_DEEP_STUBS); + when(yarnClient.createApplication().getNewApplicationResponse().getApplicationId()).thenReturn(appId1); + when(yarnClient.getApplicationReport(appId1)).thenReturn(mockReport); + TezYarnClient tezClient = new TezYarnClient(yarnClient); + tezClient.init(new TezConfiguration(false), new YarnConfiguration()); + try { + tezClient.getApplicationReport(appId1); + fail("getApplicationReport should have thrown"); + } catch (ApplicationNotFoundException e) { + } + } }
