Repository: hadoop Updated Branches: refs/heads/branch-2.6.1 f83d89894 -> f307b426f
YARN-2414. RM web UI: app page will crash if app is failed before any attempt has been created. Contributed by Wangda Tan (cherry picked from commit 81c9d17af84ed87b9ded7057cb726a3855ddd32d) (cherry picked from commit 242fd0e39ad1c5d51719cd0f6c197166066e3288) (cherry picked from commit a9d5acd898b34e1050a78f2d70ed62fdb82948a6) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f307b426 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f307b426 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f307b426 Branch: refs/heads/branch-2.6.1 Commit: f307b426f35f708e8ea1f92d4a772c99855192b6 Parents: f83d898 Author: Jason Lowe <[email protected]> Authored: Mon Nov 17 21:13:22 2014 +0000 Committer: Vinod Kumar Vavilapalli <[email protected]> Committed: Thu Aug 27 18:36:50 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 + .../server/security/ApplicationACLsManager.java | 7 ++ .../security/QueueACLsManager.java | 7 ++ .../server/resourcemanager/webapp/AppBlock.java | 30 +++++-- .../resourcemanager/webapp/TestAppPage.java | 90 ++++++++++++++++++++ 5 files changed, 130 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f307b426/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 5e8e4f9..81a4347 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -18,6 +18,9 @@ Release 2.6.1 - UNRELEASED YARN-2816. NM fail to start with NPE during container recovery (Zhihai Xu via jlowe) + YARN-2414. RM web UI: app page will crash if app is failed before any + attempt has been created (Wangda Tan via jlowe) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/f307b426/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java index e8e3cb5..4cacfca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java @@ -35,6 +35,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.security.AdminACLsManager; +import com.google.common.annotations.VisibleForTesting; + @InterfaceAudience.Private public class ApplicationACLsManager { @@ -48,6 +50,11 @@ public class ApplicationACLsManager { private final ConcurrentMap<ApplicationId, Map<ApplicationAccessType, AccessControlList>> applicationACLS = new ConcurrentHashMap<ApplicationId, Map<ApplicationAccessType, AccessControlList>>(); + @VisibleForTesting + public ApplicationACLsManager() { + this(new Configuration()); + } + public ApplicationACLsManager(Configuration conf) { this.conf = conf; this.adminAclsManager = new AdminACLsManager(this.conf); http://git-wip-us.apache.org/repos/asf/hadoop/blob/f307b426/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java index eb5037a..fb8279d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java @@ -24,9 +24,16 @@ import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import com.google.common.annotations.VisibleForTesting; + public class QueueACLsManager { private ResourceScheduler scheduler; private boolean isACLsEnable; + + @VisibleForTesting + public QueueACLsManager() { + this(null, new Configuration()); + } public QueueACLsManager(ResourceScheduler scheduler, Configuration conf) { this.scheduler = scheduler; http://git-wip-us.apache.org/repos/asf/hadoop/blob/f307b426/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java index c427ccf..a108e43 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java @@ -33,6 +33,7 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -45,6 +46,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.util.Times; +import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; @@ -113,8 +115,23 @@ public class AppBlock extends HtmlBlock { setTitle(join("Application ", aid)); RMAppMetrics appMerics = rmApp.getRMAppMetrics(); - RMAppAttemptMetrics attemptMetrics = - rmApp.getCurrentAppAttempt().getRMAppAttemptMetrics(); + + // Get attempt metrics and fields, it is possible currentAttempt of RMApp is + // null. In that case, we will assume resource preempted and number of Non + // AM container preempted on that attempt is 0 + RMAppAttemptMetrics attemptMetrics; + if (null == rmApp.getCurrentAppAttempt()) { + attemptMetrics = null; + } else { + attemptMetrics = rmApp.getCurrentAppAttempt().getRMAppAttemptMetrics(); + } + Resource attemptResourcePreempted = + attemptMetrics == null ? Resources.none() : attemptMetrics + .getResourcePreempted(); + int attemptNumNonAMContainerPreempted = + attemptMetrics == null ? 0 : attemptMetrics + .getNumNonAMContainersPreempted(); + info("Application Overview") ._("User:", app.getUser()) ._("Name:", app.getName()) @@ -143,13 +160,12 @@ public class AppBlock extends HtmlBlock { ._("Total Number of AM Containers Preempted:", String.valueOf(appMerics.getNumAMContainersPreempted())) ._("Resource Preempted from Current Attempt:", - attemptMetrics.getResourcePreempted()) + attemptResourcePreempted) ._("Number of Non-AM Containers Preempted from Current Attempt:", - String.valueOf(attemptMetrics - .getNumNonAMContainersPreempted())) + attemptNumNonAMContainerPreempted) ._("Aggregate Resource Allocation:", - String.format("%d MB-seconds, %d vcore-seconds", - appMerics.getMemorySeconds(), appMerics.getVcoreSeconds())); + String.format("%d MB-seconds, %d vcore-seconds", + appMerics.getMemorySeconds(), appMerics.getVcoreSeconds())); pdiv._(); Collection<RMAppAttempt> attempts = rmApp.getAppAttempts().values(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/f307b426/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java new file mode 100644 index 0000000..9732c19 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; + +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; +import org.apache.hadoop.yarn.webapp.YarnWebParams; +import org.apache.hadoop.yarn.webapp.test.WebAppTests; +import org.junit.Test; + +import com.google.inject.Binder; +import com.google.inject.Injector; +import com.google.inject.Module; + +public class TestAppPage { + @Test + public void testAppBlockRenderWithNullCurrentAppAttempt() throws Exception { + final ApplicationId APP_ID = ApplicationId.newInstance(1234L, 0); + Injector injector; + + // init app + RMApp app = mock(RMApp.class); + when(app.getTrackingUrl()).thenReturn("http://host:123";); + when(app.getState()).thenReturn(RMAppState.FAILED); + when(app.getApplicationId()).thenReturn(APP_ID); + when(app.getApplicationType()).thenReturn("Type"); + when(app.getUser()).thenReturn("user"); + when(app.getName()).thenReturn("Name"); + when(app.getQueue()).thenReturn("queue"); + when(app.getDiagnostics()).thenReturn(new StringBuilder()); + when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.FAILED); + when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.FAILED); + when(app.getStartTime()).thenReturn(0L); + when(app.getFinishTime()).thenReturn(0L); + when(app.createApplicationState()).thenReturn(YarnApplicationState.FAILED); + + RMAppMetrics appMetrics = new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0); + when(app.getRMAppMetrics()).thenReturn(appMetrics); + + // initialize RM Context, and create RMApp, without creating RMAppAttempt + final RMContext rmContext = TestRMWebApp.mockRMContext(15, 1, 2, 8); + rmContext.getRMApps().put(APP_ID, app); + + injector = + WebAppTests.createMockInjector(RMContext.class, rmContext, + new Module() { + @Override + public void configure(Binder binder) { + try { + binder.bind(ResourceManager.class).toInstance( + TestRMWebApp.mockRm(rmContext)); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + }); + + AppBlock instance = injector.getInstance(AppBlock.class); + instance.set(YarnWebParams.APPLICATION_ID, APP_ID.toString()); + instance.render(); + } +}
