This is an automated email from the ASF dual-hosted git repository.
jbrennan pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new c789e94 YARN-10451. RM (v1) UI NodesPage can NPE when yarn.io/gpu
resource type is defined. Contributed by Eric Payne
c789e94 is described below
commit c789e944b7166e1430c4b5c8d6def87515d7f46d
Author: Jim Brennan <jbren...@apache.org>
AuthorDate: Tue Oct 6 18:18:08 2020 +0000
YARN-10451. RM (v1) UI NodesPage can NPE when yarn.io/gpu resource type is
defined. Contributed by Eric Payne
(cherry picked from commit b361f29dda08fa9b4e406d07945d188a914bc437)
---
.../server/resourcemanager/webapp/NodesPage.java | 3 ++-
.../resourcemanager/webapp/TestNodesPage.java | 27 ++++++++++++++++++++--
.../resourcemanager/webapp/TestRMWebApp.java | 23 ++++++++++++++++--
3 files changed, 48 insertions(+), 5 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
index 446a810..169cfea 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
@@ -175,7 +175,8 @@ class NodesPage extends RmView {
.get(ResourceInformation.GPU_URI);
long usedGPUs = 0;
long availableGPUs = 0;
- if (gpuIndex != null) {
+ if (gpuIndex != null && info.getUsedResource() != null
+ && info.getAvailableResource() != null) {
usedGPUs = info.getUsedResource().getResource()
.getResourceValue(ResourceInformation.GPU_URI);
availableGPUs = info.getAvailableResource().getResource()
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
index 1f6a8c0..00ca697 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
@@ -19,12 +19,16 @@ package
org.apache.hadoop.yarn.server.resourcemanager.webapp;
import java.io.IOException;
import java.io.PrintWriter;
+import java.util.Map;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import
org.apache.hadoop.yarn.server.resourcemanager.webapp.NodesPage.NodesBlock;
+import
org.apache.hadoop.yarn.util.resource.CustomResourceTypesConfigurationProvider;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.webapp.test.WebAppTests;
import org.junit.Before;
import org.junit.Test;
@@ -56,6 +60,10 @@ public class TestNodesPage {
@Before
public void setUp() throws Exception {
+ setUpInternal(false);
+ }
+
+ private void setUpInternal(final boolean useDRC) throws Exception {
final RMContext mockRMContext =
TestRMWebApp.mockRMContext(3, numberOfRacks, numberOfNodesPerRack,
8 * TestRMWebApp.GiB);
@@ -66,7 +74,7 @@ public class TestNodesPage {
public void configure(Binder binder) {
try {
binder.bind(ResourceManager.class).toInstance(
- TestRMWebApp.mockRm(mockRMContext));
+ TestRMWebApp.mockRm(mockRMContext, useDRC));
} catch (IOException e) {
throw new IllegalStateException(e);
}
@@ -101,7 +109,22 @@ public class TestNodesPage {
Mockito.verify(writer, Mockito.times(numberOfThInMetricsTable))
.print("<td");
}
-
+
+ @Test
+ public void testNodesBlockRenderForLostNodesWithGPUResources()
+ throws Exception {
+ Map<String, ResourceInformation> oldRtMap =
+ ResourceUtils.getResourceTypes();
+ CustomResourceTypesConfigurationProvider.
+ initResourceTypes(ResourceInformation.GPU_URI);
+ this.setUpInternal(true);
+ try {
+ this.testNodesBlockRenderForLostNodes();
+ } finally {
+ ResourceUtils.initializeResourcesFromResourceInformationMap(oldRtMap);
+ }
+ }
+
@Test
public void testNodesBlockRenderForNodeLabelFilterWithNonEmptyLabel() {
NodesBlock nodesBlock = injector.getInstance(NodesBlock.class);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
index 1a4d6e3..0a83f06 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
@@ -247,8 +247,13 @@ public class TestRMWebApp {
}
public static ResourceManager mockRm(RMContext rmContext) throws IOException
{
+ return mockRm(rmContext, false);
+ }
+
+ public static ResourceManager mockRm(RMContext rmContext,
+ boolean useDRC) throws IOException {
ResourceManager rm = mock(ResourceManager.class);
- ResourceScheduler rs = mockCapacityScheduler();
+ ResourceScheduler rs = mockCapacityScheduler(useDRC);
ApplicationACLsManager aclMgr = mockAppACLsManager();
ClientRMService clientRMService = mockClientRMService(rmContext);
when(rm.getResourceScheduler()).thenReturn(rs);
@@ -259,9 +264,14 @@ public class TestRMWebApp {
}
public static CapacityScheduler mockCapacityScheduler() throws IOException {
+ return mockCapacityScheduler(false);
+ }
+
+ public static CapacityScheduler mockCapacityScheduler(boolean useDRC)
+ throws IOException {
// stolen from TestCapacityScheduler
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
- setupQueueConfiguration(conf);
+ setupQueueConfiguration(conf, useDRC);
CapacityScheduler cs = new CapacityScheduler();
YarnConfiguration yarnConf = new YarnConfiguration();
@@ -313,6 +323,11 @@ public class TestRMWebApp {
static void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
+ setupQueueConfiguration(conf, false);
+ }
+
+ static void setupQueueConfiguration(CapacitySchedulerConfiguration conf,
+ boolean useDRC) {
// Define top-level queues
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a",
"b", "c"});
@@ -358,6 +373,10 @@ public class TestRMWebApp {
conf.setCapacity(C11, 15);
conf.setCapacity(C12, 45);
conf.setCapacity(C13, 40);
+ if (useDRC) {
+ conf.set("yarn.scheduler.capacity.resource-calculator",
+ "org.apache.hadoop.yarn.util.resource.DominantResourceCalculator");
+ }
}
public static ResourceManager mockFifoRm(int apps, int racks, int nodes,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
