YARN-7330. Add support to show GPU in UI including metrics. Contributed by Wangda Tan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aab43959 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aab43959 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aab43959 Branch: refs/heads/YARN-5881 Commit: aab439593b5d3459140b6e104722d4583dfcfddb Parents: 4cc9479 Author: Sunil G <sun...@apache.org> Authored: Thu Nov 23 07:54:20 2017 +0530 Committer: Sunil G <sun...@apache.org> Committed: Thu Nov 23 07:54:20 2017 +0530 ---------------------------------------------------------------------- .../dev-support/findbugs-exclude.xml | 8 + .../hadoop/yarn/api/records/Resource.java | 21 +++ .../resources/gpu/GpuResourceAllocator.java | 19 +- .../resources/gpu/GpuResourceHandlerImpl.java | 1 - .../resourceplugin/ResourcePlugin.java | 11 ++ .../resourceplugin/gpu/AssignedGpuDevice.java | 79 ++++++++ .../resourceplugin/gpu/GpuDevice.java | 4 +- .../resourceplugin/gpu/GpuResourcePlugin.java | 24 ++- .../nodemanager/webapp/NMWebServices.java | 28 +++ .../nodemanager/webapp/dao/NMResourceInfo.java | 28 +++ .../webapp/dao/gpu/GpuDeviceInformation.java | 2 +- .../webapp/dao/gpu/NMGpuResourceInfo.java | 71 +++++++ .../webapp/dao/gpu/PerGpuDeviceInformation.java | 2 +- .../webapp/dao/gpu/PerGpuMemoryUsage.java | 2 +- .../resources/gpu/TestGpuResourceHandler.java | 6 +- .../nodemanager/webapp/TestNMWebServices.java | 185 +++++++++++++++---- .../dao/gpu/TestGpuDeviceInformationParser.java | 2 +- .../src/main/webapp/app/adapters/yarn-nm-gpu.js | 33 ++++ .../main/webapp/app/components/donut-chart.js | 18 +- .../webapp/app/components/gpu-donut-chart.js | 66 +++++++ .../src/main/webapp/app/constants.js | 13 ++ .../webapp/app/controllers/yarn-nodes/table.js | 2 +- .../main/webapp/app/models/cluster-metric.js | 69 +++++++ .../src/main/webapp/app/models/yarn-nm-gpu.js | 27 +++ .../app/models/yarn-queue/capacity-queue.js | 3 +- .../src/main/webapp/app/models/yarn-rm-node.js | 35 ++++ .../src/main/webapp/app/router.js | 5 +- .../main/webapp/app/routes/cluster-overview.js | 2 +- .../src/main/webapp/app/routes/yarn-node.js | 2 + .../webapp/app/routes/yarn-node/yarn-nm-gpu.js | 22 +++ .../main/webapp/app/serializers/yarn-nm-gpu.js | 43 +++++ .../serializers/yarn-queue/capacity-queue.js | 1 + .../main/webapp/app/serializers/yarn-rm-node.js | 4 +- .../webapp/app/templates/cluster-overview.hbs | 88 ++++++--- .../templates/components/node-menu-panel.hbs | 10 +- .../templates/components/yarn-nm-gpu-info.hbs | 69 +++++++ .../src/main/webapp/app/templates/yarn-node.hbs | 125 ------------- .../webapp/app/templates/yarn-node/info.hbs | 154 +++++++++++++++ .../app/templates/yarn-node/yarn-nm-gpu.hbs | 53 ++++++ .../src/main/webapp/app/utils/converter.js | 51 +++++ 40 files changed, 1181 insertions(+), 207 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index 670b7ee..de4b0e6 100644 --- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -643,4 +643,12 @@ <Method name="serveNIOTCP" /> <Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE" /> </Match> + + <!-- EQ_OVERRIDING_EQUALS_NOT_SYMMETRIC --> + <Match> + <Class name="org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.AssignedGpuDevice" /> + <Method name="equals" /> + <Bug pattern="EQ_OVERRIDING_EQUALS_NOT_SYMMETRIC" /> + </Match> + </FindBugsFilter> http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index 65b5dce..abd44b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -18,9 +18,14 @@ package org.apache.hadoop.yarn.api.records; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; +import com.google.common.collect.Lists; +import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.NotImplementedException; +import org.apache.curator.shaded.com.google.common.reflect.ClassPath; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; @@ -209,6 +214,22 @@ public abstract class Resource implements Comparable<Resource> { } /** + * Get list of resource information, this will be used by JAXB. + * @return list of resources copy. + */ + @InterfaceAudience.Private + @InterfaceStability.Unstable + public List<ResourceInformation> getAllResourcesListCopy() { + List<ResourceInformation> list = new ArrayList<>(); + for (ResourceInformation i : resources) { + ResourceInformation ri = new ResourceInformation(); + ResourceInformation.copy(i, ri); + list.add(ri); + } + return list; + } + + /** * Get ResourceInformation for a specified resource. * * @param resource name of the resource http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java index f2bb342..5bdffc3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java @@ -30,11 +30,13 @@ import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.AssignedGpuDevice; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDevice; import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -224,7 +226,20 @@ public class GpuResourceAllocator { } @VisibleForTesting - public synchronized Map<GpuDevice, ContainerId> getDeviceAllocationMapping() { - return new HashMap<>(usedDevices); + public synchronized Map<GpuDevice, ContainerId> getDeviceAllocationMappingCopy() { + return new HashMap<>(usedDevices); + } + + public synchronized List<GpuDevice> getAllowedGpusCopy() { + return new ArrayList<>(allowedGpuDevices); + } + + public synchronized List<AssignedGpuDevice> getAssignedGpusCopy() { + List<AssignedGpuDevice> assigns = new ArrayList<>(); + for (Map.Entry<GpuDevice, ContainerId> entry : usedDevices.entrySet()) { + assigns.add(new AssignedGpuDevice(entry.getKey().getIndex(), + entry.getKey().getMinorNumber(), entry.getValue())); + } + return assigns; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java index 2dbd3e3..8ddc227 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java @@ -153,7 +153,6 @@ public class GpuResourceHandlerImpl implements ResourceHandler { return null; } - @VisibleForTesting public GpuResourceAllocator getGpuAllocator() { return gpuAllocator; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePlugin.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePlugin.java index 99a18ed..f141179 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePlugin.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePlugin.java @@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resource import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerChain; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime; +import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo; /** * {@link ResourcePlugin} is an interface for node manager to easier support @@ -91,4 +92,14 @@ public interface ResourcePlugin { * have requirement to update docker command. */ DockerCommandPlugin getDockerCommandPluginInstance(); + + /** + * Get resource information from this plugin. + * + * @return NMResourceInfo, an example is + * {@link org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.GpuDeviceInformation} + * + * @throws YarnException when any issue occurs + */ + NMResourceInfo getNMResourceInfo() throws YarnException; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/AssignedGpuDevice.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/AssignedGpuDevice.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/AssignedGpuDevice.java new file mode 100644 index 0000000..26fd905 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/AssignedGpuDevice.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu; + +import org.apache.hadoop.yarn.api.records.ContainerId; + +/** + * In addition to {@link GpuDevice}, this include container id and more runtime + * information related to who is using the GPU device if possible + */ +public class AssignedGpuDevice extends GpuDevice { + private static final long serialVersionUID = -12983712986315L; + + String containerId; + + public AssignedGpuDevice(int index, int minorNumber, + ContainerId containerId) { + super(index, minorNumber); + this.containerId = containerId.toString(); + } + + public String getContainerId() { + return containerId; + } + + public void setContainerId(String containerId) { + this.containerId = containerId; + } + + @Override + public boolean equals(Object obj) { + if (obj == null || !(obj instanceof AssignedGpuDevice)) { + return false; + } + AssignedGpuDevice other = (AssignedGpuDevice) obj; + return index == other.index && minorNumber == other.minorNumber + && containerId.equals(other.containerId); + } + + @Override + public int compareTo(Object obj) { + if (obj == null || (!(obj instanceof AssignedGpuDevice))) { + return -1; + } + + AssignedGpuDevice other = (AssignedGpuDevice) obj; + + int result = Integer.compare(index, other.index); + if (0 != result) { + return result; + } + result = Integer.compare(minorNumber, other.minorNumber); + if (0 != result) { + return result; + } + return containerId.compareTo(other.containerId); + } + + @Override + public int hashCode() { + final int prime = 47; + return prime * (prime * index + minorNumber) + containerId.hashCode(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDevice.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDevice.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDevice.java index 8119924..bce1d9f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDevice.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDevice.java @@ -24,8 +24,8 @@ import java.io.Serializable; * This class is used to represent GPU device while allocation. */ public class GpuDevice implements Serializable, Comparable { - private int index; - private int minorNumber; + protected int index; + protected int minorNumber; private static final long serialVersionUID = -6812314470754667710L; public GpuDevice(int index, int minorNumber) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java index 4ff186f..f28218d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java @@ -18,18 +18,26 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu; +import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.gpu.GpuResourceAllocator; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.gpu.GpuResourceHandlerImpl; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.DockerCommandPlugin; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.NodeResourceUpdaterPlugin; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin; +import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo; +import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.GpuDeviceInformation; +import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.NMGpuResourceInfo; + +import java.util.List; +import java.util.Map; public class GpuResourcePlugin implements ResourcePlugin { - private ResourceHandler gpuResourceHandler = null; + private GpuResourceHandlerImpl gpuResourceHandler = null; private GpuNodeResourceUpdateHandler resourceDiscoverHandler = null; private DockerCommandPlugin dockerCommandPlugin = null; @@ -67,4 +75,18 @@ public class GpuResourcePlugin implements ResourcePlugin { public DockerCommandPlugin getDockerCommandPluginInstance() { return dockerCommandPlugin; } + + @Override + public NMResourceInfo getNMResourceInfo() throws YarnException { + GpuDeviceInformation gpuDeviceInformation = + GpuDiscoverer.getInstance().getGpuDeviceInformation(); + GpuResourceAllocator gpuResourceAllocator = + gpuResourceHandler.getGpuAllocator(); + List<GpuDevice> totalGpus = gpuResourceAllocator.getAllowedGpusCopy(); + List<AssignedGpuDevice> assignedGpuDevices = + gpuResourceAllocator.getAssignedGpusCopy(); + + return new NMGpuResourceInfo(gpuDeviceInformation, totalGpus, + assignedGpuDevices); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index c5379cc..937a8cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -27,6 +27,10 @@ import java.util.HashSet; import java.util.List; import java.util.Map.Entry; import java.util.Set; + +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager; +import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -496,6 +500,30 @@ public class NMWebServices { } } + @GET + @Path("/resources/{resourcename}") + @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + public Object getNMResourceInfo( + @PathParam("resourcename") + String resourceName) throws YarnException { + init(); + ResourcePluginManager rpm = this.nmContext.getResourcePluginManager(); + if (rpm != null && rpm.getNameToPlugins() != null) { + ResourcePlugin plugin = rpm.getNameToPlugins().get(resourceName); + if (plugin != null) { + NMResourceInfo nmResourceInfo = plugin.getNMResourceInfo(); + if (nmResourceInfo != null) { + return nmResourceInfo; + } + } + } + + throw new YarnException( + "Could not get detailed resource information for given resource-name=" + + resourceName); + } + private long parseLongParam(String bytes) { if (bytes == null || bytes.isEmpty()) { return Long.MAX_VALUE; http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NMResourceInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NMResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NMResourceInfo.java new file mode 100644 index 0000000..18ce8ea --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NMResourceInfo.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class NMResourceInfo { +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformation.java index 977032a..837d5cc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformation.java @@ -25,7 +25,7 @@ import javax.xml.bind.annotation.XmlRootElement; import java.util.List; /** - * All GPU Device Information in the system. + * All GPU Device Information in the system, fetched from nvidia-smi. */ @InterfaceAudience.Private @InterfaceStability.Unstable http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/NMGpuResourceInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/NMGpuResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/NMGpuResourceInfo.java new file mode 100644 index 0000000..bf1d463 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/NMGpuResourceInfo.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu; + +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.AssignedGpuDevice; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDevice; +import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo; + +import java.util.List; + +/** + * Gpu device information return to client when + * {@link org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices#getNMResourceInfo(String)} + * is invoked. + */ +public class NMGpuResourceInfo extends NMResourceInfo { + GpuDeviceInformation gpuDeviceInformation; + + List<GpuDevice> totalGpuDevices; + List<AssignedGpuDevice> assignedGpuDevices; + + public NMGpuResourceInfo(GpuDeviceInformation gpuDeviceInformation, + List<GpuDevice> totalGpuDevices, + List<AssignedGpuDevice> assignedGpuDevices) { + this.gpuDeviceInformation = gpuDeviceInformation; + this.totalGpuDevices = totalGpuDevices; + this.assignedGpuDevices = assignedGpuDevices; + } + + public GpuDeviceInformation getGpuDeviceInformation() { + return gpuDeviceInformation; + } + + public void setGpuDeviceInformation( + GpuDeviceInformation gpuDeviceInformation) { + this.gpuDeviceInformation = gpuDeviceInformation; + } + + public List<GpuDevice> getTotalGpuDevices() { + return totalGpuDevices; + } + + public void setTotalGpuDevices(List<GpuDevice> totalGpuDevices) { + this.totalGpuDevices = totalGpuDevices; + } + + public List<AssignedGpuDevice> getAssignedGpuDevices() { + return assignedGpuDevices; + } + + public void setAssignedGpuDevices( + List<AssignedGpuDevice> assignedGpuDevices) { + this.assignedGpuDevices = assignedGpuDevices; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuDeviceInformation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuDeviceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuDeviceInformation.java index f315313..25c2e3a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuDeviceInformation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuDeviceInformation.java @@ -135,7 +135,7 @@ public class PerGpuDeviceInformation { this.gpuUtilizations = utilizations; } - @XmlElement(name = "bar1_memory_usage") + @XmlElement(name = "fb_memory_usage") public PerGpuMemoryUsage getGpuMemoryUsage() { return gpuMemoryUsage; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuMemoryUsage.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuMemoryUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuMemoryUsage.java index 3964c4e..afc1a96 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuMemoryUsage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuMemoryUsage.java @@ -27,7 +27,7 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; @InterfaceAudience.Private @InterfaceStability.Unstable -@XmlRootElement(name = "bar1_memory_usage") +@XmlRootElement(name = "fb_memory_usage") public class PerGpuMemoryUsage { long usedMemoryMiB = -1L; long availMemoryMiB = -1L; http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java index 1e0eb7b..87ab249 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java @@ -368,7 +368,7 @@ public class TestGpuResourceHandler { gpuResourceHandler.reacquireContainer(getContainerId(1)); Map<GpuDevice, ContainerId> deviceAllocationMapping = - gpuResourceHandler.getGpuAllocator().getDeviceAllocationMapping(); + gpuResourceHandler.getGpuAllocator().getDeviceAllocationMappingCopy(); Assert.assertEquals(2, deviceAllocationMapping.size()); Assert.assertTrue( deviceAllocationMapping.keySet().contains(new GpuDevice(1, 1))); @@ -402,7 +402,7 @@ public class TestGpuResourceHandler { // Make sure internal state not changed. deviceAllocationMapping = - gpuResourceHandler.getGpuAllocator().getDeviceAllocationMapping(); + gpuResourceHandler.getGpuAllocator().getDeviceAllocationMappingCopy(); Assert.assertEquals(2, deviceAllocationMapping.size()); Assert.assertTrue(deviceAllocationMapping.keySet() .containsAll(Arrays.asList(new GpuDevice(1, 1), new GpuDevice(2, 3)))); @@ -434,7 +434,7 @@ public class TestGpuResourceHandler { // Make sure internal state not changed. deviceAllocationMapping = - gpuResourceHandler.getGpuAllocator().getDeviceAllocationMapping(); + gpuResourceHandler.getGpuAllocator().getDeviceAllocationMappingCopy(); Assert.assertEquals(2, deviceAllocationMapping.size()); Assert.assertTrue(deviceAllocationMapping.keySet() .containsAll(Arrays.asList(new GpuDevice(1, 1), new GpuDevice(2, 3)))); http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java index fbab34a..171834e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java @@ -18,31 +18,20 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp; -import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseStatusCode; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.File; -import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringReader; -import java.net.HttpURLConnection; -import java.net.URI; -import java.net.URL; -import java.util.List; -import javax.servlet.http.HttpServletResponse; -import javax.ws.rs.core.MediaType; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; - -import org.apache.hadoop.http.JettyUtils; -import org.apache.hadoop.yarn.webapp.GuiceServletConfig; +import com.google.inject.Guice; +import com.google.inject.servlet.ServletModule; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.ClientResponse.Status; +import com.sun.jersey.api.client.GenericType; +import com.sun.jersey.api.client.UniformInterfaceException; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import com.sun.jersey.test.framework.WebAppDescriptor; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.http.JettyUtils; import org.apache.hadoop.util.VersionInfo; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -50,6 +39,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType; import org.apache.hadoop.yarn.logaggregation.ContainerLogFileInfo; import org.apache.hadoop.yarn.logaggregation.TestContainerLogsUtils; @@ -61,13 +51,22 @@ import org.apache.hadoop.yarn.server.nodemanager.ResourceView; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationImpl; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState; import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.AssignedGpuDevice; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDevice; import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer.NMWebApp; +import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo; +import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.GpuDeviceInformation; +import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.NMGpuResourceInfo; +import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.PerGpuDeviceInformation; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.server.webapp.YarnWebServiceParams; import org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo; import org.apache.hadoop.yarn.util.YarnVersionInfo; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; +import org.apache.hadoop.yarn.webapp.GuiceServletConfig; import org.apache.hadoop.yarn.webapp.JerseyTestBase; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; @@ -83,22 +82,36 @@ import org.w3c.dom.Element; import org.w3c.dom.NodeList; import org.xml.sax.InputSource; -import com.google.inject.Guice; -import com.google.inject.servlet.ServletModule; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.ClientResponse.Status; -import com.sun.jersey.api.client.GenericType; -import com.sun.jersey.api.client.UniformInterfaceException; -import com.sun.jersey.api.client.WebResource; -import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; -import com.sun.jersey.test.framework.WebAppDescriptor; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.core.MediaType; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringReader; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URL; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseStatusCode; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; /** * Test the nodemanager node info web services api's */ public class TestNMWebServices extends JerseyTestBase { - private static Context nmContext; + private static NodeManager.NMContext nmContext; private static ResourceView resourceView; private static ApplicationACLsManager aclsManager; private static LocalDirsHandlerService dirsHandler; @@ -418,6 +431,116 @@ public class TestNMWebServices extends JerseyTestBase { assertFalse(redirectURL.contains(YarnWebServiceParams.NM_ID)); } + @Test + public void testGetNMResourceInfo() + throws YarnException, InterruptedException, JSONException { + ResourcePluginManager rpm = mock(ResourcePluginManager.class); + Map<String, ResourcePlugin> namesToPlugins = new HashMap<>(); + ResourcePlugin mockPlugin1 = mock(ResourcePlugin.class); + NMResourceInfo nmResourceInfo1 = new NMResourceInfo() { + public long a = 1000L; + }; + when(mockPlugin1.getNMResourceInfo()).thenReturn(nmResourceInfo1); + namesToPlugins.put("resource-1", mockPlugin1); + namesToPlugins.put("yarn.io/resource-1", mockPlugin1); + ResourcePlugin mockPlugin2 = mock(ResourcePlugin.class); + namesToPlugins.put("resource-2", mockPlugin2); + when(rpm.getNameToPlugins()).thenReturn(namesToPlugins); + + nmContext.setResourcePluginManager(rpm); + + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("node").path( + "resources").path("resource-2").accept(MediaType.APPLICATION_JSON).get( + ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + response.getType().toString()); + + // Access resource-2 should fail (null NMResourceInfo returned). + JSONObject json = response.getEntity(JSONObject.class); + assertIncludesException(json); + + // Access resource-3 should fail (unkown plugin) + response = r.path("ws").path("v1").path("node").path( + "resources").path("resource-3").accept(MediaType.APPLICATION_JSON).get( + ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + response.getType().toString()); + json = response.getEntity(JSONObject.class); + assertIncludesException(json); + + // Access resource-1 should success + response = r.path("ws").path("v1").path("node").path( + "resources").path("resource-1").accept(MediaType.APPLICATION_JSON).get( + ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + response.getType().toString()); + json = response.getEntity(JSONObject.class); + Assert.assertEquals(1000, json.get("a")); + + // Access resource-1 should success (encoded yarn.io/Fresource-1). + response = r.path("ws").path("v1").path("node").path("resources").path( + "yarn.io%2Fresource-1").accept(MediaType.APPLICATION_JSON).get( + ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + response.getType().toString()); + json = response.getEntity(JSONObject.class); + Assert.assertEquals(1000, json.get("a")); + } + + private ContainerId createContainerId(int id) { + ApplicationId appId = ApplicationId.newInstance(0, 0); + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(appId, 1); + ContainerId containerId = ContainerId.newContainerId(appAttemptId, id); + return containerId; + } + + @Test + public void testGetYarnGpuResourceInfo() + throws YarnException, InterruptedException, JSONException { + ResourcePluginManager rpm = mock(ResourcePluginManager.class); + Map<String, ResourcePlugin> namesToPlugins = new HashMap<>(); + ResourcePlugin mockPlugin1 = mock(ResourcePlugin.class); + GpuDeviceInformation gpuDeviceInformation = new GpuDeviceInformation(); + gpuDeviceInformation.setDriverVersion("1.2.3"); + gpuDeviceInformation.setGpus(Arrays.asList(new PerGpuDeviceInformation())); + NMResourceInfo nmResourceInfo1 = new NMGpuResourceInfo(gpuDeviceInformation, + Arrays.asList(new GpuDevice(1, 1), new GpuDevice(2, 2), + new GpuDevice(3, 3)), Arrays + .asList(new AssignedGpuDevice(2, 2, createContainerId(1)), + new AssignedGpuDevice(3, 3, createContainerId(2)))); + when(mockPlugin1.getNMResourceInfo()).thenReturn(nmResourceInfo1); + namesToPlugins.put("resource-1", mockPlugin1); + namesToPlugins.put("yarn.io/resource-1", mockPlugin1); + ResourcePlugin mockPlugin2 = mock(ResourcePlugin.class); + namesToPlugins.put("resource-2", mockPlugin2); + when(rpm.getNameToPlugins()).thenReturn(namesToPlugins); + + nmContext.setResourcePluginManager(rpm); + + WebResource r = resource(); + ClientResponse response; + JSONObject json; + + // Access resource-1 should success + response = r.path("ws").path("v1").path("node").path( + "resources").path("resource-1").accept(MediaType.APPLICATION_JSON).get( + ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + response.getType().toString()); + json = response.getEntity(JSONObject.class); + Assert.assertEquals("1.2.3", + json.getJSONObject("gpuDeviceInformation").get("driverVersion")); + Assert.assertEquals(3, json.getJSONArray("totalGpuDevices").length()); + Assert.assertEquals(2, json.getJSONArray("assignedGpuDevices").length()); + Assert.assertEquals(2, json.getJSONArray("assignedGpuDevices").length()); + } + + private void assertIncludesException(JSONObject json) { + Assert.assertTrue(json.has("RemoteException")); + } + private void testContainerLogs(WebResource r, ContainerId containerId) throws IOException { final String containerIdStr = containerId.toString(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/TestGpuDeviceInformationParser.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/TestGpuDeviceInformationParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/TestGpuDeviceInformationParser.java index e22597d..dc96746 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/TestGpuDeviceInformationParser.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/TestGpuDeviceInformationParser.java @@ -39,7 +39,7 @@ public class TestGpuDeviceInformationParser { Assert.assertEquals(2, info.getGpus().size()); PerGpuDeviceInformation gpu1 = info.getGpus().get(1); Assert.assertEquals("Tesla P100-PCIE-12GB", gpu1.getProductName()); - Assert.assertEquals(16384, gpu1.getGpuMemoryUsage().getTotalMemoryMiB()); + Assert.assertEquals(12193, gpu1.getGpuMemoryUsage().getTotalMemoryMiB()); Assert.assertEquals(10.3f, gpu1.getGpuUtilizations().getOverallGpuUtilization(), 1e-6); Assert.assertEquals(34f, gpu1.getTemperature().getCurrentGpuTemp(), 1e-6); http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-nm-gpu.js ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-nm-gpu.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-nm-gpu.js new file mode 100644 index 0000000..bf6307a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-nm-gpu.js @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import AbstractAdapter from './abstract'; + +export default AbstractAdapter.extend({ + + address: "localBaseAddress", + restNameSpace: "node", + serverName: "NM", + + urlForFindRecord(id/*, modelName, snapshot*/) { + var url = this._buildURL(); + url = url.replace("{nodeAddress}", id) + "/resources/yarn.io%2Fgpu"; + return url; + } + +}); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/donut-chart.js ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/donut-chart.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/donut-chart.js index b1e6ecf..03b6336 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/donut-chart.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/donut-chart.js @@ -20,6 +20,7 @@ import Ember from 'ember'; import BaseChartComponent from 'yarn-ui/components/base-chart-component'; import ColorUtils from 'yarn-ui/utils/color-utils'; import Converter from 'yarn-ui/utils/converter'; +import {Entities} from 'yarn-ui/constants'; export default BaseChartComponent.extend({ /* @@ -41,8 +42,10 @@ export default BaseChartComponent.extend({ } if (!middleValue) { - if (this.get("type") === "memory") { + if (this.get(Entities.Type) === Entities.Memory) { middleValue = Converter.memoryToSimpliedUnit(total); + } else if (this.get(Entities.Type) === Entities.Resource) { + middleValue = Converter.resourceToSimplifiedUnit(total, this.get(Entities.Unit)); } else { middleValue = total; } @@ -151,7 +154,10 @@ export default BaseChartComponent.extend({ var value = d.value; if (this.get("type") === "memory") { value = Converter.memoryToSimpliedUnit(value); + } else if (this.get("type") === "resource") { + value = Converter.resourceToSimplifiedUnit(value, this.get(Entities.Unit)); } + return d.label + ' = ' + value + suffix; }.bind(this)); } @@ -185,10 +191,18 @@ export default BaseChartComponent.extend({ } this.renderDonutChart(this.get("data"), this.get("title"), this.get("showLabels"), - this.get("middleLabel"), this.get("middleValue")); + this.get("middleLabel"), this.get("middleValue"), this.get("suffix")); }, didInsertElement: function() { + // When parentIdPrefix is specified, use parentidPrefix + name as new parent + // id + if (this.get("parentIdPrefix")) { + var newParentId = this.get("parentIdPrefix") + this.get("id"); + this.set("parentId", newParentId); + console.log(newParentId); + } + this.initChart(); this.draw(); }, http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/gpu-donut-chart.js ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/gpu-donut-chart.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/gpu-donut-chart.js new file mode 100644 index 0000000..fa5ca8a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/gpu-donut-chart.js @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import DonutChart from 'yarn-ui/components/donut-chart'; +import ColorUtils from 'yarn-ui/utils/color-utils'; + +export default DonutChart.extend({ + draw: function() { + // Construct data + var data = []; + if (this.get("gpu-render-type") === "gpu-memory") { + data.push({ + label: "Used", + value: parseFloat(this.get("gpuInfo").gpuMemoryUsage.usedMemoryMiB), + }); + data.push({ + label: "Available", + value: parseFloat(this.get("gpuInfo").gpuMemoryUsage.availMemoryMiB) + }); + } else if (this.get("gpu-render-type") === "gpu-utilization") { + var utilization = parseFloat(this.get("gpuInfo").gpuUtilizations.overallGpuUtilization); + data.push({ + label: "Utilized", + value: utilization, + }); + data.push({ + label: "Available", + value: 100 - utilization + }); + } + + var colorTargets = this.get("colorTargets"); + if (colorTargets) { + var colorTargetReverse = Boolean(this.get("colorTargetReverse")); + var targets = colorTargets.split(" "); + this.colors = ColorUtils.getColors(data.length, targets, colorTargetReverse); + } + + this.renderDonutChart(data, this.get("title"), this.get("showLabels"), + this.get("middleLabel"), this.get("middleValue"), this.get("suffix")); + }, + + didInsertElement: function() { + // ParentId includes minorNumber + var newParentId = this.get("parentId") + this.get("gpuInfo").minorNumber; + this.set("parentId", newParentId); + + this.initChart(); + this.draw(); + }, +}); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js index d2937a0..29ad4bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js @@ -22,3 +22,16 @@ export default { PARAM_SEPARATOR: '!', }; + +const BASE_UNIT = 1024 + +export const Type = 'type'; +export const Memory = 'memory'; +export const Resource = 'resource'; +export const Unit = 'unit'; +export const Entities = { + Type: 'type', + Memory:'memory', + Resource: 'resource', + Unit: 'unit' +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js index 30180dd..24e3878 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js @@ -67,7 +67,7 @@ export default Ember.Controller.extend({ getCellContent: function(row) { var node_id = row.get("id"), node_addr = row.get("nodeHTTPAddress"), - href = `#/yarn-node/${node_id}/${node_addr}`; + href = `#/yarn-node/${node_id}/${node_addr}/info`; switch(row.get("nodeState")) { case "SHUTDOWN": case "LOST": http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js index 0be0d83..bbc6f08 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js @@ -43,6 +43,8 @@ export default DS.Model.extend({ decommissionedNodes: DS.attr('number'), rebootedNodes: DS.attr('number'), activeNodes: DS.attr('number'), + totalUsedResourcesAcrossPartition: DS.attr('object'), + totalClusterResourcesAcrossPartition: DS.attr('object'), getFinishedAppsDataForDonutChart: function() { var arr = []; @@ -135,4 +137,71 @@ export default DS.Model.extend({ return arr; }.property("allocatedVirtualCores", "reservedVirtualCores", "availableVirtualCores"), + + getResourceTypes: function() { + var types = []; + if (this.get("totalClusterResourcesAcrossPartition")) { + + console.log(types); + } + }.property("totalClusterResourcesAcrossPartition"), + + /* + * Returned format + * [ + * { + * name: <resource-name> + * unit: <resource-unit> + * [ + * { + * label: <label> + * value: <value> + * }, + * { + * } + * ... + * ], + * } + * ] + */ + getAllResourceTypesDonutChart: function() { + if (this.get("totalClusterResourcesAcrossPartition") + && this.get("totalUsedResourcesAcrossPartition")) { + var usages = []; + + var clusterResourceInformations = this.get("totalClusterResourcesAcrossPartition").resourcesInformations; + var usedResourceInformations = this.get("totalUsedResourcesAcrossPartition").resourcesInformations; + + clusterResourceInformations.forEach(function(cluster) { + var perResourceTypeUsage = { + name: cluster.name, + unit: cluster.units, + data: [] + }; + + usedResourceInformations.forEach(function (used) { + if (used.name === perResourceTypeUsage.name) { + var usedValue = used.value; + perResourceTypeUsage.data.push({ + label: "Used", + value: usedValue + }, { + label: "Available", + value: cluster.value - usedValue + }); + } + }); + + usages.push(perResourceTypeUsage); + + // Make sure id is a valid w3c ID + perResourceTypeUsage.id = perResourceTypeUsage.name.replace('/', '-'); + perResourceTypeUsage.id = perResourceTypeUsage.id.replace('.', '-'); + }); + + console.log(usages); + return usages; + } + return null; + }.property() }); http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-nm-gpu.js ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-nm-gpu.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-nm-gpu.js new file mode 100644 index 0000000..b3e9c2a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-nm-gpu.js @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import DS from 'ember-data'; + +export default DS.Model.extend({ + info: DS.attr('object'), + + jsonString: function() { + return JSON.stringify(this.get("info")); + }.property(), +}); http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/capacity-queue.js ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/capacity-queue.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/capacity-queue.js index b84a473..1d162e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/capacity-queue.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/capacity-queue.js @@ -37,6 +37,7 @@ export default DS.Model.extend({ numActiveApplications: DS.attr('number'), users: DS.hasMany('YarnUser'), type: DS.attr('string'), + resources: DS.attr('object'), isLeafQueue: function() { var len = this.get("children.length"); @@ -93,5 +94,5 @@ export default DS.Model.extend({ value: this.get("numActiveApplications") || 0 } ]; - }.property("numPendingApplications", "numActiveApplications") + }.property("numPendingApplications", "numActiveApplications"), }); http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js index 1321074..da5ba01 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js @@ -32,6 +32,8 @@ export default DS.Model.extend({ availableVirtualCores: DS.attr('number'), version: DS.attr('string'), nodeLabels: DS.attr('array'), + availableResource: DS.attr('object'), + usedResource: DS.attr('object'), nodeLabelsAsString: function() { var labels = this.get("nodeLabels"); @@ -90,6 +92,39 @@ export default DS.Model.extend({ return arr; }.property("availableVirtualCores", "usedVirtualCores"), + getGpuDataForDonutChart: function() { + var arr = []; + var used = 0; + var ri; + + var resourceInformations = this.get("usedResource").resourcesInformations; + for (var i = 0; i < resourceInformations.length; i++) { + ri = resourceInformations[i]; + if (ri.name === "yarn.io/gpu") { + used = ri.value; + } + } + + var available = 0; + resourceInformations = this.get("availableResource").resourcesInformations; + for (i = 0; i < resourceInformations.length; i++) { + ri = resourceInformations[i]; + if (ri.name === "yarn.io/gpu") { + available = ri.value; + } + } + + arr.push({ + label: "Used", + value: used + }); + arr.push({ + label: "Available", + value: available + }); + return arr; + }.property("availableResource", "usedResource"), + toolTipText: function() { return "<p>Rack: " + this.get("rack") + '</p>' + "<p>Host: " + this.get("nodeHostName") + '</p>'; http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/router.js ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/router.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/router.js index 64e9ad2..27a3ac2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/router.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/router.js @@ -38,7 +38,10 @@ Router.map(function() { this.route('apps'); }); this.route('yarn-nodes-heatmap'); - this.route('yarn-node', { path: '/yarn-node/:node_id/:node_addr' }); + this.route('yarn-node', { path: '/yarn-node/:node_id/:node_addr' }, function() { + this.route("info"); + this.route("yarn-nm-gpu"); + }); this.route('yarn-node-apps', { path: '/yarn-node-apps/:node_id/:node_addr' }); this.route('yarn-node-app', { path: '/yarn-node-app/:node_id/:node_addr/:app_id' }); http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/cluster-overview.js ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/cluster-overview.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/cluster-overview.js index d03ea0d..254ece4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/cluster-overview.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/cluster-overview.js @@ -31,7 +31,7 @@ export default AbstractRoute.extend({ queues: this.store.query("yarn-queue.yarn-queue", {}).then((model) => { let type = model.get('firstObject').get('type'); return this.store.query("yarn-queue." + type + "-queue", {}); - }), + }) }); }, http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node.js ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node.js index 3d54846..7ce615c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node.js @@ -25,6 +25,7 @@ export default AbstractRoute.extend({ // Fetches data from both NM and RM. RM is queried to get node usage info. return Ember.RSVP.hash({ nodeInfo: { id: param.node_id, addr: param.node_addr }, + nmGpuInfo: this.store.findRecord('yarn-nm-gpu', param.node_addr, {reload:true}), node: this.store.findRecord('yarn-node', param.node_addr, {reload: true}), rmNode: this.store.findRecord('yarn-rm-node', param.node_id, {reload: true}) }); @@ -33,5 +34,6 @@ export default AbstractRoute.extend({ unloadAll() { this.store.unloadAll('yarn-node'); this.store.unloadAll('yarn-rm-node'); + this.store.unloadAll('yarn-nm-gpu'); } }); http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node/yarn-nm-gpu.js ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node/yarn-nm-gpu.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node/yarn-nm-gpu.js new file mode 100644 index 0000000..38ae5d1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node/yarn-nm-gpu.js @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Ember from 'ember'; + +export default Ember.Route.extend({ +}); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-nm-gpu.js ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-nm-gpu.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-nm-gpu.js new file mode 100644 index 0000000..3567c68 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-nm-gpu.js @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import DS from 'ember-data'; + +export default DS.JSONAPISerializer.extend({ + internalNormalizeSingleResponse(store, primaryModelClass, payload, id) { + if (payload.nodeInfo) { + payload = payload.nodeInfo; + } + + var fixedPayload = { + id: id, + type: primaryModelClass.modelName, + attributes: { + info: payload + } + }; + return fixedPayload; + }, + + normalizeSingleResponse(store, primaryModelClass, payload, id/*, requestType*/) { + // payload is of the form {"nodeInfo":{}} + var p = this.internalNormalizeSingleResponse(store, + primaryModelClass, payload, id); + return { data: p }; + }, +}); http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js index c7350ef..7626598 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js @@ -72,6 +72,7 @@ export default DS.JSONAPISerializer.extend({ preemptionDisabled: payload.preemptionDisabled, numPendingApplications: payload.numPendingApplications, numActiveApplications: payload.numActiveApplications, + resources: payload.resources, type: "capacity", }, // Relationships http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js index 1c6d1be..a3a1d59 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js @@ -41,7 +41,9 @@ export default DS.JSONAPISerializer.extend({ usedVirtualCores: payload.usedVirtualCores, availableVirtualCores: payload.availableVirtualCores, version: payload.version, - nodeLabels: payload.nodeLabels + nodeLabels: payload.nodeLabels, + usedResource: payload.used, + availableResource: payload.avail } }; return fixedPayload; http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/cluster-overview.hbs ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/cluster-overview.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/cluster-overview.hbs index e549ce5..ff4682a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/cluster-overview.hbs +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/cluster-overview.hbs @@ -90,41 +90,71 @@ <hr> <div class="row"> - - <div class="col-lg-4 container-fluid"> - <div class="panel panel-default"> - <div class="panel-heading"> - Resource - Memory + <!-- When getAllResourceTypesDonutChart is not null, use it to show per-resource-type usages. Otherwise only show + vcore/memory usage from metrics --> + {{#if model.clusterMetrics.firstObject.getAllResourceTypesDonutChart}} + {{#each + model.clusterMetrics.firstObject.getAllResourceTypesDonutChart as |perTypeUsage|}} + <div class="col-lg-4 container-fluid"> + <div class="panel panel-default"> + <div class="panel-heading"> + {{perTypeUsage.name}} - Usages + </div> + <div class="container-fluid" id="resource-type-{{perTypeUsage.id}}"> + {{donut-chart + data=perTypeUsage.data + showLabels=true + parentIdPrefix="resource-type-" + id=perTypeUsage.id + ratio=0.6 + unit=perTypeUsage.unit + type="resource" + maxHeight=350 + colorTargets="good" + colorTargetReverse=true}} + </div> + </div> </div> - <div class="container-fluid" id="mem-donut-chart"> - {{donut-chart data=model.clusterMetrics.firstObject.getMemoryDataForDonutChart - showLabels=true - parentId="mem-donut-chart" - ratio=0.6 - maxHeight=350 - colorTargets="good" - colorTargetReverse=true - type="memory"}} + {{/each}} + {{else}} + <div class="col-lg-4 container-fluid"> + <div class="panel panel-default"> + <div class="panel-heading"> + Resource - Memory + </div> + <div class="container-fluid" id="mem-donut-chart"> + {{donut-chart + data=model.clusterMetrics.firstObject.getMemoryDataForDonutChart + showLabels=true + parentId="mem-donut-chart" + ratio=0.6 + maxHeight=350 + colorTargets="good" + colorTargetReverse=true + type="memory"}} + </div> </div> </div> - </div> - <div class="col-lg-4 container-fluid"> - <div class="panel panel-default"> - <div class="panel-heading"> - Resource - VCores - </div> - <div class="container-fluid" id="vcore-donut-chart"> - {{donut-chart data=model.clusterMetrics.firstObject.getVCoreDataForDonutChart - showLabels=true - parentId="vcore-donut-chart" - ratio=0.6 - maxHeight=350 - colorTargets="good" - colorTargetReverse=true}} + <div class="col-lg-4 container-fluid"> + <div class="panel panel-default"> + <div class="panel-heading"> + Resource - VCores + </div> + <div class="container-fluid" id="vcore-donut-chart"> + {{donut-chart + data=model.clusterMetrics.firstObject.getVCoreDataForDonutChart + showLabels=true + parentId="vcore-donut-chart" + ratio=0.6 + maxHeight=350 + colorTargets="good" + colorTargetReverse=true}} + </div> </div> </div> - </div> + {{/if}} + </div> <div class="row"> <div class="col-lg-6 container-fluid"> http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/node-menu-panel.hbs ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/node-menu-panel.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/node-menu-panel.hbs index d2486c9..fffae30 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/node-menu-panel.hbs +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/node-menu-panel.hbs @@ -24,8 +24,8 @@ <div class="panel-body"> <ul class="nav nav-pills nav-stacked" id="stacked-menu"> <ul class="nav nav-pills nav-stacked collapse in"> - {{#link-to 'yarn-node' tagName="li"}} - {{#link-to 'yarn-node' nodeId nodeAddr}}Node Information + {{#link-to 'yarn-node.info' tagName="li"}} + {{#link-to 'yarn-node.info' nodeId nodeAddr}}Node Information {{/link-to}} {{/link-to}} {{#link-to 'yarn-node-apps' tagName="li"}} @@ -36,6 +36,12 @@ {{#link-to 'yarn-node-containers' nodeId nodeAddr}}List of Containers {{/link-to}} {{/link-to}} + {{#if nmGpuInfo}} + {{#link-to 'yarn-node.yarn-nm-gpu' tagName="li"}} + {{#link-to 'yarn-node.yarn-nm-gpu' nodeId nodeAddr }}GPU Information + {{/link-to}} + {{/link-to}} + {{/if}} </ul> </ul> </div> http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-nm-gpu-info.hbs ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-nm-gpu-info.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-nm-gpu-info.hbs new file mode 100644 index 0000000..4118b1e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-nm-gpu-info.hbs @@ -0,0 +1,69 @@ +{{! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +}} + +<div class="panel panel-default"> + <div class="panel-heading">Gpu Information - (Minor + Number {{gpu.minorNumber}}) + </div> + <table class="table"> + <tbody> + <tr> + <td>Product Name</td> + <td>{{gpu.productName}}</td> + </tr> + <tr> + <td>UUID</td> + <td>{{gpu.uuid}}</td> + </tr> + <tr> + <td>Current Temperature</td> + <td>{{gpu.temperature.currentGpuTemp}}</td> + </tr> + <tr> + <td>Max Temperature</td> + <td>{{gpu.temperature.maxGpuTemp}}</td> + </tr> + </tbody> + </table> + + <div class="col-md-5 container-fluid" id="mem-donut-chart{{gpu.minorNumber}}"> + {{gpu-donut-chart gpuInfo=gpu + showLabels=true + parentId="mem-donut-chart" + middleLabel = "Gpu Memory" + ratio=0.6 + type="memory" + gpu-render-type = "gpu-memory" + colorTargets="good" + colorTargetReverse=true + maxHeight=350}} + </div> + + <div class="col-md-5 container-fluid" id="utilization-donut-chart{{gpu.minorNumber}}"> + {{gpu-donut-chart gpuInfo=gpu + showLabels=true + parentId="utilization-donut-chart" + middleLabel = "Gpu Utilization" + ratio=0.6 + gpu-render-type = "gpu-utilization" + colorTargets="good" + colorTargetReverse=true + suffix="%" + maxHeight=350}} + </div> +</div> \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
