[
https://issues.apache.org/jira/browse/YARN-11414?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17739691#comment-17739691
]
ASF GitHub Bot commented on YARN-11414:
---------------------------------------
tomicooler commented on code in PR #5293:
URL: https://github.com/apache/hadoop/pull/5293#discussion_r1251115513
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java:
##########
@@ -103,45 +105,64 @@ public ClusterMetricsInfo(final ResourceScheduler rs) {
this.appsRunning = metrics.getAppsRunning();
this.appsFailed = metrics.getAppsFailed();
this.appsKilled = metrics.getAppsKilled();
-
- this.reservedMB = metrics.getReservedMB();
- this.availableMB = metrics.getAvailableMB();
- this.allocatedMB = metrics.getAllocatedMB();
this.pendingMB = metrics.getPendingMB();
-
- this.reservedVirtualCores = metrics.getReservedVirtualCores();
- this.availableVirtualCores = metrics.getAvailableVirtualCores();
- this.allocatedVirtualCores = metrics.getAllocatedVirtualCores();
this.pendingVirtualCores = metrics.getPendingVirtualCores();
this.containersAllocated = metrics.getAllocatedContainers();
this.containersPending = metrics.getPendingContainers();
this.containersReserved = metrics.getReservedContainers();
+ this.totalMB = new ResourceInfo(rs.getClusterResource()).getMemorySize();
+ this.totalVirtualCores = new
ResourceInfo(rs.getClusterResource()).getvCores();
+
if (rs instanceof CapacityScheduler) {
CapacityScheduler cs = (CapacityScheduler) rs;
- this.totalMB = availableMB + allocatedMB + reservedMB;
- this.totalVirtualCores =
- availableVirtualCores + allocatedVirtualCores + reservedVirtualCores;
// TODO, add support of other schedulers to get total used resources
// across partition.
- if (cs.getRootQueue() != null
- && cs.getRootQueue().getQueueResourceUsage() != null
+ if (cs.getRootQueue() != null &&
cs.getRootQueue().getQueueResourceUsage() != null
&& cs.getRootQueue().getQueueResourceUsage().getAllUsed() != null) {
- totalUsedResourcesAcrossPartition = new ResourceInfo(
- cs.getRootQueue().getQueueResourceUsage().getAllUsed());
- totalClusterResourcesAcrossPartition = new ResourceInfo(
- cs.getClusterResource());
- totalReservedResourcesAcrossPartition = new ResourceInfo(
- cs.getRootQueue().getQueueResourceUsage().getAllReserved());
+ totalUsedResourcesAcrossPartition =
+ new
ResourceInfo(cs.getRootQueue().getQueueResourceUsage().getAllUsed());
+ totalClusterResourcesAcrossPartition = new
ResourceInfo(cs.getClusterResource());
+ totalReservedResourcesAcrossPartition =
+ new
ResourceInfo(cs.getRootQueue().getQueueResourceUsage().getAllReserved());
totalAllocatedContainersAcrossPartition =
((ParentQueue) cs.getRootQueue()).getNumContainers();
crossPartitionMetricsAvailable = true;
+
+ this.allocatedMB = totalUsedResourcesAcrossPartition.getMemorySize();
+ this.allocatedVirtualCores =
totalUsedResourcesAcrossPartition.getvCores();
+
+ this.reservedMB =
totalReservedResourcesAcrossPartition.getMemorySize();
+ this.reservedVirtualCores =
totalReservedResourcesAcrossPartition.getvCores();
+ }
+ this.availableMB = this.totalMB - this.allocatedMB;
+ this.availableVirtualCores = this.totalVirtualCores -
this.allocatedVirtualCores;
+ } else if (rs instanceof FairScheduler) {
+ FairScheduler fs = (FairScheduler) rs;
+
+ if (fs.getQueueManager().getRootQueue() != null) {
Review Comment:
Based on the old todo:
```
// TODO, add support of other schedulers to get total used resources
// across partition.
```
Shouldn't we set the totalUsedResourcesAcrossPartition,
totalClusterResourcesAcrossPartition, totalReservedResourcesAcrossPartition
fields here too?
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java:
##########
@@ -103,45 +105,64 @@ public ClusterMetricsInfo(final ResourceScheduler rs) {
this.appsRunning = metrics.getAppsRunning();
this.appsFailed = metrics.getAppsFailed();
this.appsKilled = metrics.getAppsKilled();
-
- this.reservedMB = metrics.getReservedMB();
- this.availableMB = metrics.getAvailableMB();
- this.allocatedMB = metrics.getAllocatedMB();
this.pendingMB = metrics.getPendingMB();
-
- this.reservedVirtualCores = metrics.getReservedVirtualCores();
- this.availableVirtualCores = metrics.getAvailableVirtualCores();
- this.allocatedVirtualCores = metrics.getAllocatedVirtualCores();
this.pendingVirtualCores = metrics.getPendingVirtualCores();
this.containersAllocated = metrics.getAllocatedContainers();
this.containersPending = metrics.getPendingContainers();
this.containersReserved = metrics.getReservedContainers();
+ this.totalMB = new ResourceInfo(rs.getClusterResource()).getMemorySize();
+ this.totalVirtualCores = new
ResourceInfo(rs.getClusterResource()).getvCores();
+
if (rs instanceof CapacityScheduler) {
CapacityScheduler cs = (CapacityScheduler) rs;
- this.totalMB = availableMB + allocatedMB + reservedMB;
- this.totalVirtualCores =
- availableVirtualCores + allocatedVirtualCores + reservedVirtualCores;
// TODO, add support of other schedulers to get total used resources
// across partition.
- if (cs.getRootQueue() != null
- && cs.getRootQueue().getQueueResourceUsage() != null
+ if (cs.getRootQueue() != null &&
cs.getRootQueue().getQueueResourceUsage() != null
&& cs.getRootQueue().getQueueResourceUsage().getAllUsed() != null) {
- totalUsedResourcesAcrossPartition = new ResourceInfo(
- cs.getRootQueue().getQueueResourceUsage().getAllUsed());
- totalClusterResourcesAcrossPartition = new ResourceInfo(
- cs.getClusterResource());
- totalReservedResourcesAcrossPartition = new ResourceInfo(
- cs.getRootQueue().getQueueResourceUsage().getAllReserved());
+ totalUsedResourcesAcrossPartition =
+ new
ResourceInfo(cs.getRootQueue().getQueueResourceUsage().getAllUsed());
+ totalClusterResourcesAcrossPartition = new
ResourceInfo(cs.getClusterResource());
+ totalReservedResourcesAcrossPartition =
+ new
ResourceInfo(cs.getRootQueue().getQueueResourceUsage().getAllReserved());
totalAllocatedContainersAcrossPartition =
((ParentQueue) cs.getRootQueue()).getNumContainers();
crossPartitionMetricsAvailable = true;
+
+ this.allocatedMB = totalUsedResourcesAcrossPartition.getMemorySize();
+ this.allocatedVirtualCores =
totalUsedResourcesAcrossPartition.getvCores();
+
+ this.reservedMB =
totalReservedResourcesAcrossPartition.getMemorySize();
+ this.reservedVirtualCores =
totalReservedResourcesAcrossPartition.getvCores();
+ }
+ this.availableMB = this.totalMB - this.allocatedMB;
+ this.availableVirtualCores = this.totalVirtualCores -
this.allocatedVirtualCores;
+ } else if (rs instanceof FairScheduler) {
+ FairScheduler fs = (FairScheduler) rs;
+
+ if (fs.getQueueManager().getRootQueue() != null) {
+ FSParentQueue rootQueue = fs.getQueueManager().getRootQueue();
+ this.allocatedMB = rootQueue.getResourceUsage().getMemorySize();
+ this.allocatedVirtualCores =
rootQueue.getResourceUsage().getVirtualCores();
+
+ this.reservedMB = rootQueue.getReservedResource().getMemorySize();
+ this.reservedVirtualCores =
rootQueue.getReservedResource().getVirtualCores();
}
+ this.availableMB = this.totalMB - this.allocatedMB;
Review Comment:
`this.allocatedMB` might not be initialised here
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java:
##########
@@ -103,45 +105,64 @@ public ClusterMetricsInfo(final ResourceScheduler rs) {
this.appsRunning = metrics.getAppsRunning();
this.appsFailed = metrics.getAppsFailed();
this.appsKilled = metrics.getAppsKilled();
-
- this.reservedMB = metrics.getReservedMB();
- this.availableMB = metrics.getAvailableMB();
- this.allocatedMB = metrics.getAllocatedMB();
this.pendingMB = metrics.getPendingMB();
-
- this.reservedVirtualCores = metrics.getReservedVirtualCores();
- this.availableVirtualCores = metrics.getAvailableVirtualCores();
- this.allocatedVirtualCores = metrics.getAllocatedVirtualCores();
this.pendingVirtualCores = metrics.getPendingVirtualCores();
this.containersAllocated = metrics.getAllocatedContainers();
this.containersPending = metrics.getPendingContainers();
this.containersReserved = metrics.getReservedContainers();
+ this.totalMB = new ResourceInfo(rs.getClusterResource()).getMemorySize();
+ this.totalVirtualCores = new
ResourceInfo(rs.getClusterResource()).getvCores();
+
if (rs instanceof CapacityScheduler) {
CapacityScheduler cs = (CapacityScheduler) rs;
- this.totalMB = availableMB + allocatedMB + reservedMB;
- this.totalVirtualCores =
- availableVirtualCores + allocatedVirtualCores + reservedVirtualCores;
// TODO, add support of other schedulers to get total used resources
// across partition.
- if (cs.getRootQueue() != null
- && cs.getRootQueue().getQueueResourceUsage() != null
+ if (cs.getRootQueue() != null &&
cs.getRootQueue().getQueueResourceUsage() != null
&& cs.getRootQueue().getQueueResourceUsage().getAllUsed() != null) {
- totalUsedResourcesAcrossPartition = new ResourceInfo(
- cs.getRootQueue().getQueueResourceUsage().getAllUsed());
- totalClusterResourcesAcrossPartition = new ResourceInfo(
- cs.getClusterResource());
- totalReservedResourcesAcrossPartition = new ResourceInfo(
- cs.getRootQueue().getQueueResourceUsage().getAllReserved());
+ totalUsedResourcesAcrossPartition =
+ new
ResourceInfo(cs.getRootQueue().getQueueResourceUsage().getAllUsed());
+ totalClusterResourcesAcrossPartition = new
ResourceInfo(cs.getClusterResource());
+ totalReservedResourcesAcrossPartition =
+ new
ResourceInfo(cs.getRootQueue().getQueueResourceUsage().getAllReserved());
totalAllocatedContainersAcrossPartition =
((ParentQueue) cs.getRootQueue()).getNumContainers();
crossPartitionMetricsAvailable = true;
+
+ this.allocatedMB = totalUsedResourcesAcrossPartition.getMemorySize();
+ this.allocatedVirtualCores =
totalUsedResourcesAcrossPartition.getvCores();
+
+ this.reservedMB =
totalReservedResourcesAcrossPartition.getMemorySize();
+ this.reservedVirtualCores =
totalReservedResourcesAcrossPartition.getvCores();
+ }
+ this.availableMB = this.totalMB - this.allocatedMB;
+ this.availableVirtualCores = this.totalVirtualCores -
this.allocatedVirtualCores;
+ } else if (rs instanceof FairScheduler) {
+ FairScheduler fs = (FairScheduler) rs;
+
+ if (fs.getQueueManager().getRootQueue() != null) {
+ FSParentQueue rootQueue = fs.getQueueManager().getRootQueue();
+ this.allocatedMB = rootQueue.getResourceUsage().getMemorySize();
+ this.allocatedVirtualCores =
rootQueue.getResourceUsage().getVirtualCores();
+
+ this.reservedMB = rootQueue.getReservedResource().getMemorySize();
+ this.reservedVirtualCores =
rootQueue.getReservedResource().getVirtualCores();
}
+ this.availableMB = this.totalMB - this.allocatedMB;
+ this.availableVirtualCores = this.totalVirtualCores -
this.allocatedVirtualCores;
Review Comment:
`this.allocatedVirtualCores` might not be initialised here
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java:
##########
@@ -103,45 +105,64 @@ public ClusterMetricsInfo(final ResourceScheduler rs) {
this.appsRunning = metrics.getAppsRunning();
this.appsFailed = metrics.getAppsFailed();
this.appsKilled = metrics.getAppsKilled();
-
- this.reservedMB = metrics.getReservedMB();
- this.availableMB = metrics.getAvailableMB();
- this.allocatedMB = metrics.getAllocatedMB();
this.pendingMB = metrics.getPendingMB();
-
- this.reservedVirtualCores = metrics.getReservedVirtualCores();
- this.availableVirtualCores = metrics.getAvailableVirtualCores();
- this.allocatedVirtualCores = metrics.getAllocatedVirtualCores();
this.pendingVirtualCores = metrics.getPendingVirtualCores();
this.containersAllocated = metrics.getAllocatedContainers();
this.containersPending = metrics.getPendingContainers();
this.containersReserved = metrics.getReservedContainers();
+ this.totalMB = new ResourceInfo(rs.getClusterResource()).getMemorySize();
+ this.totalVirtualCores = new
ResourceInfo(rs.getClusterResource()).getvCores();
+
if (rs instanceof CapacityScheduler) {
CapacityScheduler cs = (CapacityScheduler) rs;
- this.totalMB = availableMB + allocatedMB + reservedMB;
- this.totalVirtualCores =
- availableVirtualCores + allocatedVirtualCores + reservedVirtualCores;
// TODO, add support of other schedulers to get total used resources
// across partition.
- if (cs.getRootQueue() != null
- && cs.getRootQueue().getQueueResourceUsage() != null
+ if (cs.getRootQueue() != null &&
cs.getRootQueue().getQueueResourceUsage() != null
&& cs.getRootQueue().getQueueResourceUsage().getAllUsed() != null) {
- totalUsedResourcesAcrossPartition = new ResourceInfo(
- cs.getRootQueue().getQueueResourceUsage().getAllUsed());
- totalClusterResourcesAcrossPartition = new ResourceInfo(
- cs.getClusterResource());
- totalReservedResourcesAcrossPartition = new ResourceInfo(
- cs.getRootQueue().getQueueResourceUsage().getAllReserved());
+ totalUsedResourcesAcrossPartition =
+ new
ResourceInfo(cs.getRootQueue().getQueueResourceUsage().getAllUsed());
+ totalClusterResourcesAcrossPartition = new
ResourceInfo(cs.getClusterResource());
+ totalReservedResourcesAcrossPartition =
+ new
ResourceInfo(cs.getRootQueue().getQueueResourceUsage().getAllReserved());
totalAllocatedContainersAcrossPartition =
((ParentQueue) cs.getRootQueue()).getNumContainers();
crossPartitionMetricsAvailable = true;
+
+ this.allocatedMB = totalUsedResourcesAcrossPartition.getMemorySize();
+ this.allocatedVirtualCores =
totalUsedResourcesAcrossPartition.getvCores();
+
+ this.reservedMB =
totalReservedResourcesAcrossPartition.getMemorySize();
+ this.reservedVirtualCores =
totalReservedResourcesAcrossPartition.getvCores();
+ }
+ this.availableMB = this.totalMB - this.allocatedMB;
Review Comment:
`this.allocatedMB` might not be initialised here
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java:
##########
@@ -103,45 +105,64 @@ public ClusterMetricsInfo(final ResourceScheduler rs) {
this.appsRunning = metrics.getAppsRunning();
this.appsFailed = metrics.getAppsFailed();
this.appsKilled = metrics.getAppsKilled();
-
- this.reservedMB = metrics.getReservedMB();
- this.availableMB = metrics.getAvailableMB();
- this.allocatedMB = metrics.getAllocatedMB();
this.pendingMB = metrics.getPendingMB();
-
- this.reservedVirtualCores = metrics.getReservedVirtualCores();
- this.availableVirtualCores = metrics.getAvailableVirtualCores();
- this.allocatedVirtualCores = metrics.getAllocatedVirtualCores();
this.pendingVirtualCores = metrics.getPendingVirtualCores();
this.containersAllocated = metrics.getAllocatedContainers();
this.containersPending = metrics.getPendingContainers();
this.containersReserved = metrics.getReservedContainers();
+ this.totalMB = new ResourceInfo(rs.getClusterResource()).getMemorySize();
+ this.totalVirtualCores = new
ResourceInfo(rs.getClusterResource()).getvCores();
+
if (rs instanceof CapacityScheduler) {
CapacityScheduler cs = (CapacityScheduler) rs;
- this.totalMB = availableMB + allocatedMB + reservedMB;
- this.totalVirtualCores =
- availableVirtualCores + allocatedVirtualCores + reservedVirtualCores;
// TODO, add support of other schedulers to get total used resources
// across partition.
- if (cs.getRootQueue() != null
- && cs.getRootQueue().getQueueResourceUsage() != null
+ if (cs.getRootQueue() != null &&
cs.getRootQueue().getQueueResourceUsage() != null
&& cs.getRootQueue().getQueueResourceUsage().getAllUsed() != null) {
- totalUsedResourcesAcrossPartition = new ResourceInfo(
- cs.getRootQueue().getQueueResourceUsage().getAllUsed());
- totalClusterResourcesAcrossPartition = new ResourceInfo(
- cs.getClusterResource());
- totalReservedResourcesAcrossPartition = new ResourceInfo(
- cs.getRootQueue().getQueueResourceUsage().getAllReserved());
+ totalUsedResourcesAcrossPartition =
+ new
ResourceInfo(cs.getRootQueue().getQueueResourceUsage().getAllUsed());
+ totalClusterResourcesAcrossPartition = new
ResourceInfo(cs.getClusterResource());
+ totalReservedResourcesAcrossPartition =
+ new
ResourceInfo(cs.getRootQueue().getQueueResourceUsage().getAllReserved());
totalAllocatedContainersAcrossPartition =
((ParentQueue) cs.getRootQueue()).getNumContainers();
crossPartitionMetricsAvailable = true;
+
+ this.allocatedMB = totalUsedResourcesAcrossPartition.getMemorySize();
+ this.allocatedVirtualCores =
totalUsedResourcesAcrossPartition.getvCores();
+
+ this.reservedMB =
totalReservedResourcesAcrossPartition.getMemorySize();
+ this.reservedVirtualCores =
totalReservedResourcesAcrossPartition.getvCores();
+ }
+ this.availableMB = this.totalMB - this.allocatedMB;
+ this.availableVirtualCores = this.totalVirtualCores -
this.allocatedVirtualCores;
Review Comment:
`this.allocatedVirtualCores` might not be initialised here
> ClusterMetricsInfo shows wrong availableMB when node labels enabled
> --------------------------------------------------------------------
>
> Key: YARN-11414
> URL: https://issues.apache.org/jira/browse/YARN-11414
> Project: Hadoop YARN
> Issue Type: Bug
> Affects Versions: 3.3.3
> Reporter: Prabhu Joseph
> Assignee: Ashutosh Gupta
> Priority: Major
> Labels: pull-request-available
>
> ClusterMetricsInfo shows wrong availableMB when node labels enabled. It shows
> availableMB of Default Partition alone.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
