http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java index ed71ea2..0243443 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java @@ -31,6 +31,7 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; @@ -56,8 +57,9 @@ public class ApplicationAttemptStateDataPBImpl extends private Container masterContainer = null; private ByteBuffer appAttemptTokens = null; - private Map<String, Long> resourceSecondsMap; + private Map<String, Long> guaranteedResourceSecondsMap; private Map<String, Long> preemptedResourceSecondsMap; + private Map<String, Long> opportunisticResourceSecondsMap; public ApplicationAttemptStateDataPBImpl() { builder = ApplicationAttemptStateDataProto.newBuilder(); @@ -243,30 +245,72 @@ public class ApplicationAttemptStateDataPBImpl extends } @Override + @Deprecated public long getMemorySeconds() { - ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder; - return p.getMemorySeconds(); + return getGuaranteedMemorySeconds(); } @Override + @Deprecated public long getVcoreSeconds() { - ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder; - return p.getVcoreSeconds(); + return getGuaranteedVcoreSeconds(); } @Override + @Deprecated public void setMemorySeconds(long memorySeconds) { - maybeInitBuilder(); - builder.setMemorySeconds(memorySeconds); + setGuaranteedMemorySeconds(memorySeconds); } @Override + @Deprecated public void setVcoreSeconds(long vcoreSeconds) { + setGuaranteedVcoreSeconds(vcoreSeconds); + } + + @Override + public long getGuaranteedMemorySeconds() { + ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder; + return p.getMemorySeconds(); + } + + @Override + public void setGuaranteedMemorySeconds(long memorySeconds) { + maybeInitBuilder(); + builder.setMemorySeconds(memorySeconds); + } + + @Override + public long getGuaranteedVcoreSeconds() { + ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder; + return p.getVcoreSeconds(); + } + + @Override + public void setGuaranteedVcoreSeconds(long vcoreSeconds) { maybeInitBuilder(); builder.setVcoreSeconds(vcoreSeconds); } @Override + public long getOpportunisticMemorySeconds() { + Map<String, Long> tmp = getOpportunisticResourceSecondsMap(); + if (tmp.containsKey(ResourceInformation.MEMORY_MB.getName())) { + return tmp.get(ResourceInformation.MEMORY_MB.getName()); + } + return 0; + } + + @Override + public long getOpportunisticVcoreSeconds() { + Map<String, Long> tmp = getOpportunisticResourceSecondsMap(); + if (tmp.containsKey(ResourceInformation.VCORES.getName())) { + return tmp.get(ResourceInformation.VCORES.getName()); + } + return 0; + } + + @Override public long getPreemptedMemorySeconds() { ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder; return p.getPreemptedMemorySeconds(); @@ -410,21 +454,35 @@ public class ApplicationAttemptStateDataPBImpl extends } @Override + @Deprecated public Map<String, Long> getResourceSecondsMap() { - if (this.resourceSecondsMap != null) { - return this.resourceSecondsMap; + return getGuaranteedResourceSecondsMap(); + } + + @Override + @Deprecated + public void setResourceSecondsMap(Map<String, Long> resourceSecondsMap) { + setGuaranteedResourceSecondsMap(resourceSecondsMap); + } + + @Override + public Map<String, Long> getGuaranteedResourceSecondsMap() { + if (this.guaranteedResourceSecondsMap != null) { + return this.guaranteedResourceSecondsMap; } ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder; - this.resourceSecondsMap = ProtoUtils.convertStringLongMapProtoListToMap( - p.getApplicationResourceUsageMapList()); - return this.resourceSecondsMap; + this.guaranteedResourceSecondsMap = + ProtoUtils.convertStringLongMapProtoListToMap( + p.getApplicationResourceUsageMapList()); + return this.guaranteedResourceSecondsMap; } @Override - public void setResourceSecondsMap(Map<String, Long> resourceSecondsMap) { + public void setGuaranteedResourceSecondsMap( + Map<String, Long> resourceSecondsMap) { maybeInitBuilder(); builder.clearApplicationResourceUsageMap(); - this.resourceSecondsMap = resourceSecondsMap; + this.guaranteedResourceSecondsMap = resourceSecondsMap; if (resourceSecondsMap != null) { builder.addAllApplicationResourceUsageMap( ProtoUtils.convertMapToStringLongMapProtoList(resourceSecondsMap)); @@ -454,4 +512,28 @@ public class ApplicationAttemptStateDataPBImpl extends .convertMapToStringLongMapProtoList(preemptedResourceSecondsMap)); } } + + @Override + public Map<String, Long> getOpportunisticResourceSecondsMap() { + if (this.opportunisticResourceSecondsMap != null) { + return this.opportunisticResourceSecondsMap; + } + ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder; + this.opportunisticResourceSecondsMap = + ProtoUtils.convertStringLongMapProtoListToMap( + p.getApplicationOpportunisticResourceUsageMapList()); + return this.opportunisticResourceSecondsMap; + } + + @Override + public void setOpportunisticResourceSecondsMap( + Map<String, Long> resourceSecondsMap) { + maybeInitBuilder(); + builder.clearApplicationOpportunisticResourceUsageMap(); + this.opportunisticResourceSecondsMap = resourceSecondsMap; + if (resourceSecondsMap != null) { + builder.addAllApplicationOpportunisticResourceUsageMap( + ProtoUtils.convertMapToStringLongMapProtoList(resourceSecondsMap)); + } + } }
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 4234e58..788e87d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -449,6 +449,7 @@ public class RMAppImpl implements RMApp, Recoverable { } else { this.startTime = startTime; } + LOG.info(applicationId.toString() + " started at " + this.startTime); this.applicationType = StringInterner.weakIntern(applicationType); this.applicationTags = applicationTags; this.amReqs = amReqs; @@ -768,10 +769,12 @@ public class RMAppImpl implements RMApp, Recoverable { } RMAppMetrics rmAppMetrics = getRMAppMetrics(); - appUsageReport - .setResourceSecondsMap(rmAppMetrics.getResourceSecondsMap()); + appUsageReport.setGuaranteedResourceSecondsMap( + rmAppMetrics.getGuaranteedResourceSecondsMap()); appUsageReport.setPreemptedResourceSecondsMap( rmAppMetrics.getPreemptedResourceSecondsMap()); + appUsageReport.setOpportunisticResourceSecondsMap( + rmAppMetrics.getOpportunisticResourceSecondsMap()); } if (currentApplicationAttemptId == null) { @@ -1641,8 +1644,9 @@ public class RMAppImpl implements RMApp, Recoverable { Resource resourcePreempted = Resource.newInstance(0, 0); int numAMContainerPreempted = 0; int numNonAMContainerPreempted = 0; - Map<String, Long> resourceSecondsMap = new HashMap<>(); - Map<String, Long> preemptedSecondsMap = new HashMap<>(); + Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>(2); + Map<String, Long> preemptedSecondsMap = new HashMap<>(2); + Map<String, Long> opportunsiticResourceSecondsMap = new HashMap<>(2); this.readLock.lock(); try { for (RMAppAttempt attempt : attempts.values()) { @@ -1658,20 +1662,15 @@ public class RMAppImpl implements RMApp, Recoverable { // for both running and finished containers. AggregateAppResourceUsage resUsage = attempt.getRMAppAttemptMetrics().getAggregateAppResourceUsage(); - for (Map.Entry<String, Long> entry : resUsage - .getResourceUsageSecondsMap().entrySet()) { - long value = RMServerUtils - .getOrDefault(resourceSecondsMap, entry.getKey(), 0L); - value += entry.getValue(); - resourceSecondsMap.put(entry.getKey(), value); - } - for (Map.Entry<String, Long> entry : attemptMetrics - .getPreemptedResourceSecondsMap().entrySet()) { - long value = RMServerUtils - .getOrDefault(preemptedSecondsMap, entry.getKey(), 0L); - value += entry.getValue(); - preemptedSecondsMap.put(entry.getKey(), value); - } + Resources.mergeResourceSecondsMap( + resUsage.getGuaranteedResourceUsageSecondsMap(), + guaranteedResourceSecondsMap); + Resources.mergeResourceSecondsMap( + resUsage.getOpportunisticResourceSecondsMap(), + opportunsiticResourceSecondsMap); + Resources.mergeResourceSecondsMap( + attemptMetrics.getPreemptedResourceSecondsMap(), + preemptedSecondsMap); } } } finally { @@ -1679,7 +1678,8 @@ public class RMAppImpl implements RMApp, Recoverable { } return new RMAppMetrics(resourcePreempted, numNonAMContainerPreempted, - numAMContainerPreempted, resourceSecondsMap, preemptedSecondsMap); + numAMContainerPreempted, guaranteedResourceSecondsMap, + preemptedSecondsMap, opportunsiticResourceSecondsMap); } @Private http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java index 2bb7fd1..76272e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java @@ -28,18 +28,21 @@ public class RMAppMetrics { final Resource resourcePreempted; final int numNonAMContainersPreempted; final int numAMContainersPreempted; - private final Map<String, Long> resourceSecondsMap; + private final Map<String, Long> guaranteedResourceSecondsMap; private final Map<String, Long> preemptedResourceSecondsMap; + private final Map<String, Long> opportunisticResourceSecondsMap; public RMAppMetrics(Resource resourcePreempted, int numNonAMContainersPreempted, int numAMContainersPreempted, - Map<String, Long> resourceSecondsMap, - Map<String, Long> preemptedResourceSecondsMap) { + Map<String, Long> guaranteedResourceSecondsMap, + Map<String, Long> preemptedResourceSecondsMap, + Map<String, Long> opportunisticResourceSecondsMap) { this.resourcePreempted = resourcePreempted; this.numNonAMContainersPreempted = numNonAMContainersPreempted; this.numAMContainersPreempted = numAMContainersPreempted; - this.resourceSecondsMap = resourceSecondsMap; + this.guaranteedResourceSecondsMap = guaranteedResourceSecondsMap; this.preemptedResourceSecondsMap = preemptedResourceSecondsMap; + this.opportunisticResourceSecondsMap = opportunisticResourceSecondsMap; } public Resource getResourcePreempted() { @@ -54,17 +57,25 @@ public class RMAppMetrics { return numAMContainersPreempted; } - public long getMemorySeconds() { - return RMServerUtils.getOrDefault(resourceSecondsMap, + public long getGuaranteedMemorySeconds() { + return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap, ResourceInformation.MEMORY_MB.getName(), 0L); } - public long getVcoreSeconds() { - return RMServerUtils - .getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(), - 0L); + public long getGuaranteedVcoreSeconds() { + return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap, + ResourceInformation.VCORES.getName(), 0L); + } + + public long getOpportunisticMemorySeconds() { + return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap, + ResourceInformation.MEMORY_MB.getName(), 0L); } + public long getOpportunisticVcoreSeconds() { + return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap, + ResourceInformation.VCORES.getName(), 0L); + } public long getPreemptedMemorySeconds() { return RMServerUtils.getOrDefault(preemptedResourceSecondsMap, ResourceInformation.MEMORY_MB.getName(), 0L); @@ -75,12 +86,15 @@ public class RMAppMetrics { ResourceInformation.VCORES.getName(), 0L); } - public Map<String, Long> getResourceSecondsMap() { - return resourceSecondsMap; + public Map<String, Long> getGuaranteedResourceSecondsMap() { + return guaranteedResourceSecondsMap; } public Map<String, Long> getPreemptedResourceSecondsMap() { return preemptedResourceSecondsMap; } + public Map<String, Long> getOpportunisticResourceSecondsMap() { + return opportunisticResourceSecondsMap; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java index b858712..a75cc57 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java @@ -27,30 +27,55 @@ import java.util.Map; @Private public class AggregateAppResourceUsage { - private Map<String, Long> resourceSecondsMap = new HashMap<>(); + private Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>(); + private Map<String, Long> opportunisticResourceSecondsMap = new HashMap<>(); - public AggregateAppResourceUsage(Map<String, Long> resourceSecondsMap) { - this.resourceSecondsMap.putAll(resourceSecondsMap); + public AggregateAppResourceUsage( + Map<String, Long> guaranteedResourceSecondsMap, + Map<String, Long> opportunisticResourceSecondsMap) { + this.guaranteedResourceSecondsMap.putAll( + guaranteedResourceSecondsMap); + this.opportunisticResourceSecondsMap.putAll( + opportunisticResourceSecondsMap); } /** - * @return the memorySeconds + * @return the guaranteed memory seconds */ - public long getMemorySeconds() { - return RMServerUtils.getOrDefault(resourceSecondsMap, + public long getGuaranteedMemorySeconds() { + return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap, ResourceInformation.MEMORY_MB.getName(), 0L); } /** - * @return the vcoreSeconds + * @return the guaranteed vcore seconds */ - public long getVcoreSeconds() { - return RMServerUtils - .getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(), - 0L); + public long getGuaranteedVcoreSeconds() { + return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap, + ResourceInformation.VCORES.getName(), 0L); } - public Map<String, Long> getResourceUsageSecondsMap() { - return resourceSecondsMap; + /** + * @return the opportunistic memory seconds + */ + public long getOpportunisticMemorySeconds() { + return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap, + ResourceInformation.MEMORY_MB.getName(), 0L); + } + + /** + * @return the opportunistic vcore seconds + */ + public long getOpportunisticVcoreSeconds() { + return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap, + ResourceInformation.VCORES.getName(), 0L); + } + + public Map<String, Long> getGuaranteedResourceUsageSecondsMap() { + return guaranteedResourceSecondsMap; + } + + public Map<String, Long> getOpportunisticResourceSecondsMap() { + return opportunisticResourceSecondsMap; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 1b1e2c4..a8fdfd2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -938,15 +938,18 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { this.readLock.lock(); try { ApplicationResourceUsageReport report = - scheduler.getAppResourceUsageReport(this.getAppAttemptId()); + scheduler.getAppActiveResourceUsageReport(this.getAppAttemptId()); if (report == null) { report = RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT; } AggregateAppResourceUsage resUsage = this.attemptMetrics.getAggregateAppResourceUsage(); - report.setResourceSecondsMap(resUsage.getResourceUsageSecondsMap()); + report.setGuaranteedResourceSecondsMap( + resUsage.getGuaranteedResourceUsageSecondsMap()); report.setPreemptedResourceSecondsMap( this.attemptMetrics.getPreemptedResourceSecondsMap()); + report.setOpportunisticResourceSecondsMap( + resUsage.getOpportunisticResourceSecondsMap()); return report; } finally { this.readLock.unlock(); @@ -983,10 +986,12 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { this.finalStatus = attemptState.getFinalApplicationStatus(); this.startTime = attemptState.getStartTime(); this.finishTime = attemptState.getFinishTime(); - this.attemptMetrics - .updateAggregateAppResourceUsage(attemptState.getResourceSecondsMap()); + this.attemptMetrics.updateAggregateAppGuaranteedResourceUsage( + attemptState.getGuaranteedResourceSecondsMap()); this.attemptMetrics.updateAggregatePreemptedAppResourceUsage( attemptState.getPreemptedResourceSecondsMap()); + this.attemptMetrics.updateAggregateAppOpportunisticResourceUsage( + attemptState.getOpportunisticResourceSecondsMap()); } public void transferStateFromAttempt(RMAppAttempt attempt) { @@ -1406,9 +1411,11 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { ApplicationAttemptStateData attemptState = ApplicationAttemptStateData .newInstance(applicationAttemptId, getMasterContainer(), rmStore.getCredentialsFromAppAttempt(this), startTime, - stateToBeStored, finalTrackingUrl, diags.toString(), finalStatus, exitStatus, - getFinishTime(), resUsage.getResourceUsageSecondsMap(), - this.attemptMetrics.getPreemptedResourceSecondsMap()); + stateToBeStored, finalTrackingUrl, diags.toString(), finalStatus, + exitStatus, getFinishTime(), + resUsage.getGuaranteedResourceUsageSecondsMap(), + this.attemptMetrics.getPreemptedResourceSecondsMap(), + resUsage.getOpportunisticResourceSecondsMap()); LOG.info("Updating application attempt " + applicationAttemptId + " with final state: " + targetedFinalState + ", and exit status: " + exitStatus); http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java index 015cff7..192722b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java @@ -53,8 +53,11 @@ public class RMAppAttemptMetrics { private ReadLock readLock; private WriteLock writeLock; - private Map<String, AtomicLong> resourceUsageMap = new HashMap<>(); + private Map<String, AtomicLong> guaranteedResourceUsageMap + = new HashMap<>(); private Map<String, AtomicLong> preemptedResourceMap = new HashMap<>(); + private Map<String, AtomicLong> opportunisticResourceUsageMap + = new HashMap<>(); private RMContext rmContext; private int[][] localityStatistics = @@ -129,8 +132,10 @@ public class RMAppAttemptMetrics { } public AggregateAppResourceUsage getAggregateAppResourceUsage() { - Map<String, Long> resourcesUsed = - convertAtomicLongMaptoLongMap(resourceUsageMap); + Map<String, Long> guaranteedResourcesUsed = + convertAtomicLongMaptoLongMap(guaranteedResourceUsageMap); + Map<String, Long> opportunisticResourcesUsed = + convertAtomicLongMaptoLongMap(opportunisticResourceUsageMap); // Only add in the running containers if this is the active attempt. RMApp rmApp = rmContext.getRMApps().get(attemptId.getApplicationId()); @@ -139,27 +144,25 @@ public class RMAppAttemptMetrics { if (currentAttempt != null && currentAttempt.getAppAttemptId().equals(attemptId)) { ApplicationResourceUsageReport appResUsageReport = - rmContext.getScheduler().getAppResourceUsageReport(attemptId); + rmContext.getScheduler().getAppActiveResourceUsageReport(attemptId); if (appResUsageReport != null) { - Map<String, Long> tmp = appResUsageReport.getResourceSecondsMap(); - for (Map.Entry<String, Long> entry : tmp.entrySet()) { - Long value = resourcesUsed.get(entry.getKey()); - if (value != null) { - value += entry.getValue(); - } else { - value = entry.getValue(); - } - resourcesUsed.put(entry.getKey(), value); - } + Resources.mergeResourceSecondsMap( + appResUsageReport.getGuaranteedResourceSecondsMap(), + guaranteedResourcesUsed); + Resources.mergeResourceSecondsMap( + appResUsageReport.getOpportunisticResourceSecondsMap(), + opportunisticResourcesUsed); } } } - return new AggregateAppResourceUsage(resourcesUsed); + + return new AggregateAppResourceUsage(guaranteedResourcesUsed, + opportunisticResourcesUsed); } public void updateAggregateAppResourceUsage(Resource allocated, long deltaUsedMillis) { - updateUsageMap(allocated, deltaUsedMillis, resourceUsageMap); + updateUsageMap(allocated, deltaUsedMillis, guaranteedResourceUsageMap); } public void updateAggregatePreemptedAppResourceUsage(Resource allocated, @@ -167,9 +170,14 @@ public class RMAppAttemptMetrics { updateUsageMap(allocated, deltaUsedMillis, preemptedResourceMap); } - public void updateAggregateAppResourceUsage( + public void updateAggregateAppGuaranteedResourceUsage( Map<String, Long> resourceSecondsMap) { - updateUsageMap(resourceSecondsMap, resourceUsageMap); + updateUsageMap(resourceSecondsMap, guaranteedResourceUsageMap); + } + + public void updateAggregateAppOpportunisticResourceUsage( + Map<String, Long> resourceSecondsMap) { + updateUsageMap(resourceSecondsMap, opportunisticResourceUsageMap); } public void updateAggregatePreemptedAppResourceUsage( @@ -199,10 +207,9 @@ public class RMAppAttemptMetrics { if (!targetMap.containsKey(entry.getKey())) { resourceUsed = new AtomicLong(0); targetMap.put(entry.getKey(), resourceUsed); - } resourceUsed = targetMap.get(entry.getKey()); - resourceUsed.set(entry.getValue()); + resourceUsed.addAndGet(entry.getValue()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java new file mode 100644 index 0000000..c91dae8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer; + +import org.apache.hadoop.classification.InterfaceAudience.Private; + +import java.util.Map; + +/** + * The resource usage information of a <code>Container</code>. + */ +@Private +public class ContainerResourceUsageReport { + private final Map<String, Long> guaranteedResourceSecondsMap; + private final Map<String, Long> opportunisticResourceSecondsMap; + + public ContainerResourceUsageReport( + Map<String, Long> guaranteedResourceSecondsMap, + Map<String, Long> opportunisticResourceSecondsMap) { + this.guaranteedResourceSecondsMap = guaranteedResourceSecondsMap; + this.opportunisticResourceSecondsMap = opportunisticResourceSecondsMap; + } + + public Map<String, Long> getGuaranteedResourceUsageSecondsMap() { + return guaranteedResourceSecondsMap; + } + + public Map<String, Long> getOpportunisticResourceSecondsMap() { + return opportunisticResourceSecondsMap; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java index 8f751b0..4897312 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java @@ -87,7 +87,9 @@ public interface RMContainer extends EventHandler<RMContainerEvent>, ContainerState getContainerState(); ContainerReport createContainerReport(); - + + ContainerResourceUsageReport getResourceUsageReport(); + boolean isAMContainer(); ContainerRequest getContainerRequest(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java index 541621b..28b4f10 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java @@ -21,10 +21,13 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer; import java.util.Collections; import java.util.EnumSet; import java.util.Set; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; +import org.apache.commons.lang.time.DateUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -39,6 +42,7 @@ import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; @@ -175,7 +179,7 @@ public class RMContainerImpl implements RMContainer { private NodeId reservedNode; private SchedulerRequestKey reservedSchedulerKey; private long creationTime; - private long finishTime; + private long finishTime = -1L; private ContainerStatus finishedStatus; private boolean isAMContainer; private ContainerRequest containerRequestForRecovery; @@ -190,6 +194,10 @@ public class RMContainerImpl implements RMContainer { private volatile Set<String> allocationTags = null; + // This is updated whenever the container instance is updated in + // the case of container increase/decrease/promotion/demotion + private long lastContainerUpdateTime; + public RMContainerImpl(Container container, SchedulerRequestKey schedulerKey, ApplicationAttemptId appAttemptId, NodeId nodeId, String user, RMContext rmContext) { @@ -231,6 +239,7 @@ public class RMContainerImpl implements RMContainer { this.appAttemptId = appAttemptId; this.user = user; this.creationTime = creationTime; + this.lastContainerUpdateTime = creationTime; this.rmContext = rmContext; this.eventHandler = rmContext.getDispatcher().getEventHandler(); this.containerAllocationExpirer = rmContext.getContainerAllocationExpirer(); @@ -281,9 +290,87 @@ public class RMContainerImpl implements RMContainer { } public void setContainer(Container container) { + // containers are updated by resetting the underlying Container instance, + // always update RMContainer resource usage before the update. + updateAppAttemptMetrics(true); this.container = container; } + /** + * Collect the resource usage information of the current underlying Container + * instance and update its RMAppAttemptMetrics. This is called whenever the + * RMContainer is updated (by updating its underlying container) or finishes. + * @param containerUpdated if it is called upon container update event + */ + private void updateAppAttemptMetrics(boolean containerUpdated) { + RMAppAttempt rmAttempt = rmContext.getRMApps() + .get(getApplicationAttemptId().getApplicationId()) + .getCurrentAppAttempt(); + + if (rmAttempt != null) { + // collect resource usage information of the current Container instance + ContainerResourceUsageReport resourceUsage = + getCurrentContainerResourceUsage(containerUpdated); + + // If this is a preempted container, update preemption metrics + if (finishedStatus != null && + ContainerExitStatus.PREEMPTED == finishedStatus.getExitStatus()) { + rmAttempt.getRMAppAttemptMetrics() + .updatePreemptionInfo(container.getResource(), this); + rmAttempt.getRMAppAttemptMetrics() + .updateAggregateAppOpportunisticResourceUsage( + resourceUsage.getGuaranteedResourceUsageSecondsMap()); + } + rmAttempt.getRMAppAttemptMetrics() + .updateAggregateAppOpportunisticResourceUsage( + resourceUsage.getOpportunisticResourceSecondsMap()); + rmAttempt.getRMAppAttemptMetrics() + .updateAggregateAppGuaranteedResourceUsage( + resourceUsage.getGuaranteedResourceUsageSecondsMap()); + } + } + + /** + * Get resource usage of the current underlying Container instance. + * @param containerUpdated if this is called upon container update + * @return resource usage of the current <code>container</code> + */ + private ContainerResourceUsageReport getCurrentContainerResourceUsage( + boolean containerUpdated) { + ContainerResourceUsageReport report; + + // A container generates usage until it finishes which is indicated + // by a positive finish timestamp that is set once it finishes + final long currentTimeMillis = + finishTime < 0 ? System.currentTimeMillis() : finishTime; + final long usedSeconds = (currentTimeMillis - + lastContainerUpdateTime) / DateUtils.MILLIS_PER_SECOND; + Resource resource = container.getResource(); + + if (container.getExecutionType() == ExecutionType.GUARANTEED) { + Map<String, Long> guaranteedResourceSeconds = new HashMap<>(2); + for (ResourceInformation entry : resource.getResources()) { + guaranteedResourceSeconds.put( + entry.getName(), entry.getValue() * usedSeconds); + } + report = new ContainerResourceUsageReport(guaranteedResourceSeconds, + Collections.emptyMap()); + } else { + Map<String, Long> opportunisticResourceSeconds = new HashMap<>(2); + for (ResourceInformation entry : resource.getResources()) { + opportunisticResourceSeconds.put( + entry.getName(), entry.getValue() * usedSeconds); + } + report = new ContainerResourceUsageReport(Collections.emptyMap(), + opportunisticResourceSeconds); + } + + if (containerUpdated) { + lastContainerUpdateTime = currentTimeMillis; + } + return report; + } + @Override public RMContainerState getState() { this.readLock.lock(); @@ -712,7 +799,7 @@ public class RMContainerImpl implements RMContainer { // Inform AppAttempt // container.getContainer() can return null when a RMContainer is a // reserved container - updateAttemptMetrics(container); + container.updateAppAttemptMetrics(false); container.eventHandler.handle(new RMAppAttemptContainerFinishedEvent( container.appAttemptId, finishedEvent.getRemoteContainerStatus(), @@ -734,27 +821,6 @@ public class RMContainerImpl implements RMContainer { } } - - private static void updateAttemptMetrics(RMContainerImpl container) { - Resource resource = container.getContainer().getResource(); - RMAppAttempt rmAttempt = container.rmContext.getRMApps() - .get(container.getApplicationAttemptId().getApplicationId()) - .getCurrentAppAttempt(); - - if (rmAttempt != null) { - long usedMillis = container.finishTime - container.creationTime; - rmAttempt.getRMAppAttemptMetrics() - .updateAggregateAppResourceUsage(resource, usedMillis); - // If this is a preempted container, update preemption metrics - if (ContainerExitStatus.PREEMPTED == container.finishedStatus - .getExitStatus()) { - rmAttempt.getRMAppAttemptMetrics() - .updatePreemptionInfo(resource, container); - rmAttempt.getRMAppAttemptMetrics() - .updateAggregatePreemptedAppResourceUsage(resource, usedMillis); - } - } - } } private static final class KillTransition extends FinishedTransition { @@ -793,6 +859,11 @@ public class RMContainerImpl implements RMContainer { } @Override + public ContainerResourceUsageReport getResourceUsageReport() { + return getCurrentContainerResourceUsage(false); + } + + @Override public String getNodeHttpAddress() { try { readLock.lock(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index e3b56d3..265ccc8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -417,7 +417,7 @@ public abstract class AbstractYarnScheduler } @Override - public ApplicationResourceUsageReport getAppResourceUsageReport( + public ApplicationResourceUsageReport getAppActiveResourceUsageReport( ApplicationAttemptId appAttemptId) { SchedulerApplicationAttempt attempt = getApplicationAttempt(appAttemptId); if (attempt == null) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index 857e736..186fa2d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -34,7 +34,6 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.time.DateUtils; import org.apache.commons.lang.time.FastDateFormat; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -61,12 +60,12 @@ import org.apache.hadoop.yarn.api.records.UpdateContainerError; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.server.api.ContainerType; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; -import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerResourceUsageReport; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; @@ -108,7 +107,10 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { private static final long MEM_AGGREGATE_ALLOCATION_CACHE_MSECS = 3000; protected long lastMemoryAggregateAllocationUpdateTime = 0; - private Map<String, Long> lastResourceSecondsMap = new HashMap<>(); + private Map<String, Long> lastGuaranteedResourceSecondsMap = + new HashMap<>(); + private Map<String, Long> lastOpportunisticResourceSecondsMap = + new HashMap<>(); protected final AppSchedulingInfo appSchedulingInfo; protected ApplicationAttemptId attemptId; protected Map<ContainerId, RMContainer> liveContainers = @@ -1082,34 +1084,47 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { // recently. if ((currentTimeMillis - lastMemoryAggregateAllocationUpdateTime) > MEM_AGGREGATE_ALLOCATION_CACHE_MSECS) { - Map<String, Long> resourceSecondsMap = new HashMap<>(); + Map<String, Long> aggregatedGuaranteedResourceSecondsMap = + new HashMap<>(); + Map<String, Long> aggregatedOpportunisticResourceSecondsMap = + new HashMap<>(); for (RMContainer rmContainer : this.liveContainers.values()) { - long usedMillis = currentTimeMillis - rmContainer.getCreationTime(); - Resource resource = rmContainer.getContainer().getResource(); - for (ResourceInformation entry : resource.getResources()) { - long value = RMServerUtils - .getOrDefault(resourceSecondsMap, entry.getName(), 0L); - value += entry.getValue() * usedMillis - / DateUtils.MILLIS_PER_SECOND; - resourceSecondsMap.put(entry.getName(), value); - } + ContainerResourceUsageReport containerResourceUsageReport = + rmContainer.getResourceUsageReport(); + Resources.mergeResourceSecondsMap( + containerResourceUsageReport.getGuaranteedResourceUsageSecondsMap(), + aggregatedGuaranteedResourceSecondsMap); + Resources.mergeResourceSecondsMap( + containerResourceUsageReport.getOpportunisticResourceSecondsMap(), + aggregatedOpportunisticResourceSecondsMap); } - + lastGuaranteedResourceSecondsMap = + aggregatedGuaranteedResourceSecondsMap; + lastOpportunisticResourceSecondsMap = + aggregatedOpportunisticResourceSecondsMap; lastMemoryAggregateAllocationUpdateTime = currentTimeMillis; - lastResourceSecondsMap = resourceSecondsMap; } - return new AggregateAppResourceUsage(lastResourceSecondsMap); + return new AggregateAppResourceUsage(lastGuaranteedResourceSecondsMap, + lastOpportunisticResourceSecondsMap); } + /** + * Get the resources that are actively being used by the app attempt. + * @return active resource usage + */ public ApplicationResourceUsageReport getResourceUsageReport() { try { writeLock.lock(); AggregateAppResourceUsage runningResourceUsage = getRunningAggregateAppResourceUsage(); - Resource usedResourceClone = Resources.clone( + Resource guaranteedResourceUsedClone = Resources.clone( attemptResourceUsage.getAllUsed()); + Resource opportunisticResourceUsedClone = Resources.clone( + attemptOpportunisticResourceUsage.getAllUsed()); Resource reservedResourceClone = Resources.clone( attemptResourceUsage.getReserved()); + Resource neededResource = Resources.add(opportunisticResourceUsedClone, + Resources.add(guaranteedResourceUsedClone, reservedResourceClone)); Resource cluster = rmContext.getScheduler().getClusterResource(); ResourceCalculator calc = rmContext.getScheduler().getResourceCalculator(); @@ -1124,18 +1139,20 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { float queueCapacityPerc = queue.getQueueInfo(false, false) .getCapacity(); if (queueCapacityPerc != 0) { - queueUsagePerc = calc.divide(cluster, usedResourceClone, + queueUsagePerc = calc.divide(cluster, guaranteedResourceUsedClone, Resources.multiply(cluster, queueCapacityPerc)) * 100; } clusterUsagePerc = - calc.divide(cluster, usedResourceClone, cluster) * 100; + calc.divide(cluster, guaranteedResourceUsedClone, cluster) * 100; } return ApplicationResourceUsageReport .newInstance(liveContainers.size(), reservedContainers.size(), - usedResourceClone, reservedResourceClone, - Resources.add(usedResourceClone, reservedResourceClone), - runningResourceUsage.getResourceUsageSecondsMap(), queueUsagePerc, - clusterUsagePerc, preemptedResourceSecondsMaps); + guaranteedResourceUsedClone, reservedResourceClone, + neededResource, + runningResourceUsage.getGuaranteedResourceUsageSecondsMap(), + queueUsagePerc, clusterUsagePerc, preemptedResourceSecondsMaps, + opportunisticResourceUsedClone, + runningResourceUsage.getOpportunisticResourceSecondsMap()); } finally { writeLock.unlock(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java index 43d55c4..52920ac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java @@ -172,7 +172,7 @@ public interface YarnScheduler extends EventHandler<SchedulerEvent> { */ @LimitedPrivate("yarn") @Evolving - ApplicationResourceUsageReport getAppResourceUsageReport( + ApplicationResourceUsageReport getAppActiveResourceUsageReport( ApplicationAttemptId appAttemptId); /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index 8b6b946..7318604 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -1046,7 +1046,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt { .getEffectiveCapacity(getAppAMNodePartitionName()); if (!effCap.equals(Resources.none())) { queueUsagePerc = calc.divide(totalPartitionRes, - report.getUsedResources(), effCap) * 100; + report.getGuaranteedResourcesUsed(), effCap) * 100; } report.setQueueUsagePercentage(queueUsagePerc); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index 0719ca6..9ef4d31 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -169,8 +169,13 @@ public class FSAppAttempt extends SchedulerApplicationAttempt queue.getMetrics().releaseResources( rmContainer.getNodeLabelExpression(), getUser(), 1, containerResource); - this.attemptResourceUsage.decUsed(containerResource); - getQueue().decUsedGuaranteedResource(containerResource); + if (rmContainer.getExecutionType().equals(ExecutionType.GUARANTEED)) { + this.attemptResourceUsage.decUsed(containerResource); + getQueue().decUsedGuaranteedResource(containerResource); + } else { + this.attemptOpportunisticResourceUsage.decUsed(containerResource); + } + // Clear resource utilization metrics cache. lastMemoryAggregateAllocationUpdateTime = -1; http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java index ac88f86..14dab9f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java @@ -95,6 +95,10 @@ public class FairSchedulerAppsBlock extends HtmlBlock { th(".runningcontainer", "Running Containers"). th(".allocatedCpu", "Allocated CPU VCores"). th(".allocatedMemory", "Allocated Memory MB"). + th(".allocatedOpportunisticCpu", + "Allocated Opportunistic CPU Vcores"). + th(".allocatedOpportunisticMemory", + "Allocated Opportunistic Memory MB"). th(".reservedCpu", "Reserved CPU VCores"). th(".reservedMemory", "Reserved Memory MB"). th(".progress", "Progress"). @@ -144,6 +148,10 @@ public class FairSchedulerAppsBlock extends HtmlBlock { .valueOf(appInfo.getAllocatedVCores())).append("\",\"") .append(appInfo.getAllocatedMB() == -1 ? "N/A" : String .valueOf(appInfo.getAllocatedMB())).append("\",\"") + .append(appInfo.getAllocatedOpportunisticVCores() == -1 ? "N/A" : String + .valueOf(appInfo.getAllocatedOpportunisticVCores())).append("\",\"") + .append(appInfo.getAllocatedOpportunisticMB() == -1 ? "N/A" : String + .valueOf(appInfo.getAllocatedOpportunisticMB())).append("\",\"") .append(appInfo.getReservedVCores() == -1 ? "N/A" : String .valueOf(appInfo.getReservedVCores())).append("\",\"") .append(appInfo.getReservedMB() == -1 ? "N/A" : String http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java index 8553d8c..6cd10f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java @@ -107,9 +107,12 @@ public class RMAppBlock extends AppBlock{ attemptResourcePreempted) .__("Number of Non-AM Containers Preempted from Current Attempt:", attemptNumNonAMContainerPreempted) - .__("Aggregate Resource Allocation:", appMetrics == null ? "N/A" : - StringHelper - .getResourceSecondsString(appMetrics.getResourceSecondsMap())) + .__("Aggregate Guaranteed Resource Allocation:", appMetrics == null ? + "N/A" : StringHelper.getResourceSecondsString( + appMetrics.getGuaranteedResourceSecondsMap())) + .__("Aggregate Opportunistic Resource Allocation:", appMetrics == null ? + "N/A" : StringHelper.getResourceSecondsString( + appMetrics.getOpportunisticResourceSecondsMap())) .__("Aggregate Preempted Resource Allocation:", appMetrics == null ? "N/A" : StringHelper.getResourceSecondsString( appMetrics.getPreemptedResourceSecondsMap())); http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java index d0dccab..0510d62 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java @@ -69,6 +69,10 @@ public class RMAppsBlock extends AppsBlock { .th(".allocatedMemory", "Allocated Memory MB") .th(".reservedCpu", "Reserved CPU VCores") .th(".reservedMemory", "Reserved Memory MB") + .th(".allocatedOpportunisticCpu", + "Allocated Opportunistic CPU VCores") + .th(".allocatedOpportunisticMemory", + "Allocated Opportunistic Memory MB") .th(".queuePercentage", "% of Queue") .th(".clusterPercentage", "% of Cluster") .th(".progress", "Progress") @@ -155,6 +159,12 @@ public class RMAppsBlock extends AppsBlock { .append(app.getReservedMemoryMB() == -1 ? "N/A" : String.valueOf(app.getReservedMemoryMB())) .append("\",\"") + .append(app.getAllocatedOpportunisticCpuVcores() == -1 ? "N/A" : String + .valueOf(app.getAllocatedOpportunisticCpuVcores())) + .append("\",\"") + .append(app.getAllocatedOpportunisticMemoryMB() == -1 ? "N/A" : + String.valueOf(app.getAllocatedOpportunisticMemoryMB())) + .append("\",\"") .append(queuePercent) .append("\",\"") .append(clusterPercent) http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 631c908..e7ac27e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -95,6 +95,8 @@ public class AppInfo { private String masterNodeId; private long allocatedMB; private long allocatedVCores; + private long allocatedOpportunisticMB; + private long allocatedOpportunisticVCores; private long reservedMB; private long reservedVCores; private int runningContainers; @@ -103,6 +105,9 @@ public class AppInfo { protected float queueUsagePercentage; protected float clusterUsagePercentage; protected Map<String, Long> resourceSecondsMap; + private long opportunisticMemorySeconds; + private long opportunisticVcoreSeconds; + private Map<String, Long> opportunisticResourceSecondsMap; // preemption info fields private long preemptedResourceMB; @@ -202,10 +207,17 @@ public class AppInfo { ApplicationResourceUsageReport resourceReport = attempt.getApplicationResourceUsageReport(); if (resourceReport != null) { - Resource usedResources = resourceReport.getUsedResources(); + Resource guaranteedResourcesUsed = + resourceReport.getGuaranteedResourcesUsed(); + Resource opportunisticResourceUsed = + resourceReport.getOpportunisticResourcesUsed(); Resource reservedResources = resourceReport.getReservedResources(); - allocatedMB = usedResources.getMemorySize(); - allocatedVCores = usedResources.getVirtualCores(); + allocatedMB = guaranteedResourcesUsed.getMemorySize(); + allocatedVCores = guaranteedResourcesUsed.getVirtualCores(); + allocatedOpportunisticMB = + opportunisticResourceUsed.getMemorySize(); + allocatedOpportunisticVCores = + opportunisticResourceUsed.getVirtualCores(); reservedMB = reservedResources.getMemorySize(); reservedVCores = reservedResources.getVirtualCores(); runningContainers = resourceReport.getNumUsedContainers(); @@ -239,9 +251,13 @@ public class AppInfo { numNonAMContainerPreempted = appMetrics.getNumNonAMContainersPreempted(); preemptedResourceVCores = appMetrics.getResourcePreempted().getVirtualCores(); - memorySeconds = appMetrics.getMemorySeconds(); - vcoreSeconds = appMetrics.getVcoreSeconds(); - resourceSecondsMap = appMetrics.getResourceSecondsMap(); + memorySeconds = appMetrics.getGuaranteedMemorySeconds(); + vcoreSeconds = appMetrics.getGuaranteedVcoreSeconds(); + resourceSecondsMap = appMetrics.getGuaranteedResourceSecondsMap(); + opportunisticMemorySeconds = appMetrics.getOpportunisticMemorySeconds(); + opportunisticVcoreSeconds = appMetrics.getOpportunisticVcoreSeconds(); + opportunisticResourceSecondsMap = + appMetrics.getOpportunisticResourceSecondsMap(); preemptedMemorySeconds = appMetrics.getPreemptedMemorySeconds(); preemptedVcoreSeconds = appMetrics.getPreemptedVcoreSeconds(); preemptedResourceSecondsMap = appMetrics.getPreemptedResourceSecondsMap(); @@ -454,6 +470,13 @@ public class AppInfo { return this.allocatedVCores; } + public long getAllocatedOpportunisticMB() { + return this.allocatedOpportunisticMB; + } + + public long getAllocatedOpportunisticVCores() { + return this.allocatedOpportunisticVCores; + } public long getReservedMB() { return this.reservedMB; } @@ -478,18 +501,30 @@ public class AppInfo { return numAMContainerPreempted; } - public long getMemorySeconds() { + public long getGuaranteedMemorySeconds() { return memorySeconds; } - public long getVcoreSeconds() { + public long getGuaranteedVcoreSeconds() { return vcoreSeconds; } - public Map<String, Long> getResourceSecondsMap() { + public long getOpportunisticMemorySeconds() { + return opportunisticMemorySeconds; + } + + public long getOpportunisticVcoreSeconds() { + return opportunisticVcoreSeconds; + } + + public Map<String, Long> getGuaranteedResourceSecondsMap() { return resourceSecondsMap; } + public Map<String, Long> getOpportunisticResourceSecondsMap() { + return opportunisticResourceSecondsMap; + } + public long getPreemptedMemorySeconds() { return preemptedMemorySeconds; } @@ -582,6 +617,15 @@ public class AppInfo { this.allocatedVCores = allocatedVCores; } + public void setAllocatedOpportunisticMB(long allocatedOpportunisticMB) { + this.allocatedOpportunisticMB = allocatedOpportunisticMB; + } + + public void setAllocatedOpportunisticVCores( + long allocatedOpportunisticVCores) { + this.allocatedOpportunisticVCores = allocatedOpportunisticVCores; + } + public void setReservedMB(long reservedMB) { this.reservedMB = reservedMB; } @@ -594,12 +638,20 @@ public class AppInfo { this.runningContainers = runningContainers; } - public void setMemorySeconds(long memorySeconds) { - this.memorySeconds = memorySeconds; + public void setGuaranteedMemorySeconds(long guaranteedMemorySeconds) { + this.memorySeconds = guaranteedMemorySeconds; + } + + public void setGuaranteedVcoreSeconds(long guaranteedVcoreSeconds) { + this.vcoreSeconds = guaranteedVcoreSeconds; + } + + public void setOpportunisticMemorySeconds(long oppMemorySeconds) { + this.opportunisticMemorySeconds = oppMemorySeconds; } - public void setVcoreSeconds(long vcoreSeconds) { - this.vcoreSeconds = vcoreSeconds; + public void setOpportunisticVcoreSeconds(long oppVcoreSeconds) { + this.opportunisticVcoreSeconds = oppVcoreSeconds; } public void setAppId(String appId) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto index 39a56a8..1721b1a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto @@ -89,6 +89,7 @@ message ApplicationAttemptStateDataProto { optional int64 preempted_vcore_seconds = 14; repeated StringLongMapProto application_resource_usage_map = 15; repeated StringLongMapProto preempted_resource_usage_map = 16; + repeated StringLongMapProto application_opportunistic_resource_usage_map = 17; } message EpochProto { http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index 5296dc9..7881ded 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -1044,12 +1044,20 @@ public class TestAppManager{ when(app.getState()).thenReturn(RMAppState.RUNNING); when(app.getApplicationType()).thenReturn("MAPREDUCE"); when(app.getSubmitTime()).thenReturn(1000L); - Map<String, Long> resourceSecondsMap = new HashMap<>(); - resourceSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), 16384L); - resourceSecondsMap.put(ResourceInformation.VCORES.getName(), 64L); + Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>(); + guaranteedResourceSecondsMap.put( + ResourceInformation.MEMORY_MB.getName(), 16384L); + guaranteedResourceSecondsMap.put( + ResourceInformation.VCORES.getName(), 64L); + Map<String, Long> opportunisticResourceSecondsMap = new HashMap<>(); + opportunisticResourceSecondsMap.put( + ResourceInformation.MEMORY_MB.getName(), 16384L); + opportunisticResourceSecondsMap.put( + ResourceInformation.VCORES.getName(), 64L); RMAppMetrics metrics = new RMAppMetrics(Resource.newInstance(1234, 56), - 10, 1, resourceSecondsMap, new HashMap<>()); + 10, 1, guaranteedResourceSecondsMap, + new HashMap<>(), opportunisticResourceSecondsMap); when(app.getRMAppMetrics()).thenReturn(metrics); RMAppManager.ApplicationSummary.SummaryBuilder summary = @@ -1060,16 +1068,19 @@ public class TestAppManager{ Assert.assertFalse(msg.contains("\r")); String escaped = "\\n\\n\\r\\r"; - assertTrue(msg.contains("Multiline" + escaped +"AppName")); - assertTrue(msg.contains("Multiline" + escaped +"UserName")); - assertTrue(msg.contains("Multiline" + escaped +"QueueName")); - assertTrue(msg.contains("submitTime=1000")); - assertTrue(msg.contains("memorySeconds=16384")); - assertTrue(msg.contains("vcoreSeconds=64")); - assertTrue(msg.contains("preemptedAMContainers=1")); - assertTrue(msg.contains("preemptedNonAMContainers=10")); - assertTrue(msg.contains("preemptedResources=<memory:1234\\, vCores:56>")); - assertTrue(msg.contains("applicationType=MAPREDUCE")); + Assert.assertTrue(msg.contains("Multiline" + escaped +"AppName")); + Assert.assertTrue(msg.contains("Multiline" + escaped +"UserName")); + Assert.assertTrue(msg.contains("Multiline" + escaped +"QueueName")); + Assert.assertTrue(msg.contains("submitTime=1000")); + Assert.assertTrue(msg.contains("guaranteedMemorySeconds=16384")); + Assert.assertTrue(msg.contains("guaranteedVcoreSeconds=64")); + Assert.assertTrue(msg.contains("opportunisticMemorySeconds=16384")); + Assert.assertTrue(msg.contains("opportunisticVcoreSeconds=64")); + Assert.assertTrue(msg.contains("preemptedAMContainers=1")); + Assert.assertTrue(msg.contains("preemptedNonAMContainers=10")); + Assert.assertTrue( + msg.contains("preemptedResources=<memory:1234\\, vCores:56>")); + Assert.assertTrue(msg.contains("applicationType=MAPREDUCE")); } @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java index c7ed02c..698ca90 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java @@ -426,9 +426,11 @@ public class TestApplicationACLs extends ParameterizedSchedulerTestBase { Assert.assertEquals("Enemy should not see app reserved containers", -1, usageReport.getNumReservedContainers()); Assert.assertEquals("Enemy should not see app used resources", - -1, usageReport.getUsedResources().getMemorySize()); + -1, usageReport.getGuaranteedResourcesUsed().getMemorySize()); Assert.assertEquals("Enemy should not see app reserved resources", -1, usageReport.getReservedResources().getMemorySize()); + Assert.assertEquals("Enemy should not see app used opportunistic resources", + -1, usageReport.getOpportunisticResourcesUsed().getMemorySize()); Assert.assertEquals("Enemy should not see app needed resources", -1, usageReport.getNeededResources().getMemorySize()); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 1c50dd3..a66a91b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -370,8 +370,8 @@ public class TestClientRMService { ApplicationReport report = response.getApplicationReport(); ApplicationResourceUsageReport usageReport = report.getApplicationResourceUsageReport(); - Assert.assertEquals(10, usageReport.getMemorySeconds()); - Assert.assertEquals(3, usageReport.getVcoreSeconds()); + Assert.assertEquals(10, usageReport.getGuaranteedMemorySeconds()); + Assert.assertEquals(3, usageReport.getGuaranteedVcoreSeconds()); Assert.assertEquals("<Not set>", report.getAmNodeLabelExpression()); Assert.assertEquals("<Not set>", report.getAppNodeLabelExpression()); @@ -1402,26 +1402,28 @@ public class TestClientRMService { when(asContext.getMaxAppAttempts()).thenReturn(1); when(asContext.getNodeLabelExpression()).thenReturn(appNodeLabelExpression); when(asContext.getPriority()).thenReturn(Priority.newInstance(0)); - RMAppImpl app = - spy(new RMAppImpl(applicationId3, rmContext, config, null, null, + ResourceRequest resourceRequest = BuilderUtils.newResourceRequest( + RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, + Resource.newInstance(1024, 1), 1); + RMAppImpl rmApp = + new RMAppImpl(applicationId3, rmContext, config, null, null, queueName, asContext, yarnScheduler, null, System.currentTimeMillis(), "YARN", null, - Collections.singletonList(BuilderUtils.newResourceRequest( - RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, - Resource.newInstance(1024, 1), 1))){ - @Override - public ApplicationReport createAndGetApplicationReport( - String clientUserName, boolean allowAccess) { - ApplicationReport report = super.createAndGetApplicationReport( - clientUserName, allowAccess); - ApplicationResourceUsageReport usageReport = - report.getApplicationResourceUsageReport(); - usageReport.setMemorySeconds(memorySeconds); - usageReport.setVcoreSeconds(vcoreSeconds); - report.setApplicationResourceUsageReport(usageReport); - return report; - } - }); + Collections.singletonList(resourceRequest)) { + @Override + public ApplicationReport createAndGetApplicationReport( + String clientUserName, boolean allowAccess) { + ApplicationReport report = super.createAndGetApplicationReport( + clientUserName, allowAccess); + ApplicationResourceUsageReport usageReport = + report.getApplicationResourceUsageReport(); + usageReport.setGuaranteedMemorySeconds(memorySeconds); + usageReport.setGuaranteedVcoreSeconds(vcoreSeconds); + report.setApplicationResourceUsageReport(usageReport); + return report; + } + }; + RMAppImpl app = spy(rmApp); app.getAMResourceRequests().get(0) .setNodeLabelExpression(amNodeLabelExpression); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance( @@ -1474,7 +1476,8 @@ public class TestClientRMService { when(yarnScheduler.getAppsInQueue(QUEUE_2)).thenReturn( Arrays.asList(getApplicationAttemptId(103))); ApplicationAttemptId attemptId = getApplicationAttemptId(1); - when(yarnScheduler.getAppResourceUsageReport(attemptId)).thenReturn(null); + when(yarnScheduler.getAppActiveResourceUsageReport(attemptId)) + .thenReturn(null); ResourceCalculator rs = mock(ResourceCalculator.class); when(yarnScheduler.getResourceCalculator()).thenReturn(rs); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org