YARN-415. Capture aggregate memory allocation at the app-level for chargeback. Contributed by Eric Payne & Andrey Klochkov
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/83be3ad4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/83be3ad4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/83be3ad4 Branch: refs/heads/HDFS-6584 Commit: 83be3ad44484bf8a24cb90de4b9c26ab59d226a8 Parents: 5ec7fcd Author: Jian He <jia...@apache.org> Authored: Wed Sep 10 18:19:53 2014 -0700 Committer: Jian He <jia...@apache.org> Committed: Wed Sep 10 18:20:54 2014 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 + .../records/ApplicationResourceUsageReport.java | 41 +- .../src/main/proto/yarn_protos.proto | 2 + .../hadoop/yarn/client/cli/ApplicationCLI.java | 6 + .../hadoop/yarn/client/cli/TestYarnCLI.java | 8 +- .../ApplicationResourceUsageReportPBImpl.java | 24 ++ .../hadoop/yarn/server/utils/BuilderUtils.java | 5 +- .../server/resourcemanager/RMServerUtils.java | 2 +- .../recovery/FileSystemRMStateStore.java | 4 +- .../recovery/MemoryRMStateStore.java | 9 +- .../resourcemanager/recovery/RMStateStore.java | 24 +- .../recovery/ZKRMStateStore.java | 4 +- .../records/ApplicationAttemptStateData.java | 32 +- .../pb/ApplicationAttemptStateDataPBImpl.java | 24 ++ .../server/resourcemanager/rmapp/RMAppImpl.java | 17 +- .../resourcemanager/rmapp/RMAppMetrics.java | 15 +- .../attempt/AggregateAppResourceUsage.java | 60 +++ .../rmapp/attempt/RMAppAttemptImpl.java | 16 +- .../rmapp/attempt/RMAppAttemptMetrics.java | 36 +- .../rmcontainer/RMContainerImpl.java | 26 +- .../scheduler/SchedulerApplicationAttempt.java | 41 +- .../scheduler/common/fica/FiCaSchedulerApp.java | 3 + .../scheduler/fair/FSAppAttempt.java | 3 + .../server/resourcemanager/webapp/AppBlock.java | 5 +- .../resourcemanager/webapp/dao/AppInfo.java | 12 + .../yarn_server_resourcemanager_recovery.proto | 4 +- .../resourcemanager/TestClientRMService.java | 60 ++- .../TestContainerResourceUsage.java | 401 +++++++++++++++++++ .../applicationsmanager/MockAsm.java | 19 +- .../recovery/RMStateStoreTestBase.java | 18 +- .../rmapp/TestRMAppTransitions.java | 16 +- .../attempt/TestRMAppAttemptTransitions.java | 72 +++- .../rmcontainer/TestRMContainerImpl.java | 11 + .../scheduler/capacity/TestLeafQueue.java | 89 ++-- .../scheduler/fair/FairSchedulerTestBase.java | 4 +- .../webapp/TestRMWebAppFairScheduler.java | 2 +- .../webapp/TestRMWebServicesApps.java | 2 +- .../src/site/apt/ResourceManagerRest.apt.vm | 22 +- 38 files changed, 1045 insertions(+), 97 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 7968071..b28f8f8 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -67,6 +67,9 @@ Release 2.6.0 - UNRELEASED YARN-2394. FairScheduler: Configure fairSharePreemptionThreshold per queue. (Wei Yan via kasha) + YARN-415. Capture aggregate memory allocation at the app-level for chargeback. + (Eric Payne & Andrey Klochkov via jianhe) + IMPROVEMENTS YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java index 6e9c76f..b20d832 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java @@ -35,7 +35,8 @@ public abstract class ApplicationResourceUsageReport { @Unstable public static ApplicationResourceUsageReport newInstance( int numUsedContainers, int numReservedContainers, Resource usedResources, - Resource reservedResources, Resource neededResources) { + Resource reservedResources, Resource neededResources, long memorySeconds, + long vcoreSeconds) { ApplicationResourceUsageReport report = Records.newRecord(ApplicationResourceUsageReport.class); report.setNumUsedContainers(numUsedContainers); @@ -43,6 +44,8 @@ public abstract class ApplicationResourceUsageReport { report.setUsedResources(usedResources); report.setReservedResources(reservedResources); report.setNeededResources(neededResources); + report.setMemorySeconds(memorySeconds); + report.setVcoreSeconds(vcoreSeconds); return report; } @@ -113,4 +116,40 @@ public abstract class ApplicationResourceUsageReport { @Private @Unstable public abstract void setNeededResources(Resource needed_resources); + + /** + * Set the aggregated amount of memory (in megabytes) the application has + * allocated times the number of seconds the application has been running. + * @param memory_seconds the aggregated amount of memory seconds + */ + @Private + @Unstable + public abstract void setMemorySeconds(long memory_seconds); + + /** + * Get the aggregated amount of memory (in megabytes) the application has + * allocated times the number of seconds the application has been running. + * @return the aggregated amount of memory seconds + */ + @Public + @Unstable + public abstract long getMemorySeconds(); + + /** + * Set the aggregated number of vcores that the application has allocated + * times the number of seconds the application has been running. + * @param vcore_seconds the aggregated number of vcore seconds + */ + @Private + @Unstable + public abstract void setVcoreSeconds(long vcore_seconds); + + /** + * Get the aggregated number of vcores that the application has allocated + * times the number of seconds the application has been running. + * @return the aggregated number of vcore seconds + */ + @Public + @Unstable + public abstract long getVcoreSeconds(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 3f1fa6c..7e7f21b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -167,6 +167,8 @@ message ApplicationResourceUsageReportProto { optional ResourceProto used_resources = 3; optional ResourceProto reserved_resources = 4; optional ResourceProto needed_resources = 5; + optional int64 memory_seconds = 6; + optional int64 vcore_seconds = 7; } message ApplicationReportProto { http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 7d61a23..54cfe91 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -38,6 +38,7 @@ import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; @@ -460,6 +461,11 @@ public class ApplicationCLI extends YarnCLI { appReportStr.println(appReport.getRpcPort()); appReportStr.print("\tAM Host : "); appReportStr.println(appReport.getHost()); + appReportStr.print("\tAggregate Resource Allocation : "); + + ApplicationResourceUsageReport usageReport = appReport.getApplicationResourceUsageReport(); + appReportStr.print(usageReport.getMemorySeconds() + " MB-seconds, "); + appReportStr.println(usageReport.getVcoreSeconds() + " vcore-seconds"); appReportStr.print("\tDiagnostics : "); appReportStr.print(appReport.getDiagnostics()); } else { http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index b408b61..47fa5ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -87,11 +88,15 @@ public class TestYarnCLI { public void testGetApplicationReport() throws Exception { ApplicationCLI cli = createAndGetAppCLI(); ApplicationId applicationId = ApplicationId.newInstance(1234, 5); + ApplicationResourceUsageReport usageReport = + ApplicationResourceUsageReport.newInstance( + 2, 0, null, null, null, 123456, 4567); ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, - FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); + FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN", + null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport); int result = cli.run(new String[] { "application", "-status", applicationId.toString() }); @@ -113,6 +118,7 @@ public class TestYarnCLI { pw.println("\tTracking-URL : N/A"); pw.println("\tRPC Port : 124"); pw.println("\tAM Host : host"); + pw.println("\tAggregate Resource Allocation : 123456 MB-seconds, 4567 vcore-seconds"); pw.println("\tDiagnostics : diagnostics"); pw.close(); String appReportStr = baos.toString("UTF-8"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java index ada7165..1cabadd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java @@ -200,6 +200,30 @@ extends ApplicationResourceUsageReport { this.neededResources = reserved_resources; } + @Override + public synchronized void setMemorySeconds(long memory_seconds) { + maybeInitBuilder(); + builder.setMemorySeconds(memory_seconds); + } + + @Override + public synchronized long getMemorySeconds() { + ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder; + return p.getMemorySeconds(); + } + + @Override + public synchronized void setVcoreSeconds(long vcore_seconds) { + maybeInitBuilder(); + builder.setVcoreSeconds(vcore_seconds); + } + + @Override + public synchronized long getVcoreSeconds() { + ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder; + return (p.getVcoreSeconds()); + } + private ResourcePBImpl convertFromProtoFormat(ResourceProto p) { return new ResourcePBImpl(p); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java index 64eb428..0cfd911 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java @@ -370,7 +370,8 @@ public class BuilderUtils { public static ApplicationResourceUsageReport newApplicationResourceUsageReport( int numUsedContainers, int numReservedContainers, Resource usedResources, - Resource reservedResources, Resource neededResources) { + Resource reservedResources, Resource neededResources, long memorySeconds, + long vcoreSeconds) { ApplicationResourceUsageReport report = recordFactory.newRecordInstance(ApplicationResourceUsageReport.class); report.setNumUsedContainers(numUsedContainers); @@ -378,6 +379,8 @@ public class BuilderUtils { report.setUsedResources(usedResources); report.setReservedResources(reservedResources); report.setNeededResources(neededResources); + report.setMemorySeconds(memorySeconds); + report.setVcoreSeconds(vcoreSeconds); return report; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index d93c45d..29c5953 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -236,5 +236,5 @@ public class RMServerUtils { DUMMY_APPLICATION_RESOURCE_USAGE_REPORT = BuilderUtils.newApplicationResourceUsageReport(-1, -1, Resources.createResource(-1, -1), Resources.createResource(-1, -1), - Resources.createResource(-1, -1)); + Resources.createResource(-1, -1), 0, 0); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java index 0a3b269..4c01a61 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java @@ -280,7 +280,9 @@ public class FileSystemRMStateStore extends RMStateStore { attemptStateData.getFinalTrackingUrl(), attemptStateData.getDiagnostics(), attemptStateData.getFinalApplicationStatus(), - attemptStateData.getAMContainerExitStatus()); + attemptStateData.getAMContainerExitStatus(), + attemptStateData.getMemorySeconds(), + attemptStateData.getVcoreSeconds()); // assert child node name is same as application attempt id assert attemptId.equals(attemptState.getAttemptId()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java index f56517c..efaa039 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java @@ -138,7 +138,10 @@ public class MemoryRMStateStore extends RMStateStore { ApplicationAttemptState attemptState = new ApplicationAttemptState(appAttemptId, attemptStateData.getMasterContainer(), credentials, - attemptStateData.getStartTime()); + attemptStateData.getStartTime(), + attemptStateData.getMemorySeconds(), + attemptStateData.getVcoreSeconds()); + ApplicationState appState = state.getApplicationState().get( attemptState.getAttemptId().getApplicationId()); @@ -167,7 +170,9 @@ public class MemoryRMStateStore extends RMStateStore { attemptStateData.getFinalTrackingUrl(), attemptStateData.getDiagnostics(), attemptStateData.getFinalApplicationStatus(), - attemptStateData.getAMContainerExitStatus()); + attemptStateData.getAMContainerExitStatus(), + attemptStateData.getMemorySeconds(), + attemptStateData.getVcoreSeconds()); ApplicationState appState = state.getApplicationState().get( http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java index 714a108..3074d33 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java @@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; @@ -265,19 +266,21 @@ public abstract class RMStateStore extends AbstractService { String diagnostics; int exitStatus = ContainerExitStatus.INVALID; FinalApplicationStatus amUnregisteredFinalStatus; + long memorySeconds; + long vcoreSeconds; public ApplicationAttemptState(ApplicationAttemptId attemptId, Container masterContainer, Credentials appAttemptCredentials, - long startTime) { + long startTime, long memorySeconds, long vcoreSeconds) { this(attemptId, masterContainer, appAttemptCredentials, startTime, null, - null, "", null, ContainerExitStatus.INVALID); + null, "", null, ContainerExitStatus.INVALID, memorySeconds, vcoreSeconds); } public ApplicationAttemptState(ApplicationAttemptId attemptId, Container masterContainer, Credentials appAttemptCredentials, long startTime, RMAppAttemptState state, String finalTrackingUrl, String diagnostics, FinalApplicationStatus amUnregisteredFinalStatus, - int exitStatus) { + int exitStatus, long memorySeconds, long vcoreSeconds) { this.attemptId = attemptId; this.masterContainer = masterContainer; this.appAttemptCredentials = appAttemptCredentials; @@ -287,6 +290,8 @@ public abstract class RMStateStore extends AbstractService { this.diagnostics = diagnostics == null ? "" : diagnostics; this.amUnregisteredFinalStatus = amUnregisteredFinalStatus; this.exitStatus = exitStatus; + this.memorySeconds = memorySeconds; + this.vcoreSeconds = vcoreSeconds; } public Container getMasterContainer() { @@ -316,6 +321,12 @@ public abstract class RMStateStore extends AbstractService { public int getAMContainerExitStatus(){ return this.exitStatus; } + public long getMemorySeconds() { + return memorySeconds; + } + public long getVcoreSeconds() { + return vcoreSeconds; + } } /** @@ -587,10 +598,13 @@ public abstract class RMStateStore extends AbstractService { public synchronized void storeNewApplicationAttempt(RMAppAttempt appAttempt) { Credentials credentials = getCredentialsFromAppAttempt(appAttempt); + AggregateAppResourceUsage resUsage = + appAttempt.getRMAppAttemptMetrics().getAggregateAppResourceUsage(); ApplicationAttemptState attemptState = new ApplicationAttemptState(appAttempt.getAppAttemptId(), appAttempt.getMasterContainer(), credentials, - appAttempt.getStartTime()); + appAttempt.getStartTime(), resUsage.getMemorySeconds(), + resUsage.getVcoreSeconds()); dispatcher.getEventHandler().handle( new RMStateStoreAppAttemptEvent(attemptState)); @@ -746,7 +760,7 @@ public abstract class RMStateStore extends AbstractService { ApplicationAttemptState attemptState = new ApplicationAttemptState(appAttempt.getAppAttemptId(), appAttempt.getMasterContainer(), credentials, - appAttempt.getStartTime()); + appAttempt.getStartTime(), 0, 0); appState.attempts.put(attemptState.getAttemptId(), attemptState); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java index 1b1ec76..25f3819 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java @@ -603,7 +603,9 @@ public class ZKRMStateStore extends RMStateStore { attemptStateData.getFinalTrackingUrl(), attemptStateData.getDiagnostics(), attemptStateData.getFinalApplicationStatus(), - attemptStateData.getAMContainerExitStatus()); + attemptStateData.getAMContainerExitStatus(), + attemptStateData.getMemorySeconds(), + attemptStateData.getVcoreSeconds()); appState.attempts.put(attemptState.getAttemptId(), attemptState); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java index 5cb9787..ad8cdae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java @@ -43,7 +43,8 @@ public abstract class ApplicationAttemptStateData { ApplicationAttemptId attemptId, Container container, ByteBuffer attemptTokens, long startTime, RMAppAttemptState finalState, String finalTrackingUrl, String diagnostics, - FinalApplicationStatus amUnregisteredFinalStatus, int exitStatus) { + FinalApplicationStatus amUnregisteredFinalStatus, int exitStatus, + long memorySeconds, long vcoreSeconds) { ApplicationAttemptStateData attemptStateData = Records.newRecord(ApplicationAttemptStateData.class); attemptStateData.setAttemptId(attemptId); @@ -55,6 +56,8 @@ public abstract class ApplicationAttemptStateData { attemptStateData.setStartTime(startTime); attemptStateData.setFinalApplicationStatus(amUnregisteredFinalStatus); attemptStateData.setAMContainerExitStatus(exitStatus); + attemptStateData.setMemorySeconds(memorySeconds); + attemptStateData.setVcoreSeconds(vcoreSeconds); return attemptStateData; } @@ -72,7 +75,8 @@ public abstract class ApplicationAttemptStateData { attemptState.getStartTime(), attemptState.getState(), attemptState.getFinalTrackingUrl(), attemptState.getDiagnostics(), attemptState.getFinalApplicationStatus(), - attemptState.getAMContainerExitStatus()); + attemptState.getAMContainerExitStatus(), + attemptState.getMemorySeconds(), attemptState.getVcoreSeconds()); } public abstract ApplicationAttemptStateDataProto getProto(); @@ -157,4 +161,28 @@ public abstract class ApplicationAttemptStateData { public abstract int getAMContainerExitStatus(); public abstract void setAMContainerExitStatus(int exitStatus); + + /** + * Get the <em>memory seconds</em> (in MB seconds) of the application. + * @return <em>memory seconds</em> (in MB seconds) of the application + */ + @Public + @Unstable + public abstract long getMemorySeconds(); + + @Public + @Unstable + public abstract void setMemorySeconds(long memorySeconds); + + /** + * Get the <em>vcore seconds</em> of the application. + * @return <em>vcore seconds</em> of the application + */ + @Public + @Unstable + public abstract long getVcoreSeconds(); + + @Public + @Unstable + public abstract void setVcoreSeconds(long vcoreSeconds); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java index 5c62d63..4d6212d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java @@ -229,6 +229,30 @@ public class ApplicationAttemptStateDataPBImpl extends } @Override + public long getMemorySeconds() { + ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder; + return p.getMemorySeconds(); + } + + @Override + public long getVcoreSeconds() { + ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder; + return p.getVcoreSeconds(); + } + + @Override + public void setMemorySeconds(long memorySeconds) { + maybeInitBuilder(); + builder.setMemorySeconds(memorySeconds); + } + + @Override + public void setVcoreSeconds(long vcoreSeconds) { + maybeInitBuilder(); + builder.setVcoreSeconds(vcoreSeconds); + } + + @Override public FinalApplicationStatus getFinalApplicationStatus() { ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder; if (!p.hasFinalApplicationStatus()) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 0b81f96..5b6df00 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.Appli import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppNodeUpdateEvent.RMAppNodeUpdateType; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; @@ -561,6 +562,10 @@ public class RMAppImpl implements RMApp, Recoverable { } } } + + RMAppMetrics rmAppMetrics = getRMAppMetrics(); + appUsageReport.setMemorySeconds(rmAppMetrics.getMemorySeconds()); + appUsageReport.setVcoreSeconds(rmAppMetrics.getVcoreSeconds()); } if (currentApplicationAttemptId == null) { @@ -1117,7 +1122,6 @@ public class RMAppImpl implements RMApp, Recoverable { @Override public RMAppState transition(RMAppImpl app, RMAppEvent event) { - if (!app.submissionContext.getUnmanagedAM() && app.getNumFailedAppAttempts() < app.maxAppAttempts) { boolean transferStateFromPreviousAttempt = false; @@ -1199,6 +1203,8 @@ public class RMAppImpl implements RMApp, Recoverable { Resource resourcePreempted = Resource.newInstance(0, 0); int numAMContainerPreempted = 0; int numNonAMContainerPreempted = 0; + long memorySeconds = 0; + long vcoreSeconds = 0; for (RMAppAttempt attempt : attempts.values()) { if (null != attempt) { RMAppAttemptMetrics attemptMetrics = @@ -1208,10 +1214,17 @@ public class RMAppImpl implements RMApp, Recoverable { numAMContainerPreempted += attemptMetrics.getIsPreempted() ? 1 : 0; numNonAMContainerPreempted += attemptMetrics.getNumNonAMContainersPreempted(); + // getAggregateAppResourceUsage() will calculate resource usage stats + // for both running and finished containers. + AggregateAppResourceUsage resUsage = + attempt.getRMAppAttemptMetrics().getAggregateAppResourceUsage(); + memorySeconds += resUsage.getMemorySeconds(); + vcoreSeconds += resUsage.getVcoreSeconds(); } } return new RMAppMetrics(resourcePreempted, - numNonAMContainerPreempted, numAMContainerPreempted); + numNonAMContainerPreempted, numAMContainerPreempted, + memorySeconds, vcoreSeconds); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java index 645db16..5091470 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java @@ -24,12 +24,17 @@ public class RMAppMetrics { final Resource resourcePreempted; final int numNonAMContainersPreempted; final int numAMContainersPreempted; + final long memorySeconds; + final long vcoreSeconds; public RMAppMetrics(Resource resourcePreempted, - int numNonAMContainersPreempted, int numAMContainersPreempted) { + int numNonAMContainersPreempted, int numAMContainersPreempted, + long memorySeconds, long vcoreSeconds) { this.resourcePreempted = resourcePreempted; this.numNonAMContainersPreempted = numNonAMContainersPreempted; this.numAMContainersPreempted = numAMContainersPreempted; + this.memorySeconds = memorySeconds; + this.vcoreSeconds = vcoreSeconds; } public Resource getResourcePreempted() { @@ -43,4 +48,12 @@ public class RMAppMetrics { public int getNumAMContainersPreempted() { return numAMContainersPreempted; } + + public long getMemorySeconds() { + return memorySeconds; + } + + public long getVcoreSeconds() { + return vcoreSeconds; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java new file mode 100644 index 0000000..f0c2b34 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt; + +import org.apache.hadoop.classification.InterfaceAudience.Private; + +@Private +public class AggregateAppResourceUsage { + long memorySeconds; + long vcoreSeconds; + + public AggregateAppResourceUsage(long memorySeconds, long vcoreSeconds) { + this.memorySeconds = memorySeconds; + this.vcoreSeconds = vcoreSeconds; + } + + /** + * @return the memorySeconds + */ + public long getMemorySeconds() { + return memorySeconds; + } + + /** + * @param memorySeconds the memorySeconds to set + */ + public void setMemorySeconds(long memorySeconds) { + this.memorySeconds = memorySeconds; + } + + /** + * @return the vcoreSeconds + */ + public long getVcoreSeconds() { + return vcoreSeconds; + } + + /** + * @param vcoreSeconds the vcoreSeconds to set + */ + public void setVcoreSeconds(long vcoreSeconds) { + this.vcoreSeconds = vcoreSeconds; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 19fc800..93db340 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -85,6 +85,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAt import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; @@ -430,7 +431,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { this.proxiedTrackingUrl = generateProxyUriWithScheme(null); this.maybeLastAttempt = maybeLastAttempt; this.stateMachine = stateMachineFactory.make(this); - this.attemptMetrics = new RMAppAttemptMetrics(applicationAttemptId); + this.attemptMetrics = + new RMAppAttemptMetrics(applicationAttemptId, rmContext); } @Override @@ -704,6 +706,10 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { if (report == null) { report = RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT; } + AggregateAppResourceUsage resUsage = + this.attemptMetrics.getAggregateAppResourceUsage(); + report.setMemorySeconds(resUsage.getMemorySeconds()); + report.setVcoreSeconds(resUsage.getVcoreSeconds()); return report; } finally { this.readLock.unlock(); @@ -733,6 +739,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { this.proxiedTrackingUrl = generateProxyUriWithScheme(originalTrackingUrl); this.finalStatus = attemptState.getFinalApplicationStatus(); this.startTime = attemptState.getStartTime(); + this.attemptMetrics.updateAggregateAppResourceUsage( + attemptState.getMemorySeconds(),attemptState.getVcoreSeconds()); } public void transferStateFromPreviousAttempt(RMAppAttempt attempt) { @@ -1017,12 +1025,14 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { default: break; } - + AggregateAppResourceUsage resUsage = + this.attemptMetrics.getAggregateAppResourceUsage(); RMStateStore rmStore = rmContext.getStateStore(); ApplicationAttemptState attemptState = new ApplicationAttemptState(applicationAttemptId, getMasterContainer(), rmStore.getCredentialsFromAppAttempt(this), startTime, - stateToBeStored, finalTrackingUrl, diags, finalStatus, exitStatus); + stateToBeStored, finalTrackingUrl, diags, finalStatus, exitStatus, + resUsage.getMemorySeconds(), resUsage.getVcoreSeconds()); LOG.info("Updating application attempt " + applicationAttemptId + " with final state: " + targetedFinalState + ", and exit status: " + exitStatus); http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java index 96b41c3..0e60fd5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; @@ -27,7 +28,9 @@ import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.util.resource.Resources; @@ -42,12 +45,17 @@ public class RMAppAttemptMetrics { private ReadLock readLock; private WriteLock writeLock; - - public RMAppAttemptMetrics(ApplicationAttemptId attemptId) { + private AtomicLong finishedMemorySeconds = new AtomicLong(0); + private AtomicLong finishedVcoreSeconds = new AtomicLong(0); + private RMContext rmContext; + + public RMAppAttemptMetrics(ApplicationAttemptId attemptId, + RMContext rmContext) { this.attemptId = attemptId; ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); this.readLock = lock.readLock(); this.writeLock = lock.writeLock(); + this.rmContext = rmContext; } public void updatePreemptionInfo(Resource resource, RMContainer container) { @@ -94,4 +102,28 @@ public class RMAppAttemptMetrics { public boolean getIsPreempted() { return this.isPreempted.get(); } + + public AggregateAppResourceUsage getAggregateAppResourceUsage() { + long memorySeconds = finishedMemorySeconds.get(); + long vcoreSeconds = finishedVcoreSeconds.get(); + + // Only add in the running containers if this is the active attempt. + RMAppAttempt currentAttempt = rmContext.getRMApps() + .get(attemptId.getApplicationId()).getCurrentAppAttempt(); + if (currentAttempt.getAppAttemptId().equals(attemptId)) { + ApplicationResourceUsageReport appResUsageReport = rmContext + .getScheduler().getAppResourceUsageReport(attemptId); + if (appResUsageReport != null) { + memorySeconds += appResUsageReport.getMemorySeconds(); + vcoreSeconds += appResUsageReport.getVcoreSeconds(); + } + } + return new AggregateAppResourceUsage(memorySeconds, vcoreSeconds); + } + + public void updateAggregateAppResourceUsage(long finishedMemorySeconds, + long finishedVcoreSeconds) { + this.finishedMemorySeconds.addAndGet(finishedMemorySeconds); + this.finishedVcoreSeconds.addAndGet(finishedVcoreSeconds); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java index eef361f..e7bb98e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java @@ -24,6 +24,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; +import org.apache.commons.lang.time.DateUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -40,6 +41,7 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRunningOnNodeEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerAllocatedEvent; @@ -488,7 +490,7 @@ public class RMContainerImpl implements RMContainer { // Inform AppAttempt // container.getContainer() can return null when a RMContainer is a // reserved container - updateMetricsIfPreempted(container); + updateAttemptMetrics(container); container.eventHandler.handle(new RMAppAttemptContainerFinishedEvent( container.appAttemptId, finishedEvent.getRemoteContainerStatus())); @@ -497,19 +499,27 @@ public class RMContainerImpl implements RMContainer { container); } - private static void updateMetricsIfPreempted(RMContainerImpl container) { + private static void updateAttemptMetrics(RMContainerImpl container) { // If this is a preempted container, update preemption metrics + Resource resource = container.getContainer().getResource(); + RMAppAttempt rmAttempt = container.rmContext.getRMApps() + .get(container.getApplicationAttemptId().getApplicationId()) + .getCurrentAppAttempt(); if (ContainerExitStatus.PREEMPTED == container.finishedStatus .getExitStatus()) { - - Resource resource = container.getContainer().getResource(); - RMAppAttempt rmAttempt = - container.rmContext.getRMApps() - .get(container.getApplicationAttemptId().getApplicationId()) - .getCurrentAppAttempt(); rmAttempt.getRMAppAttemptMetrics().updatePreemptionInfo(resource, container); } + + if (rmAttempt != null) { + long usedMillis = container.finishTime - container.creationTime; + long memorySeconds = resource.getMemory() + * usedMillis / DateUtils.MILLIS_PER_SECOND; + long vcoreSeconds = resource.getVirtualCores() + * usedMillis / DateUtils.MILLIS_PER_SECOND; + rmAttempt.getRMAppAttemptMetrics() + .updateAggregateAppResourceUsage(memorySeconds,vcoreSeconds); + } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index 933f456..7032e3c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.commons.lang.time.DateUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -43,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent; @@ -69,6 +71,11 @@ public class SchedulerApplicationAttempt { private static final Log LOG = LogFactory .getLog(SchedulerApplicationAttempt.class); + private static final long MEM_AGGREGATE_ALLOCATION_CACHE_MSECS = 3000; + protected long lastMemoryAggregateAllocationUpdateTime = 0; + private long lastMemorySeconds = 0; + private long lastVcoreSeconds = 0; + protected final AppSchedulingInfo appSchedulingInfo; protected Map<ContainerId, RMContainer> liveContainers = @@ -505,12 +512,38 @@ public class SchedulerApplicationAttempt { lastScheduledContainer.put(priority, currentTimeMs); schedulingOpportunities.setCount(priority, 0); } - + + synchronized AggregateAppResourceUsage getRunningAggregateAppResourceUsage() { + long currentTimeMillis = System.currentTimeMillis(); + // Don't walk the whole container list if the resources were computed + // recently. + if ((currentTimeMillis - lastMemoryAggregateAllocationUpdateTime) + > MEM_AGGREGATE_ALLOCATION_CACHE_MSECS) { + long memorySeconds = 0; + long vcoreSeconds = 0; + for (RMContainer rmContainer : this.liveContainers.values()) { + long usedMillis = currentTimeMillis - rmContainer.getCreationTime(); + Resource resource = rmContainer.getContainer().getResource(); + memorySeconds += resource.getMemory() * usedMillis / + DateUtils.MILLIS_PER_SECOND; + vcoreSeconds += resource.getVirtualCores() * usedMillis + / DateUtils.MILLIS_PER_SECOND; + } + + lastMemoryAggregateAllocationUpdateTime = currentTimeMillis; + lastMemorySeconds = memorySeconds; + lastVcoreSeconds = vcoreSeconds; + } + return new AggregateAppResourceUsage(lastMemorySeconds, lastVcoreSeconds); + } + public synchronized ApplicationResourceUsageReport getResourceUsageReport() { + AggregateAppResourceUsage resUsage = getRunningAggregateAppResourceUsage(); return ApplicationResourceUsageReport.newInstance(liveContainers.size(), - reservedContainers.size(), Resources.clone(currentConsumption), - Resources.clone(currentReservation), - Resources.add(currentConsumption, currentReservation)); + reservedContainers.size(), Resources.clone(currentConsumption), + Resources.clone(currentReservation), + Resources.add(currentConsumption, currentReservation), + resUsage.getMemorySeconds(), resUsage.getVcoreSeconds()); } public synchronized Map<ContainerId, RMContainer> getLiveContainersMap() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index 846d1e1..167dcd8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -106,6 +106,9 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt { queue.getMetrics().releaseResources(getUser(), 1, containerResource); Resources.subtractFrom(currentConsumption, containerResource); + // Clear resource utilization metrics cache. + lastMemoryAggregateAllocationUpdateTime = -1; + return true; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index bf54376..825c398 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -146,6 +146,9 @@ public class FSAppAttempt extends SchedulerApplicationAttempt // remove from preemption map if it is completed preemptionMap.remove(rmContainer); + + // Clear resource utilization metrics cache. + lastMemoryAggregateAllocationUpdateTime = -1; } private synchronized void unreserveInternal( http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java index b9f2dda..c427ccf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java @@ -146,7 +146,10 @@ public class AppBlock extends HtmlBlock { attemptMetrics.getResourcePreempted()) ._("Number of Non-AM Containers Preempted from Current Attempt:", String.valueOf(attemptMetrics - .getNumNonAMContainersPreempted())); + .getNumNonAMContainersPreempted())) + ._("Aggregate Resource Allocation:", + String.format("%d MB-seconds, %d vcore-seconds", + appMerics.getMemorySeconds(), appMerics.getVcoreSeconds())); pdiv._(); Collection<RMAppAttempt> attempts = rmApp.getAppAttempts().values(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index c7354ce..2b0dedc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -79,6 +79,8 @@ public class AppInfo { protected int allocatedMB; protected int allocatedVCores; protected int runningContainers; + protected long memorySeconds; + protected long vcoreSeconds; // preemption info fields protected int preemptedResourceMB; @@ -165,6 +167,8 @@ public class AppInfo { appMetrics.getNumNonAMContainersPreempted(); preemptedResourceVCores = appMetrics.getResourcePreempted().getVirtualCores(); + memorySeconds = appMetrics.getMemorySeconds(); + vcoreSeconds = appMetrics.getVcoreSeconds(); } } @@ -287,4 +291,12 @@ public class AppInfo { public int getNumAMContainersPreempted() { return numAMContainerPreempted; } + + public long getMemorySeconds() { + return memorySeconds; + } + + public long getVcoreSeconds() { + return vcoreSeconds; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto index eab6af1..5125a27 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto @@ -78,6 +78,8 @@ message ApplicationAttemptStateDataProto { optional int64 start_time = 7; optional FinalApplicationStatusProto final_application_status = 8; optional int32 am_container_exit_status = 9 [default = -1000]; + optional int64 memory_seconds = 10; + optional int64 vcore_seconds = 11; } message EpochProto { @@ -87,4 +89,4 @@ message EpochProto { message AMRMTokenSecretManagerStateProto { optional MasterKeyProto current_master_key = 1; optional MasterKeyProto next_master_key = 2; -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 4f4da37..571c96f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRes import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; @@ -79,6 +80,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -223,7 +225,7 @@ public class TestClientRMService { } @Test - public void testGetApplicationReport() throws YarnException { + public void testNonExistingApplicationReport() throws YarnException { RMContext rmContext = mock(RMContext.class); when(rmContext.getRMApps()).thenReturn( new ConcurrentHashMap<ApplicationId, RMApp>()); @@ -242,6 +244,38 @@ public class TestClientRMService { + "' doesn't exist in RM."); } } + + @Test + public void testGetApplicationReport() throws Exception { + YarnScheduler yarnScheduler = mock(YarnScheduler.class); + RMContext rmContext = mock(RMContext.class); + mockRMContext(yarnScheduler, rmContext); + + ApplicationId appId1 = getApplicationId(1); + + ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class); + when( + mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), + ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true); + + ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler, + null, mockAclsManager, null, null); + try { + RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); + GetApplicationReportRequest request = recordFactory + .newRecordInstance(GetApplicationReportRequest.class); + request.setApplicationId(appId1); + GetApplicationReportResponse response = + rmService.getApplicationReport(request); + ApplicationReport report = response.getApplicationReport(); + ApplicationResourceUsageReport usageReport = + report.getApplicationResourceUsageReport(); + Assert.assertEquals(10, usageReport.getMemorySeconds()); + Assert.assertEquals(3, usageReport.getVcoreSeconds()); + } finally { + rmService.close(); + } + } @Test public void testGetApplicationAttemptReport() throws YarnException, @@ -1065,11 +1099,11 @@ public class TestClientRMService { ApplicationId applicationId3 = getApplicationId(3); YarnConfiguration config = new YarnConfiguration(); apps.put(applicationId1, getRMApp(rmContext, yarnScheduler, applicationId1, - config, "testqueue")); + config, "testqueue", 10, 3)); apps.put(applicationId2, getRMApp(rmContext, yarnScheduler, applicationId2, - config, "a")); + config, "a", 20, 2)); apps.put(applicationId3, getRMApp(rmContext, yarnScheduler, applicationId3, - config, "testqueue")); + config, "testqueue", 40, 5)); return apps; } @@ -1091,12 +1125,26 @@ public class TestClientRMService { } private RMAppImpl getRMApp(RMContext rmContext, YarnScheduler yarnScheduler, - ApplicationId applicationId3, YarnConfiguration config, String queueName) { + ApplicationId applicationId3, YarnConfiguration config, String queueName, + final long memorySeconds, final long vcoreSeconds) { ApplicationSubmissionContext asContext = mock(ApplicationSubmissionContext.class); when(asContext.getMaxAppAttempts()).thenReturn(1); RMAppImpl app = spy(new RMAppImpl(applicationId3, rmContext, config, null, null, queueName, asContext, yarnScheduler, null, - System.currentTimeMillis(), "YARN", null)); + System.currentTimeMillis(), "YARN", null) { + @Override + public ApplicationReport createAndGetApplicationReport( + String clientUserName, boolean allowAccess) { + ApplicationReport report = super.createAndGetApplicationReport( + clientUserName, allowAccess); + ApplicationResourceUsageReport usageReport = + report.getApplicationResourceUsageReport(); + usageReport.setMemorySeconds(memorySeconds); + usageReport.setVcoreSeconds(vcoreSeconds); + report.setApplicationResourceUsageReport(usageReport); + return report; + } + }); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance( ApplicationId.newInstance(123456, 1), 1); RMAppAttemptImpl rmAppAttemptImpl = spy(new RMAppAttemptImpl(attemptId,
