Repository: hadoop Updated Branches: refs/heads/trunk 3a9841953 -> 93eeb1316
YARN-4218. Metric for resource*time that was preempted. Contributed by Chang Li. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/93eeb131 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/93eeb131 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/93eeb131 Branch: refs/heads/trunk Commit: 93eeb13164707d0e3556c2bf737bd2ee09a335c6 Parents: 3a98419 Author: Eric Payne <epa...@apache.org> Authored: Thu Nov 10 22:35:12 2016 +0000 Committer: Eric Payne <epa...@apache.org> Committed: Thu Nov 10 22:35:12 2016 +0000 ---------------------------------------------------------------------- .../records/ApplicationResourceUsageReport.java | 43 +++++++++++++++++++- .../src/main/proto/yarn_protos.proto | 2 + .../hadoop/yarn/client/cli/ApplicationCLI.java | 7 ++++ .../hadoop/yarn/client/cli/TestYarnCLI.java | 4 +- .../ApplicationResourceUsageReportPBImpl.java | 28 +++++++++++++ ...pplicationHistoryManagerOnTimelineStore.java | 8 +++- ...pplicationHistoryManagerOnTimelineStore.java | 11 ++++- .../metrics/ApplicationMetricsConstants.java | 6 +++ .../hadoop/yarn/server/utils/BuilderUtils.java | 5 ++- .../server/resourcemanager/RMAppManager.java | 2 + .../server/resourcemanager/RMServerUtils.java | 2 +- .../metrics/TimelineServiceV1Publisher.java | 4 ++ .../metrics/TimelineServiceV2Publisher.java | 6 +++ .../resourcemanager/recovery/RMStateStore.java | 10 +++-- .../records/ApplicationAttemptStateData.java | 39 ++++++++++++++++-- .../pb/ApplicationAttemptStateDataPBImpl.java | 24 +++++++++++ .../server/resourcemanager/rmapp/RMAppImpl.java | 13 +++++- .../resourcemanager/rmapp/RMAppMetrics.java | 16 +++++++- .../rmapp/attempt/RMAppAttemptImpl.java | 11 ++++- .../rmapp/attempt/RMAppAttemptMetrics.java | 16 ++++++++ .../rmcontainer/RMContainerImpl.java | 16 ++++---- .../scheduler/SchedulerApplicationAttempt.java | 2 +- .../resourcemanager/webapp/RMAppBlock.java | 8 +++- .../resourcemanager/webapp/dao/AppInfo.java | 11 +++++ .../yarn_server_resourcemanager_recovery.proto | 2 + .../server/resourcemanager/TestAppManager.java | 3 +- .../applicationsmanager/MockAsm.java | 4 +- .../metrics/TestSystemMetricsPublisher.java | 13 +++++- .../TestSystemMetricsPublisherForV2.java | 4 +- .../recovery/RMStateStoreTestBase.java | 4 +- .../recovery/TestZKRMStateStore.java | 2 +- .../resourcemanager/webapp/TestAppPage.java | 3 +- .../webapp/TestRMWebAppFairScheduler.java | 3 +- .../webapp/TestRMWebServicesApps.java | 2 +- 34 files changed, 297 insertions(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java index 34efee8..3cf8f3d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java @@ -36,7 +36,8 @@ public abstract class ApplicationResourceUsageReport { public static ApplicationResourceUsageReport newInstance( int numUsedContainers, int numReservedContainers, Resource usedResources, Resource reservedResources, Resource neededResources, long memorySeconds, - long vcoreSeconds, float queueUsagePerc, float clusterUsagePerc) { + long vcoreSeconds, float queueUsagePerc, float clusterUsagePerc, + long preemptedMemorySeconds, long preemptedVcoresSeconds) { ApplicationResourceUsageReport report = Records.newRecord(ApplicationResourceUsageReport.class); report.setNumUsedContainers(numUsedContainers); @@ -48,6 +49,8 @@ public abstract class ApplicationResourceUsageReport { report.setVcoreSeconds(vcoreSeconds); report.setQueueUsagePercentage(queueUsagePerc); report.setClusterUsagePercentage(clusterUsagePerc); + report.setPreemptedMemorySeconds(preemptedMemorySeconds); + report.setPreemptedVcoreSeconds(preemptedVcoresSeconds); return report; } @@ -188,4 +191,42 @@ public abstract class ApplicationResourceUsageReport { @Private @Unstable public abstract void setClusterUsagePercentage(float clusterUsagePerc); + + /** + * Set the aggregated amount of memory preempted (in megabytes) + * the application has allocated times the number of seconds + * the application has been running. + * @param memorySeconds the aggregated amount of memory seconds + */ + @Private + @Unstable + public abstract void setPreemptedMemorySeconds(long memorySeconds); + + /** + * Get the aggregated amount of memory preempted(in megabytes) + * the application has allocated times the number of + * seconds the application has been running. + * @return the aggregated amount of memory seconds + */ + @Public + @Unstable + public abstract long getPreemptedMemorySeconds(); + + /** + * Set the aggregated number of vcores preempted that the application has + * allocated times the number of seconds the application has been running. + * @param vcoreSeconds the aggregated number of vcore seconds + */ + @Private + @Unstable + public abstract void setPreemptedVcoreSeconds(long vcoreSeconds); + + /** + * Get the aggregated number of vcores preempted that the application has + * allocated times the number of seconds the application has been running. + * @return the aggregated number of vcore seconds + */ + @Public + @Unstable + public abstract long getPreemptedVcoreSeconds(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index b59d02b..1022a38 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -184,6 +184,8 @@ message ApplicationResourceUsageReportProto { optional int64 vcore_seconds = 7; optional float queue_usage_percentage = 8; optional float cluster_usage_percentage = 9; + optional int64 preempted_memory_seconds = 10; + optional int64 preempted_vcore_seconds = 11; } message ApplicationReportProto { http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 21df6a2..b841601 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -658,8 +658,15 @@ public class ApplicationCLI extends YarnCLI { //completed app report in the timeline server doesn't have usage report appReportStr.print(usageReport.getMemorySeconds() + " MB-seconds, "); appReportStr.println(usageReport.getVcoreSeconds() + " vcore-seconds"); + appReportStr.print("\tAggregate Resource Preempted : "); + appReportStr.print(usageReport.getPreemptedMemorySeconds() + + " MB-seconds, "); + appReportStr.println(usageReport.getPreemptedVcoreSeconds() + + " vcore-seconds"); } else { appReportStr.println("N/A"); + appReportStr.print("\tAggregate Resource Preempted : "); + appReportStr.println("N/A"); } appReportStr.print("\tLog Aggregation Status : "); appReportStr.println(appReport.getLogAggregationStatus() == null ? "N/A" http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index c1cabb0..09dec89 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -115,7 +115,7 @@ public class TestYarnCLI { ApplicationId applicationId = ApplicationId.newInstance(1234, 5); ApplicationResourceUsageReport usageReport = i == 0 ? null : ApplicationResourceUsageReport.newInstance( - 2, 0, null, null, null, 123456, 4567, 0, 0); + 2, 0, null, null, null, 123456, 4567, 0, 0, 1111, 2222); ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, @@ -148,6 +148,8 @@ public class TestYarnCLI { pw.println("\tAM Host : host"); pw.println("\tAggregate Resource Allocation : " + (i == 0 ? "N/A" : "123456 MB-seconds, 4567 vcore-seconds")); + pw.println("\tAggregate Resource Preempted : " + + (i == 0 ? "N/A" : "1111 MB-seconds, 2222 vcore-seconds")); pw.println("\tLog Aggregation Status : SUCCEEDED"); pw.println("\tDiagnostics : diagnostics"); pw.println("\tUnmanaged Application : false"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java index 5b1b85e..b6a38e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java @@ -224,6 +224,34 @@ extends ApplicationResourceUsageReport { return (p.getVcoreSeconds()); } + @Override + public synchronized void setPreemptedMemorySeconds( + long preemptedMemorySeconds) { + maybeInitBuilder(); + builder.setPreemptedMemorySeconds(preemptedMemorySeconds); + } + + @Override + public synchronized long getPreemptedMemorySeconds() { + ApplicationResourceUsageReportProtoOrBuilder p = + viaProto ? proto : builder; + return p.getPreemptedMemorySeconds(); + } + + @Override + public synchronized void setPreemptedVcoreSeconds( + long vcoreSeconds) { + maybeInitBuilder(); + builder.setPreemptedVcoreSeconds(vcoreSeconds); + } + + @Override + public synchronized long getPreemptedVcoreSeconds() { + ApplicationResourceUsageReportProtoOrBuilder p = + viaProto ? proto : builder; + return (p.getPreemptedVcoreSeconds()); + } + private ResourcePBImpl convertFromProtoFormat(ResourceProto p) { return new ResourcePBImpl(p); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index 6e6576a..3249c8f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -334,9 +334,15 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService ApplicationMetricsConstants.APP_CPU_METRICS).toString()); long memorySeconds=Long.parseLong(entityInfo.get( ApplicationMetricsConstants.APP_MEM_METRICS).toString()); + long preemptedMemorySeconds = Long.parseLong(entityInfo.get( + ApplicationMetricsConstants + .APP_MEM_PREEMPT_METRICS).toString()); + long preemptedVcoreSeconds = Long.parseLong(entityInfo.get( + ApplicationMetricsConstants + .APP_CPU_PREEMPT_METRICS).toString()); appResources = ApplicationResourceUsageReport .newInstance(0, 0, null, null, null, memorySeconds, vcoreSeconds, 0, - 0); + 0, preemptedMemorySeconds, preemptedVcoreSeconds); } if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) { appTags = new HashSet<String>(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java index 526cb8f..3d4e91b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java @@ -245,6 +245,11 @@ public class TestApplicationHistoryManagerOnTimelineStore { applicationResourceUsageReport.getMemorySeconds()); Assert .assertEquals(345, applicationResourceUsageReport.getVcoreSeconds()); + Assert.assertEquals(456, + applicationResourceUsageReport.getPreemptedMemorySeconds()); + Assert + .assertEquals(789, applicationResourceUsageReport + .getPreemptedVcoreSeconds()); Assert.assertEquals(FinalApplicationStatus.UNDEFINED, app.getFinalApplicationStatus()); Assert.assertEquals(YarnApplicationState.FINISHED, @@ -503,8 +508,10 @@ public class TestApplicationHistoryManagerOnTimelineStore { Priority.newInstance(0)); entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO, Integer.MAX_VALUE + 1L); - entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS,123); - entityInfo.put(ApplicationMetricsConstants.APP_CPU_METRICS,345); + entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS, 123); + entityInfo.put(ApplicationMetricsConstants.APP_CPU_METRICS, 345); + entityInfo.put(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS,456); + entityInfo.put(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS,789); if (emptyACLs) { entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, ""); } else { http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java index 521e0af..4cec409 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java @@ -88,6 +88,12 @@ public class ApplicationMetricsConstants { public static final String APP_AM_CONTAINER_PREEMPTED = "YARN_APPLICATION_AM_CONTAINER_PREEMPTED"; + public static final String APP_CPU_PREEMPT_METRICS = + "YARN_APPLICATION_CPU_PREEMPT_METRIC"; + + public static final String APP_MEM_PREEMPT_METRICS = + "YARN_APPLICATION_MEM_PREEMPT_METRIC"; + public static final String LATEST_APP_ATTEMPT_EVENT_INFO = "YARN_APPLICATION_LATEST_APP_ATTEMPT"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java index 50df12e..74c06ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java @@ -446,7 +446,8 @@ public class BuilderUtils { public static ApplicationResourceUsageReport newApplicationResourceUsageReport( int numUsedContainers, int numReservedContainers, Resource usedResources, Resource reservedResources, Resource neededResources, long memorySeconds, - long vcoreSeconds) { + long vcoreSeconds, long preemptedMemorySeconds, + long preemptedVcoreSeconds) { ApplicationResourceUsageReport report = recordFactory.newRecordInstance(ApplicationResourceUsageReport.class); report.setNumUsedContainers(numUsedContainers); @@ -456,6 +457,8 @@ public class BuilderUtils { report.setNeededResources(neededResources); report.setMemorySeconds(memorySeconds); report.setVcoreSeconds(vcoreSeconds); + report.setPreemptedMemorySeconds(preemptedMemorySeconds); + report.setPreemptedVcoreSeconds(preemptedVcoreSeconds); return report; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 7144421..e566243 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -182,6 +182,8 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>, .add("finalStatus", app.getFinalApplicationStatus()) .add("memorySeconds", metrics.getMemorySeconds()) .add("vcoreSeconds", metrics.getVcoreSeconds()) + .add("preemptedMemorySeconds", metrics.getPreemptedMemorySeconds()) + .add("preemptedVcoreSeconds", metrics.getPreemptedVcoreSeconds()) .add("preemptedAMContainers", metrics.getNumAMContainersPreempted()) .add("preemptedNonAMContainers", metrics.getNumNonAMContainersPreempted()) .add("preemptedResources", metrics.getResourcePreempted()) http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index 7e31e70..98a34d9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -453,7 +453,7 @@ public class RMServerUtils { DUMMY_APPLICATION_RESOURCE_USAGE_REPORT = BuilderUtils.newApplicationResourceUsageReport(-1, -1, Resources.createResource(-1, -1), Resources.createResource(-1, -1), - Resources.createResource(-1, -1), 0, 0); + Resources.createResource(-1, -1), 0, 0, 0, 0); /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java index cbf6a73..aa3ef0a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java @@ -144,6 +144,10 @@ public class TimelineServiceV1Publisher extends AbstractSystemMetricsPublisher { appMetrics.getVcoreSeconds()); entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_METRICS, appMetrics.getMemorySeconds()); + entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS, + appMetrics.getPreemptedMemorySeconds()); + entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS, + appMetrics.getPreemptedVcoreSeconds()); tEvent.setEventInfo(eventInfo); entity.addEvent(tEvent); http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java index f039ebe..6eb6eb9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java @@ -194,6 +194,12 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher { ApplicationMetricsConstants.APP_MEM_METRICS, timestamp, appMetrics.getMemorySeconds())); entityMetrics.add(getTimelineMetric( + ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS, timestamp, + appMetrics.getPreemptedMemorySeconds())); + entityMetrics.add(getTimelineMetric( + ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS, timestamp, + appMetrics.getPreemptedVcoreSeconds())); + entityMetrics.add(getTimelineMetric( ApplicationMetricsConstants.APP_RESOURCE_PREEMPTED_CPU, timestamp, appMetrics.getResourcePreempted().getVirtualCores())); entityMetrics.add(getTimelineMetric( http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java index d1f8b40..fc12522 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java @@ -69,6 +69,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppR import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics; import org.apache.hadoop.yarn.state.InvalidStateTransitionException; import org.apache.hadoop.yarn.state.MultipleArcTransition; import org.apache.hadoop.yarn.state.StateMachine; @@ -826,16 +827,19 @@ public abstract class RMStateStore extends AbstractService { */ public void storeNewApplicationAttempt(RMAppAttempt appAttempt) { Credentials credentials = getCredentialsFromAppAttempt(appAttempt); - + RMAppAttemptMetrics attempMetrics = appAttempt.getRMAppAttemptMetrics(); AggregateAppResourceUsage resUsage = - appAttempt.getRMAppAttemptMetrics().getAggregateAppResourceUsage(); + attempMetrics.getAggregateAppResourceUsage(); ApplicationAttemptStateData attemptState = ApplicationAttemptStateData.newInstance( appAttempt.getAppAttemptId(), appAttempt.getMasterContainer(), credentials, appAttempt.getStartTime(), resUsage.getMemorySeconds(), - resUsage.getVcoreSeconds()); + resUsage.getVcoreSeconds(), + attempMetrics.getPreemptedMemory(), + attempMetrics.getPreemptedVcore() + ); dispatcher.getEventHandler().handle( new RMStateStoreAppAttemptEvent(attemptState)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java index 391783b..67aaf94 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java @@ -40,7 +40,8 @@ public abstract class ApplicationAttemptStateData { Credentials attemptTokens, long startTime, RMAppAttemptState finalState, String finalTrackingUrl, String diagnostics, FinalApplicationStatus amUnregisteredFinalStatus, int exitStatus, - long finishTime, long memorySeconds, long vcoreSeconds) { + long finishTime, long memorySeconds, long vcoreSeconds, + long preemptedMemorySeconds, long preemptedVcoreSeconds) { ApplicationAttemptStateData attemptStateData = Records.newRecord(ApplicationAttemptStateData.class); attemptStateData.setAttemptId(attemptId); @@ -55,16 +56,20 @@ public abstract class ApplicationAttemptStateData { attemptStateData.setFinishTime(finishTime); attemptStateData.setMemorySeconds(memorySeconds); attemptStateData.setVcoreSeconds(vcoreSeconds); + attemptStateData.setPreemptedMemorySeconds(preemptedMemorySeconds); + attemptStateData.setPreemptedVcoreSeconds(preemptedVcoreSeconds); return attemptStateData; } public static ApplicationAttemptStateData newInstance( ApplicationAttemptId attemptId, Container masterContainer, Credentials attemptTokens, long startTime, long memorySeconds, - long vcoreSeconds) { + long vcoreSeconds, long preemptedMemorySeconds, + long preemptedVcoreSeconds) { return newInstance(attemptId, masterContainer, attemptTokens, startTime, null, "N/A", "", null, ContainerExitStatus.INVALID, 0, - memorySeconds, vcoreSeconds); + memorySeconds, vcoreSeconds, + preemptedMemorySeconds, preemptedVcoreSeconds); } @@ -182,4 +187,32 @@ public abstract class ApplicationAttemptStateData { @Public @Unstable public abstract void setVcoreSeconds(long vcoreSeconds); + + /** + * Get the <em>preempted memory seconds</em> + * (in MB seconds) of the application. + * @return <em>preempted memory seconds</em> + * (in MB seconds) of the application + */ + @Public + @Unstable + public abstract long getPreemptedMemorySeconds(); + + @Public + @Unstable + public abstract void setPreemptedMemorySeconds(long memorySeconds); + + /** + * Get the <em>preempted vcore seconds</em> + * of the application. + * @return <em>preempted vcore seconds</em> + * of the application + */ + @Public + @Unstable + public abstract long getPreemptedVcoreSeconds(); + + @Public + @Unstable + public abstract void setPreemptedVcoreSeconds(long vcoreSeconds); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java index bae3f9c..e89726f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java @@ -263,6 +263,30 @@ public class ApplicationAttemptStateDataPBImpl extends } @Override + public long getPreemptedMemorySeconds() { + ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder; + return p.getPreemptedMemorySeconds(); + } + + @Override + public long getPreemptedVcoreSeconds() { + ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder; + return p.getPreemptedVcoreSeconds(); + } + + @Override + public void setPreemptedMemorySeconds(long memorySeconds) { + maybeInitBuilder(); + builder.setPreemptedMemorySeconds(memorySeconds); + } + + @Override + public void setPreemptedVcoreSeconds(long vcoreSeconds) { + maybeInitBuilder(); + builder.setPreemptedVcoreSeconds(vcoreSeconds); + } + + @Override public FinalApplicationStatus getFinalApplicationStatus() { ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder; if (!p.hasFinalApplicationStatus()) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 74e641d..a376311 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -757,6 +757,12 @@ public class RMAppImpl implements RMApp, Recoverable { RMAppMetrics rmAppMetrics = getRMAppMetrics(); appUsageReport.setMemorySeconds(rmAppMetrics.getMemorySeconds()); appUsageReport.setVcoreSeconds(rmAppMetrics.getVcoreSeconds()); + appUsageReport. + setPreemptedMemorySeconds(rmAppMetrics. + getPreemptedMemorySeconds()); + appUsageReport. + setPreemptedVcoreSeconds(rmAppMetrics. + getPreemptedVcoreSeconds()); } if (currentApplicationAttemptId == null) { @@ -1630,6 +1636,8 @@ public class RMAppImpl implements RMApp, Recoverable { int numNonAMContainerPreempted = 0; long memorySeconds = 0; long vcoreSeconds = 0; + long preemptedMemorySeconds = 0; + long preemptedVcoreSeconds = 0; for (RMAppAttempt attempt : attempts.values()) { if (null != attempt) { RMAppAttemptMetrics attemptMetrics = @@ -1645,12 +1653,15 @@ public class RMAppImpl implements RMApp, Recoverable { attempt.getRMAppAttemptMetrics().getAggregateAppResourceUsage(); memorySeconds += resUsage.getMemorySeconds(); vcoreSeconds += resUsage.getVcoreSeconds(); + preemptedMemorySeconds += attemptMetrics.getPreemptedMemory(); + preemptedVcoreSeconds += attemptMetrics.getPreemptedVcore(); } } return new RMAppMetrics(resourcePreempted, numNonAMContainerPreempted, numAMContainerPreempted, - memorySeconds, vcoreSeconds); + memorySeconds, vcoreSeconds, + preemptedMemorySeconds, preemptedVcoreSeconds); } @Private http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java index 5091470..fa068ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java @@ -26,15 +26,20 @@ public class RMAppMetrics { final int numAMContainersPreempted; final long memorySeconds; final long vcoreSeconds; + private final long preemptedMemorySeconds; + private final long preemptedVcoreSeconds; public RMAppMetrics(Resource resourcePreempted, int numNonAMContainersPreempted, int numAMContainersPreempted, - long memorySeconds, long vcoreSeconds) { + long memorySeconds, long vcoreSeconds, long preemptedMemorySeconds, + long preemptedVcoreSeconds) { this.resourcePreempted = resourcePreempted; this.numNonAMContainersPreempted = numNonAMContainersPreempted; this.numAMContainersPreempted = numAMContainersPreempted; this.memorySeconds = memorySeconds; this.vcoreSeconds = vcoreSeconds; + this.preemptedMemorySeconds = preemptedMemorySeconds; + this.preemptedVcoreSeconds = preemptedVcoreSeconds; } public Resource getResourcePreempted() { @@ -56,4 +61,13 @@ public class RMAppMetrics { public long getVcoreSeconds() { return vcoreSeconds; } + + public long getPreemptedMemorySeconds() { + return preemptedMemorySeconds; + } + + public long getPreemptedVcoreSeconds() { + return preemptedVcoreSeconds; + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 5b78eb8..e3d1855 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -889,6 +889,10 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { this.attemptMetrics.getAggregateAppResourceUsage(); report.setMemorySeconds(resUsage.getMemorySeconds()); report.setVcoreSeconds(resUsage.getVcoreSeconds()); + report.setPreemptedMemorySeconds( + this.attemptMetrics.getPreemptedMemory()); + report.setPreemptedVcoreSeconds( + this.attemptMetrics.getPreemptedVcore()); return report; } finally { this.readLock.unlock(); @@ -921,6 +925,9 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { this.finishTime = attemptState.getFinishTime(); this.attemptMetrics.updateAggregateAppResourceUsage( attemptState.getMemorySeconds(),attemptState.getVcoreSeconds()); + this.attemptMetrics.updateAggregatePreemptedAppResourceUsage( + attemptState.getPreemptedMemorySeconds(), + attemptState.getPreemptedVcoreSeconds()); } public void transferStateFromAttempt(RMAppAttempt attempt) { @@ -1296,7 +1303,9 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { startTime, stateToBeStored, finalTrackingUrl, diags, finalStatus, exitStatus, getFinishTime(), resUsage.getMemorySeconds(), - resUsage.getVcoreSeconds()); + resUsage.getVcoreSeconds(), + this.attemptMetrics.getPreemptedMemory(), + this.attemptMetrics.getPreemptedVcore()); LOG.info("Updating application attempt " + applicationAttemptId + " with final state: " + targetedFinalState + ", and exit status: " + exitStatus); http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java index 0a3638b..a642e45 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java @@ -50,6 +50,8 @@ public class RMAppAttemptMetrics { private WriteLock writeLock; private AtomicLong finishedMemorySeconds = new AtomicLong(0); private AtomicLong finishedVcoreSeconds = new AtomicLong(0); + private AtomicLong preemptedMemorySeconds = new AtomicLong(0); + private AtomicLong preemptedVcoreSeconds = new AtomicLong(0); private RMContext rmContext; private int[][] localityStatistics = @@ -98,6 +100,14 @@ public class RMAppAttemptMetrics { } } + public long getPreemptedMemory() { + return preemptedMemorySeconds.get(); + } + + public long getPreemptedVcore() { + return preemptedVcoreSeconds.get(); + } + public int getNumNonAMContainersPreempted() { return numNonAMContainersPreempted.get(); } @@ -134,6 +144,12 @@ public class RMAppAttemptMetrics { this.finishedVcoreSeconds.addAndGet(finishedVcoreSeconds); } + public void updateAggregatePreemptedAppResourceUsage( + long preemptedMemorySeconds, long preemptedVcoreSeconds) { + this.preemptedMemorySeconds.addAndGet(preemptedMemorySeconds); + this.preemptedVcoreSeconds.addAndGet(preemptedVcoreSeconds); + } + public void incNumAllocatedContainers(NodeType containerType, NodeType requestType) { localityStatistics[containerType.index][requestType.index]++; http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java index 4294ef0..a651d54 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java @@ -729,19 +729,12 @@ public class RMContainerImpl implements RMContainer, Comparable<RMContainer> { } private static void updateAttemptMetrics(RMContainerImpl container) { - // If this is a preempted container, update preemption metrics Resource resource = container.getContainer().getResource(); RMAppAttempt rmAttempt = container.rmContext.getRMApps() .get(container.getApplicationAttemptId().getApplicationId()) .getCurrentAppAttempt(); if (rmAttempt != null) { - if (ContainerExitStatus.PREEMPTED == container.finishedStatus - .getExitStatus()) { - rmAttempt.getRMAppAttemptMetrics().updatePreemptionInfo(resource, - container); - } - long usedMillis = container.finishTime - container.creationTime; long memorySeconds = resource.getMemorySize() * usedMillis / DateUtils.MILLIS_PER_SECOND; @@ -749,6 +742,15 @@ public class RMContainerImpl implements RMContainer, Comparable<RMContainer> { * usedMillis / DateUtils.MILLIS_PER_SECOND; rmAttempt.getRMAppAttemptMetrics() .updateAggregateAppResourceUsage(memorySeconds,vcoreSeconds); + // If this is a preempted container, update preemption metrics + if (ContainerExitStatus.PREEMPTED == container.finishedStatus + .getExitStatus()) { + rmAttempt.getRMAppAttemptMetrics().updatePreemptionInfo(resource, + container); + rmAttempt.getRMAppAttemptMetrics() + .updateAggregatePreemptedAppResourceUsage(memorySeconds, + vcoreSeconds); + } } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index bb1d461..bc52816 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -914,7 +914,7 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { Resources.add(usedResourceClone, reservedResourceClone), runningResourceUsage.getMemorySeconds(), runningResourceUsage.getVcoreSeconds(), queueUsagePerc, - clusterUsagePerc); + clusterUsagePerc, 0, 0); } finally { writeLock.unlock(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java index cfa21dd..e5d6c16 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java @@ -101,7 +101,13 @@ public class RMAppBlock extends AppBlock{ ._("Aggregate Resource Allocation:", String.format("%d MB-seconds, %d vcore-seconds", appMetrics == null ? "N/A" : appMetrics.getMemorySeconds(), - appMetrics == null ? "N/A" : appMetrics.getVcoreSeconds())); + appMetrics == null ? "N/A" : appMetrics.getVcoreSeconds())) + ._("Aggregate Preempted Resource Allocation:", + String.format("%d MB-seconds, %d vcore-seconds", + appMetrics == null ? "N/A" : appMetrics.getPreemptedMemorySeconds(), + appMetrics == null ? "N/A" : + appMetrics.getPreemptedVcoreSeconds())); + pdiv._(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 19cbe43..42449b0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -100,6 +100,8 @@ public class AppInfo { protected long preemptedResourceVCores; protected int numNonAMContainerPreempted; protected int numAMContainerPreempted; + private long preemptedMemorySeconds; + private long preemptedVcoreSeconds; // list of resource requests @XmlElement(name = "resourceRequests") @@ -216,6 +218,8 @@ public class AppInfo { appMetrics.getResourcePreempted().getVirtualCores(); memorySeconds = appMetrics.getMemorySeconds(); vcoreSeconds = appMetrics.getVcoreSeconds(); + preemptedMemorySeconds = appMetrics.getPreemptedMemorySeconds(); + preemptedVcoreSeconds = appMetrics.getPreemptedVcoreSeconds(); unmanagedApplication = appSubmissionContext.getUnmanagedAM(); appNodeLabelExpression = @@ -382,6 +386,13 @@ public class AppInfo { return vcoreSeconds; } + public long getPreemptedMemorySeconds() { + return preemptedMemorySeconds; + } + + public long getPreemptedVcoreSeconds() { + return preemptedVcoreSeconds; + } public List<ResourceRequestInfo> getResourceRequests() { return this.resourceRequests; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto index 4693818..247cd21 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto @@ -85,6 +85,8 @@ message ApplicationAttemptStateDataProto { optional int64 memory_seconds = 10; optional int64 vcore_seconds = 11; optional int64 finish_time = 12; + optional int64 preempted_memory_seconds = 13; + optional int64 preempted_vcore_seconds = 14; } message EpochProto { http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index 3db0d65..981b045 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -689,7 +689,8 @@ public class TestAppManager{ when(app.getState()).thenReturn(RMAppState.RUNNING); when(app.getApplicationType()).thenReturn("MAPREDUCE"); RMAppMetrics metrics = - new RMAppMetrics(Resource.newInstance(1234, 56), 10, 1, 16384, 64); + new RMAppMetrics(Resource.newInstance(1234, 56), + 10, 1, 16384, 64, 0, 0); when(app.getRMAppMetrics()).thenReturn(metrics); RMAppManager.ApplicationSummary.SummaryBuilder summary = http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index e5b166d..f21c7c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -194,7 +194,7 @@ public abstract class MockAsm extends MockApps { @Override public RMAppMetrics getRMAppMetrics() { - return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0); + return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0, 0, 0); } @Override @@ -328,7 +328,7 @@ public abstract class MockAsm extends MockApps { String clientUserName, boolean allowAccess) { ApplicationResourceUsageReport usageReport = ApplicationResourceUsageReport.newInstance(0, 0, null, null, null, - 0, 0, 0, 0); + 0, 0, 0, 0, 0, 0); ApplicationReport report = ApplicationReport.newInstance( getApplicationId(), appAttemptId, getUser(), getQueue(), getName(), null, 0, null, null, getDiagnostics().toString(), http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index b38091e..2b1740f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -223,6 +223,16 @@ public class TestSystemMetricsPublisher { app.getRMAppMetrics().getVcoreSeconds(), Long.parseLong(entity.getOtherInfo() .get(ApplicationMetricsConstants.APP_CPU_METRICS).toString())); + Assert.assertEquals( + app.getRMAppMetrics().getPreemptedMemorySeconds(), + Long.parseLong(entity.getOtherInfo() + .get(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS) + .toString())); + Assert.assertEquals( + app.getRMAppMetrics().getPreemptedVcoreSeconds(), + Long.parseLong(entity.getOtherInfo() + .get(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS) + .toString())); } Assert.assertEquals("context", entity.getOtherInfo() .get(ApplicationMetricsConstants.YARN_APP_CALLER_CONTEXT)); @@ -496,7 +506,8 @@ public class TestSystemMetricsPublisher { when(app.getFinalApplicationStatus()).thenReturn( FinalApplicationStatus.UNDEFINED); when(app.getRMAppMetrics()).thenReturn( - new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE)); + new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE, + Integer.MAX_VALUE, Long.MAX_VALUE)); Set<String> appTags = new HashSet<String>(); appTags.add("test"); appTags.add("tags"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java index 4a19744..3ab2501 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java @@ -211,7 +211,7 @@ public class TestSystemMetricsPublisherForV2 { + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION; File appFile = new File(outputDirApp, timelineServiceFileName); Assert.assertTrue(appFile.exists()); - verifyEntity(appFile, 3, ApplicationMetricsConstants.CREATED_EVENT_TYPE, 6); + verifyEntity(appFile, 3, ApplicationMetricsConstants.CREATED_EVENT_TYPE, 8); } @Test(timeout = 10000) @@ -355,7 +355,7 @@ public class TestSystemMetricsPublisherForV2 { FinalApplicationStatus.UNDEFINED); when(app.getRMAppMetrics()).thenReturn( new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, Integer.MAX_VALUE, - Long.MAX_VALUE)); + Long.MAX_VALUE, 0, 0)); when(app.getApplicationTags()).thenReturn(Collections.<String> emptySet()); ApplicationSubmissionContext appSubmissionContext = mock(ApplicationSubmissionContext.class); http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java index 758bbae..8544c13 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java @@ -361,7 +361,7 @@ public class RMStateStoreTestBase { oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED, "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.SUCCEEDED, 100, - oldAttemptState.getFinishTime(), 0, 0); + oldAttemptState.getFinishTime(), 0, 0, 0, 0); store.updateApplicationAttemptState(newAttemptState); // test updating the state of an app/attempt whose initial state was not @@ -385,7 +385,7 @@ public class RMStateStoreTestBase { oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED, "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.SUCCEEDED, 111, - oldAttemptState.getFinishTime(), 0, 0); + oldAttemptState.getFinishTime(), 0, 0, 0, 0); store.updateApplicationAttemptState(dummyAttempt); // let things settle down http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java index 19d3064..aef0dfe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java @@ -428,7 +428,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase { store.getCredentialsFromAppAttempt(mockAttempt), startTime, RMAppAttemptState.FINISHED, "testUrl", "test", FinalApplicationStatus.SUCCEEDED, 100, - finishTime, 0, 0); + finishTime, 0, 0, 0, 0); store.updateApplicationAttemptState(newAttemptState); assertEquals("RMStateStore should have been in fenced state", true, store.isFencedState()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java index 8c7b14d..a009bc0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java @@ -64,7 +64,8 @@ public class TestAppPage { when(app.getFinishTime()).thenReturn(0L); when(app.createApplicationState()).thenReturn(YarnApplicationState.FAILED); - RMAppMetrics appMetrics = new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0); + RMAppMetrics appMetrics = new RMAppMetrics( + Resource.newInstance(0, 0), 0, 0, 0, 0, 0, 0); when(app.getRMAppMetrics()).thenReturn(appMetrics); // initialize RM Context, and create RMApp, without creating RMAppAttempt http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java index 06fa0d4..78fadef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java @@ -142,7 +142,8 @@ public class TestRMWebAppFairScheduler { MockRMApp app = new MockRMApp(i, i, state) { @Override public RMAppMetrics getRMAppMetrics() { - return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0); + return new RMAppMetrics(Resource.newInstance(0, 0), + 0, 0, 0, 0, 0, 0); } @Override public YarnApplicationState createApplicationState() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index da28754..fffa55b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1419,7 +1419,7 @@ public class TestRMWebServicesApps extends JerseyTestBase { public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException, Exception { - int expectedNumberOfElements = 32; + int expectedNumberOfElements = 34; String appNodeLabelExpression = null; String amNodeLabelExpression = null; if (app.getApplicationSubmissionContext() --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
