[01/50] hadoop git commit: YARN-4218. Metric for resource*time that was preempted. Contributed by Chang Li. [Forced Update!]

2016-11-16 Thread kasha 
Repository: hadoop
Updated Branches:
  refs/heads/YARN-4752 dd24d316d -> 59bc9d16a (forced update)


YARN-4218. Metric for resource*time that was preempted. Contributed by Chang Li.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/93eeb131
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/93eeb131
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/93eeb131

Branch: refs/heads/YARN-4752
Commit: 93eeb13164707d0e3556c2bf737bd2ee09a335c6
Parents: 3a98419
Author: Eric Payne 
Authored: Thu Nov 10 22:35:12 2016 +
Committer: Eric Payne 
Committed: Thu Nov 10 22:35:12 2016 +

--
 .../records/ApplicationResourceUsageReport.java | 43 +++-
 .../src/main/proto/yarn_protos.proto|  2 +
 .../hadoop/yarn/client/cli/ApplicationCLI.java  |  7 
 .../hadoop/yarn/client/cli/TestYarnCLI.java |  4 +-
 .../ApplicationResourceUsageReportPBImpl.java   | 28 +
 ...pplicationHistoryManagerOnTimelineStore.java |  8 +++-
 ...pplicationHistoryManagerOnTimelineStore.java | 11 -
 .../metrics/ApplicationMetricsConstants.java|  6 +++
 .../hadoop/yarn/server/utils/BuilderUtils.java  |  5 ++-
 .../server/resourcemanager/RMAppManager.java|  2 +
 .../server/resourcemanager/RMServerUtils.java   |  2 +-
 .../metrics/TimelineServiceV1Publisher.java |  4 ++
 .../metrics/TimelineServiceV2Publisher.java |  6 +++
 .../resourcemanager/recovery/RMStateStore.java  | 10 +++--
 .../records/ApplicationAttemptStateData.java| 39 --
 .../pb/ApplicationAttemptStateDataPBImpl.java   | 24 +++
 .../server/resourcemanager/rmapp/RMAppImpl.java | 13 +-
 .../resourcemanager/rmapp/RMAppMetrics.java | 16 +++-
 .../rmapp/attempt/RMAppAttemptImpl.java | 11 -
 .../rmapp/attempt/RMAppAttemptMetrics.java  | 16 
 .../rmcontainer/RMContainerImpl.java| 16 
 .../scheduler/SchedulerApplicationAttempt.java  |  2 +-
 .../resourcemanager/webapp/RMAppBlock.java  |  8 +++-
 .../resourcemanager/webapp/dao/AppInfo.java | 11 +
 .../yarn_server_resourcemanager_recovery.proto  |  2 +
 .../server/resourcemanager/TestAppManager.java  |  3 +-
 .../applicationsmanager/MockAsm.java|  4 +-
 .../metrics/TestSystemMetricsPublisher.java | 13 +-
 .../TestSystemMetricsPublisherForV2.java|  4 +-
 .../recovery/RMStateStoreTestBase.java  |  4 +-
 .../recovery/TestZKRMStateStore.java|  2 +-
 .../resourcemanager/webapp/TestAppPage.java |  3 +-
 .../webapp/TestRMWebAppFairScheduler.java   |  3 +-
 .../webapp/TestRMWebServicesApps.java   |  2 +-
 34 files changed, 297 insertions(+), 37 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
index 34efee8..3cf8f3d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
@@ -36,7 +36,8 @@ public abstract class ApplicationResourceUsageReport {
   public static ApplicationResourceUsageReport newInstance(
   int numUsedContainers, int numReservedContainers, Resource usedResources,
   Resource reservedResources, Resource neededResources, long memorySeconds,
-  long vcoreSeconds, float queueUsagePerc, float clusterUsagePerc) {
+  long vcoreSeconds, float queueUsagePerc, float clusterUsagePerc,
+  long preemptedMemorySeconds, long preemptedVcoresSeconds) {
 ApplicationResourceUsageReport report =
 Records.newRecord(ApplicationResourceUsageReport.class);
 report.setNumUsedContainers(numUsedContainers);
@@ -48,6 +49,8 @@ public abstract class ApplicationResourceUsageReport {
 report.setVcoreSeconds(vcoreSeconds);
 report.setQueueUsagePercentage(queueUsagePerc);
 report.setClusterUsagePercentage(clusterUsagePerc);
+report.setPreemptedMemorySeconds(preemptedMemorySeconds);
+report.setPreemptedVcoreSeconds(preemptedVcoresSeconds);
 return report;
   }
 
@@ -188,4 +191,42 @@ public abstract class ApplicationResourceUsageReport {
   @Private
   @Unstable
   public abstract void

[06/50] [abbrv] hadoop git commit: YARN-4218. Metric for resource*time that was preempted. Contributed by Chang Li.

2016-11-16 Thread vvasudev 
YARN-4218. Metric for resource*time that was preempted. Contributed by Chang Li.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/93eeb131
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/93eeb131
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/93eeb131

Branch: refs/heads/YARN-3926
Commit: 93eeb13164707d0e3556c2bf737bd2ee09a335c6
Parents: 3a98419
Author: Eric Payne 
Authored: Thu Nov 10 22:35:12 2016 +
Committer: Eric Payne 
Committed: Thu Nov 10 22:35:12 2016 +

--
 .../records/ApplicationResourceUsageReport.java | 43 +++-
 .../src/main/proto/yarn_protos.proto|  2 +
 .../hadoop/yarn/client/cli/ApplicationCLI.java  |  7 
 .../hadoop/yarn/client/cli/TestYarnCLI.java |  4 +-
 .../ApplicationResourceUsageReportPBImpl.java   | 28 +
 ...pplicationHistoryManagerOnTimelineStore.java |  8 +++-
 ...pplicationHistoryManagerOnTimelineStore.java | 11 -
 .../metrics/ApplicationMetricsConstants.java|  6 +++
 .../hadoop/yarn/server/utils/BuilderUtils.java  |  5 ++-
 .../server/resourcemanager/RMAppManager.java|  2 +
 .../server/resourcemanager/RMServerUtils.java   |  2 +-
 .../metrics/TimelineServiceV1Publisher.java |  4 ++
 .../metrics/TimelineServiceV2Publisher.java |  6 +++
 .../resourcemanager/recovery/RMStateStore.java  | 10 +++--
 .../records/ApplicationAttemptStateData.java| 39 --
 .../pb/ApplicationAttemptStateDataPBImpl.java   | 24 +++
 .../server/resourcemanager/rmapp/RMAppImpl.java | 13 +-
 .../resourcemanager/rmapp/RMAppMetrics.java | 16 +++-
 .../rmapp/attempt/RMAppAttemptImpl.java | 11 -
 .../rmapp/attempt/RMAppAttemptMetrics.java  | 16 
 .../rmcontainer/RMContainerImpl.java| 16 
 .../scheduler/SchedulerApplicationAttempt.java  |  2 +-
 .../resourcemanager/webapp/RMAppBlock.java  |  8 +++-
 .../resourcemanager/webapp/dao/AppInfo.java | 11 +
 .../yarn_server_resourcemanager_recovery.proto  |  2 +
 .../server/resourcemanager/TestAppManager.java  |  3 +-
 .../applicationsmanager/MockAsm.java|  4 +-
 .../metrics/TestSystemMetricsPublisher.java | 13 +-
 .../TestSystemMetricsPublisherForV2.java|  4 +-
 .../recovery/RMStateStoreTestBase.java  |  4 +-
 .../recovery/TestZKRMStateStore.java|  2 +-
 .../resourcemanager/webapp/TestAppPage.java |  3 +-
 .../webapp/TestRMWebAppFairScheduler.java   |  3 +-
 .../webapp/TestRMWebServicesApps.java   |  2 +-
 34 files changed, 297 insertions(+), 37 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
index 34efee8..3cf8f3d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
@@ -36,7 +36,8 @@ public abstract class ApplicationResourceUsageReport {
   public static ApplicationResourceUsageReport newInstance(
   int numUsedContainers, int numReservedContainers, Resource usedResources,
   Resource reservedResources, Resource neededResources, long memorySeconds,
-  long vcoreSeconds, float queueUsagePerc, float clusterUsagePerc) {
+  long vcoreSeconds, float queueUsagePerc, float clusterUsagePerc,
+  long preemptedMemorySeconds, long preemptedVcoresSeconds) {
 ApplicationResourceUsageReport report =
 Records.newRecord(ApplicationResourceUsageReport.class);
 report.setNumUsedContainers(numUsedContainers);
@@ -48,6 +49,8 @@ public abstract class ApplicationResourceUsageReport {
 report.setVcoreSeconds(vcoreSeconds);
 report.setQueueUsagePercentage(queueUsagePerc);
 report.setClusterUsagePercentage(clusterUsagePerc);
+report.setPreemptedMemorySeconds(preemptedMemorySeconds);
+report.setPreemptedVcoreSeconds(preemptedVcoresSeconds);
 return report;
   }
 
@@ -188,4 +191,42 @@ public abstract class ApplicationResourceUsageReport {
   @Private
   @Unstable
   public abstract void setClusterUsagePercentage(float clusterUsagePerc);
+
+  /**
+   * Set the aggregated amount of memory preempted (in

[14/50] [abbrv] hadoop git commit: YARN-4218. Metric for resource*time that was preempted. Contributed by Chang Li.

2016-11-11 Thread subru 
YARN-4218. Metric for resource*time that was preempted. Contributed by Chang Li.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/93eeb131
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/93eeb131
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/93eeb131

Branch: refs/heads/YARN-2915
Commit: 93eeb13164707d0e3556c2bf737bd2ee09a335c6
Parents: 3a98419
Author: Eric Payne 
Authored: Thu Nov 10 22:35:12 2016 +
Committer: Eric Payne 
Committed: Thu Nov 10 22:35:12 2016 +

--
 .../records/ApplicationResourceUsageReport.java | 43 +++-
 .../src/main/proto/yarn_protos.proto|  2 +
 .../hadoop/yarn/client/cli/ApplicationCLI.java  |  7 
 .../hadoop/yarn/client/cli/TestYarnCLI.java |  4 +-
 .../ApplicationResourceUsageReportPBImpl.java   | 28 +
 ...pplicationHistoryManagerOnTimelineStore.java |  8 +++-
 ...pplicationHistoryManagerOnTimelineStore.java | 11 -
 .../metrics/ApplicationMetricsConstants.java|  6 +++
 .../hadoop/yarn/server/utils/BuilderUtils.java  |  5 ++-
 .../server/resourcemanager/RMAppManager.java|  2 +
 .../server/resourcemanager/RMServerUtils.java   |  2 +-
 .../metrics/TimelineServiceV1Publisher.java |  4 ++
 .../metrics/TimelineServiceV2Publisher.java |  6 +++
 .../resourcemanager/recovery/RMStateStore.java  | 10 +++--
 .../records/ApplicationAttemptStateData.java| 39 --
 .../pb/ApplicationAttemptStateDataPBImpl.java   | 24 +++
 .../server/resourcemanager/rmapp/RMAppImpl.java | 13 +-
 .../resourcemanager/rmapp/RMAppMetrics.java | 16 +++-
 .../rmapp/attempt/RMAppAttemptImpl.java | 11 -
 .../rmapp/attempt/RMAppAttemptMetrics.java  | 16 
 .../rmcontainer/RMContainerImpl.java| 16 
 .../scheduler/SchedulerApplicationAttempt.java  |  2 +-
 .../resourcemanager/webapp/RMAppBlock.java  |  8 +++-
 .../resourcemanager/webapp/dao/AppInfo.java | 11 +
 .../yarn_server_resourcemanager_recovery.proto  |  2 +
 .../server/resourcemanager/TestAppManager.java  |  3 +-
 .../applicationsmanager/MockAsm.java|  4 +-
 .../metrics/TestSystemMetricsPublisher.java | 13 +-
 .../TestSystemMetricsPublisherForV2.java|  4 +-
 .../recovery/RMStateStoreTestBase.java  |  4 +-
 .../recovery/TestZKRMStateStore.java|  2 +-
 .../resourcemanager/webapp/TestAppPage.java |  3 +-
 .../webapp/TestRMWebAppFairScheduler.java   |  3 +-
 .../webapp/TestRMWebServicesApps.java   |  2 +-
 34 files changed, 297 insertions(+), 37 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
index 34efee8..3cf8f3d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
@@ -36,7 +36,8 @@ public abstract class ApplicationResourceUsageReport {
   public static ApplicationResourceUsageReport newInstance(
   int numUsedContainers, int numReservedContainers, Resource usedResources,
   Resource reservedResources, Resource neededResources, long memorySeconds,
-  long vcoreSeconds, float queueUsagePerc, float clusterUsagePerc) {
+  long vcoreSeconds, float queueUsagePerc, float clusterUsagePerc,
+  long preemptedMemorySeconds, long preemptedVcoresSeconds) {
 ApplicationResourceUsageReport report =
 Records.newRecord(ApplicationResourceUsageReport.class);
 report.setNumUsedContainers(numUsedContainers);
@@ -48,6 +49,8 @@ public abstract class ApplicationResourceUsageReport {
 report.setVcoreSeconds(vcoreSeconds);
 report.setQueueUsagePercentage(queueUsagePerc);
 report.setClusterUsagePercentage(clusterUsagePerc);
+report.setPreemptedMemorySeconds(preemptedMemorySeconds);
+report.setPreemptedVcoreSeconds(preemptedVcoresSeconds);
 return report;
   }
 
@@ -188,4 +191,42 @@ public abstract class ApplicationResourceUsageReport {
   @Private
   @Unstable
   public abstract void setClusterUsagePercentage(float clusterUsagePerc);
+
+  /**
+   * Set the aggregated amount of memory preempted (in

[44/50] [abbrv] hadoop git commit: YARN-4218. Metric for resource*time that was preempted. Contributed by Chang Li.

2016-11-11 Thread aengineer 
YARN-4218. Metric for resource*time that was preempted. Contributed by Chang Li.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/93eeb131
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/93eeb131
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/93eeb131

Branch: refs/heads/HDFS-7240
Commit: 93eeb13164707d0e3556c2bf737bd2ee09a335c6
Parents: 3a98419
Author: Eric Payne 
Authored: Thu Nov 10 22:35:12 2016 +
Committer: Eric Payne 
Committed: Thu Nov 10 22:35:12 2016 +

--
 .../records/ApplicationResourceUsageReport.java | 43 +++-
 .../src/main/proto/yarn_protos.proto|  2 +
 .../hadoop/yarn/client/cli/ApplicationCLI.java  |  7 
 .../hadoop/yarn/client/cli/TestYarnCLI.java |  4 +-
 .../ApplicationResourceUsageReportPBImpl.java   | 28 +
 ...pplicationHistoryManagerOnTimelineStore.java |  8 +++-
 ...pplicationHistoryManagerOnTimelineStore.java | 11 -
 .../metrics/ApplicationMetricsConstants.java|  6 +++
 .../hadoop/yarn/server/utils/BuilderUtils.java  |  5 ++-
 .../server/resourcemanager/RMAppManager.java|  2 +
 .../server/resourcemanager/RMServerUtils.java   |  2 +-
 .../metrics/TimelineServiceV1Publisher.java |  4 ++
 .../metrics/TimelineServiceV2Publisher.java |  6 +++
 .../resourcemanager/recovery/RMStateStore.java  | 10 +++--
 .../records/ApplicationAttemptStateData.java| 39 --
 .../pb/ApplicationAttemptStateDataPBImpl.java   | 24 +++
 .../server/resourcemanager/rmapp/RMAppImpl.java | 13 +-
 .../resourcemanager/rmapp/RMAppMetrics.java | 16 +++-
 .../rmapp/attempt/RMAppAttemptImpl.java | 11 -
 .../rmapp/attempt/RMAppAttemptMetrics.java  | 16 
 .../rmcontainer/RMContainerImpl.java| 16 
 .../scheduler/SchedulerApplicationAttempt.java  |  2 +-
 .../resourcemanager/webapp/RMAppBlock.java  |  8 +++-
 .../resourcemanager/webapp/dao/AppInfo.java | 11 +
 .../yarn_server_resourcemanager_recovery.proto  |  2 +
 .../server/resourcemanager/TestAppManager.java  |  3 +-
 .../applicationsmanager/MockAsm.java|  4 +-
 .../metrics/TestSystemMetricsPublisher.java | 13 +-
 .../TestSystemMetricsPublisherForV2.java|  4 +-
 .../recovery/RMStateStoreTestBase.java  |  4 +-
 .../recovery/TestZKRMStateStore.java|  2 +-
 .../resourcemanager/webapp/TestAppPage.java |  3 +-
 .../webapp/TestRMWebAppFairScheduler.java   |  3 +-
 .../webapp/TestRMWebServicesApps.java   |  2 +-
 34 files changed, 297 insertions(+), 37 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
index 34efee8..3cf8f3d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
@@ -36,7 +36,8 @@ public abstract class ApplicationResourceUsageReport {
   public static ApplicationResourceUsageReport newInstance(
   int numUsedContainers, int numReservedContainers, Resource usedResources,
   Resource reservedResources, Resource neededResources, long memorySeconds,
-  long vcoreSeconds, float queueUsagePerc, float clusterUsagePerc) {
+  long vcoreSeconds, float queueUsagePerc, float clusterUsagePerc,
+  long preemptedMemorySeconds, long preemptedVcoresSeconds) {
 ApplicationResourceUsageReport report =
 Records.newRecord(ApplicationResourceUsageReport.class);
 report.setNumUsedContainers(numUsedContainers);
@@ -48,6 +49,8 @@ public abstract class ApplicationResourceUsageReport {
 report.setVcoreSeconds(vcoreSeconds);
 report.setQueueUsagePercentage(queueUsagePerc);
 report.setClusterUsagePercentage(clusterUsagePerc);
+report.setPreemptedMemorySeconds(preemptedMemorySeconds);
+report.setPreemptedVcoreSeconds(preemptedVcoresSeconds);
 return report;
   }
 
@@ -188,4 +191,42 @@ public abstract class ApplicationResourceUsageReport {
   @Private
   @Unstable
   public abstract void setClusterUsagePercentage(float clusterUsagePerc);
+
+  /**
+   * Set the aggregated amount of memory preempted (in

hadoop git commit: YARN-4218. Metric for resource*time that was preempted. Contributed by Chang Li.

2016-11-10 Thread epayne 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 d81706cd9 -> 1a6a5af44


YARN-4218. Metric for resource*time that was preempted. Contributed by Chang Li.

(cherry picked from commit dd5b9dabf90dbd68afdc87421101a5f4bad357d9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1a6a5af4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1a6a5af4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1a6a5af4

Branch: refs/heads/branch-2.8
Commit: 1a6a5af44a7a820cfacffe16521bbe71c968077e
Parents: d81706c
Author: Eric Payne 
Authored: Thu Nov 10 23:02:42 2016 +
Committer: Eric Payne 
Committed: Thu Nov 10 23:32:17 2016 +

--
 .../records/ApplicationResourceUsageReport.java | 43 +++-
 .../src/main/proto/yarn_protos.proto|  2 +
 .../hadoop/yarn/client/cli/ApplicationCLI.java  |  7 
 .../hadoop/yarn/client/cli/TestYarnCLI.java |  4 +-
 .../ApplicationResourceUsageReportPBImpl.java   | 28 +
 ...pplicationHistoryManagerOnTimelineStore.java |  8 +++-
 ...pplicationHistoryManagerOnTimelineStore.java | 11 -
 .../metrics/ApplicationMetricsConstants.java|  6 +++
 .../hadoop/yarn/server/utils/BuilderUtils.java  |  5 ++-
 .../server/resourcemanager/RMAppManager.java|  2 +
 .../server/resourcemanager/RMServerUtils.java   |  2 +-
 .../metrics/SystemMetricsPublisher.java |  6 ++-
 .../resourcemanager/recovery/RMStateStore.java  | 10 +++--
 .../records/ApplicationAttemptStateData.java| 39 --
 .../pb/ApplicationAttemptStateDataPBImpl.java   | 24 +++
 .../server/resourcemanager/rmapp/RMAppImpl.java | 13 +-
 .../resourcemanager/rmapp/RMAppMetrics.java | 16 +++-
 .../rmapp/attempt/RMAppAttemptImpl.java | 11 -
 .../rmapp/attempt/RMAppAttemptMetrics.java  | 16 
 .../rmcontainer/RMContainerImpl.java| 15 ---
 .../scheduler/SchedulerApplicationAttempt.java  |  4 +-
 .../resourcemanager/webapp/RMAppBlock.java  |  8 +++-
 .../resourcemanager/webapp/dao/AppInfo.java | 12 ++
 .../yarn_server_resourcemanager_recovery.proto  |  2 +
 .../server/resourcemanager/TestAppManager.java  |  3 +-
 .../applicationsmanager/MockAsm.java|  4 +-
 .../metrics/TestSystemMetricsPublisher.java | 13 +-
 .../recovery/RMStateStoreTestBase.java  |  4 +-
 .../recovery/TestZKRMStateStore.java|  2 +-
 .../resourcemanager/webapp/TestAppPage.java |  3 +-
 .../webapp/TestRMWebAppFairScheduler.java   |  3 +-
 .../webapp/TestRMWebServicesApps.java   |  2 +-
 32 files changed, 292 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a6a5af4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
index 34efee8..3cf8f3d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
@@ -36,7 +36,8 @@ public abstract class ApplicationResourceUsageReport {
   public static ApplicationResourceUsageReport newInstance(
   int numUsedContainers, int numReservedContainers, Resource usedResources,
   Resource reservedResources, Resource neededResources, long memorySeconds,
-  long vcoreSeconds, float queueUsagePerc, float clusterUsagePerc) {
+  long vcoreSeconds, float queueUsagePerc, float clusterUsagePerc,
+  long preemptedMemorySeconds, long preemptedVcoresSeconds) {
 ApplicationResourceUsageReport report =
 Records.newRecord(ApplicationResourceUsageReport.class);
 report.setNumUsedContainers(numUsedContainers);
@@ -48,6 +49,8 @@ public abstract class ApplicationResourceUsageReport {
 report.setVcoreSeconds(vcoreSeconds);
 report.setQueueUsagePercentage(queueUsagePerc);
 report.setClusterUsagePercentage(clusterUsagePerc);
+report.setPreemptedMemorySeconds(preemptedMemorySeconds);
+report.setPreemptedVcoreSeconds(preemptedVcoresSeconds);
 return report;
   }
 
@@ -188,4 +191,42 @@ public abstract class ApplicationResourceUsageReport {
   @Private
   @Unstable
   public abstract void setClusterUsagePercentage(float clusterUsagePerc);
+
+  /**
+   * Set the

hadoop git commit: YARN-4218. Metric for resource*time that was preempted. Contributed by Chang Li.

2016-11-10 Thread epayne 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 f4c8dc478 -> dd5b9dabf


YARN-4218. Metric for resource*time that was preempted. Contributed by Chang Li.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dd5b9dab
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dd5b9dab
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dd5b9dab

Branch: refs/heads/branch-2
Commit: dd5b9dabf90dbd68afdc87421101a5f4bad357d9
Parents: f4c8dc4
Author: Eric Payne 
Authored: Thu Nov 10 23:02:42 2016 +
Committer: Eric Payne 
Committed: Thu Nov 10 23:02:42 2016 +

--
 .../records/ApplicationResourceUsageReport.java | 43 +++-
 .../src/main/proto/yarn_protos.proto|  2 +
 .../hadoop/yarn/client/cli/ApplicationCLI.java  |  7 
 .../hadoop/yarn/client/cli/TestYarnCLI.java |  4 +-
 .../ApplicationResourceUsageReportPBImpl.java   | 28 +
 ...pplicationHistoryManagerOnTimelineStore.java |  8 +++-
 ...pplicationHistoryManagerOnTimelineStore.java | 11 -
 .../metrics/ApplicationMetricsConstants.java|  6 +++
 .../hadoop/yarn/server/utils/BuilderUtils.java  |  5 ++-
 .../server/resourcemanager/RMAppManager.java|  2 +
 .../server/resourcemanager/RMServerUtils.java   |  2 +-
 .../metrics/SystemMetricsPublisher.java |  6 ++-
 .../resourcemanager/recovery/RMStateStore.java  | 10 +++--
 .../records/ApplicationAttemptStateData.java| 39 --
 .../pb/ApplicationAttemptStateDataPBImpl.java   | 24 +++
 .../server/resourcemanager/rmapp/RMAppImpl.java | 13 +-
 .../resourcemanager/rmapp/RMAppMetrics.java | 16 +++-
 .../rmapp/attempt/RMAppAttemptImpl.java | 11 -
 .../rmapp/attempt/RMAppAttemptMetrics.java  | 16 
 .../rmcontainer/RMContainerImpl.java| 15 ---
 .../scheduler/SchedulerApplicationAttempt.java  |  2 +-
 .../resourcemanager/webapp/RMAppBlock.java  |  8 +++-
 .../resourcemanager/webapp/dao/AppInfo.java | 11 +
 .../yarn_server_resourcemanager_recovery.proto  |  2 +
 .../server/resourcemanager/TestAppManager.java  |  3 +-
 .../applicationsmanager/MockAsm.java|  4 +-
 .../metrics/TestSystemMetricsPublisher.java | 13 +-
 .../recovery/RMStateStoreTestBase.java  |  4 +-
 .../recovery/TestZKRMStateStore.java|  2 +-
 .../resourcemanager/webapp/TestAppPage.java |  3 +-
 .../webapp/TestRMWebAppFairScheduler.java   |  3 +-
 .../webapp/TestRMWebServicesApps.java   |  2 +-
 32 files changed, 290 insertions(+), 35 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd5b9dab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
index 34efee8..3cf8f3d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
@@ -36,7 +36,8 @@ public abstract class ApplicationResourceUsageReport {
   public static ApplicationResourceUsageReport newInstance(
   int numUsedContainers, int numReservedContainers, Resource usedResources,
   Resource reservedResources, Resource neededResources, long memorySeconds,
-  long vcoreSeconds, float queueUsagePerc, float clusterUsagePerc) {
+  long vcoreSeconds, float queueUsagePerc, float clusterUsagePerc,
+  long preemptedMemorySeconds, long preemptedVcoresSeconds) {
 ApplicationResourceUsageReport report =
 Records.newRecord(ApplicationResourceUsageReport.class);
 report.setNumUsedContainers(numUsedContainers);
@@ -48,6 +49,8 @@ public abstract class ApplicationResourceUsageReport {
 report.setVcoreSeconds(vcoreSeconds);
 report.setQueueUsagePercentage(queueUsagePerc);
 report.setClusterUsagePercentage(clusterUsagePerc);
+report.setPreemptedMemorySeconds(preemptedMemorySeconds);
+report.setPreemptedVcoreSeconds(preemptedVcoresSeconds);
 return report;
   }
 
@@ -188,4 +191,42 @@ public abstract class ApplicationResourceUsageReport {
   @Private
   @Unstable
   public abstract void setClusterUsagePercentage(float clusterUsagePerc);
+
+  /**
+   * Set the aggregated amount of memory preempted (in megabytes)
+   * the application has

hadoop git commit: YARN-4218. Metric for resource*time that was preempted. Contributed by Chang Li.

2016-11-10 Thread epayne 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 3a9841953 -> 93eeb1316


YARN-4218. Metric for resource*time that was preempted. Contributed by Chang Li.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/93eeb131
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/93eeb131
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/93eeb131

Branch: refs/heads/trunk
Commit: 93eeb13164707d0e3556c2bf737bd2ee09a335c6
Parents: 3a98419
Author: Eric Payne 
Authored: Thu Nov 10 22:35:12 2016 +
Committer: Eric Payne 
Committed: Thu Nov 10 22:35:12 2016 +

--
 .../records/ApplicationResourceUsageReport.java | 43 +++-
 .../src/main/proto/yarn_protos.proto|  2 +
 .../hadoop/yarn/client/cli/ApplicationCLI.java  |  7 
 .../hadoop/yarn/client/cli/TestYarnCLI.java |  4 +-
 .../ApplicationResourceUsageReportPBImpl.java   | 28 +
 ...pplicationHistoryManagerOnTimelineStore.java |  8 +++-
 ...pplicationHistoryManagerOnTimelineStore.java | 11 -
 .../metrics/ApplicationMetricsConstants.java|  6 +++
 .../hadoop/yarn/server/utils/BuilderUtils.java  |  5 ++-
 .../server/resourcemanager/RMAppManager.java|  2 +
 .../server/resourcemanager/RMServerUtils.java   |  2 +-
 .../metrics/TimelineServiceV1Publisher.java |  4 ++
 .../metrics/TimelineServiceV2Publisher.java |  6 +++
 .../resourcemanager/recovery/RMStateStore.java  | 10 +++--
 .../records/ApplicationAttemptStateData.java| 39 --
 .../pb/ApplicationAttemptStateDataPBImpl.java   | 24 +++
 .../server/resourcemanager/rmapp/RMAppImpl.java | 13 +-
 .../resourcemanager/rmapp/RMAppMetrics.java | 16 +++-
 .../rmapp/attempt/RMAppAttemptImpl.java | 11 -
 .../rmapp/attempt/RMAppAttemptMetrics.java  | 16 
 .../rmcontainer/RMContainerImpl.java| 16 
 .../scheduler/SchedulerApplicationAttempt.java  |  2 +-
 .../resourcemanager/webapp/RMAppBlock.java  |  8 +++-
 .../resourcemanager/webapp/dao/AppInfo.java | 11 +
 .../yarn_server_resourcemanager_recovery.proto  |  2 +
 .../server/resourcemanager/TestAppManager.java  |  3 +-
 .../applicationsmanager/MockAsm.java|  4 +-
 .../metrics/TestSystemMetricsPublisher.java | 13 +-
 .../TestSystemMetricsPublisherForV2.java|  4 +-
 .../recovery/RMStateStoreTestBase.java  |  4 +-
 .../recovery/TestZKRMStateStore.java|  2 +-
 .../resourcemanager/webapp/TestAppPage.java |  3 +-
 .../webapp/TestRMWebAppFairScheduler.java   |  3 +-
 .../webapp/TestRMWebServicesApps.java   |  2 +-
 34 files changed, 297 insertions(+), 37 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
index 34efee8..3cf8f3d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
@@ -36,7 +36,8 @@ public abstract class ApplicationResourceUsageReport {
   public static ApplicationResourceUsageReport newInstance(
   int numUsedContainers, int numReservedContainers, Resource usedResources,
   Resource reservedResources, Resource neededResources, long memorySeconds,
-  long vcoreSeconds, float queueUsagePerc, float clusterUsagePerc) {
+  long vcoreSeconds, float queueUsagePerc, float clusterUsagePerc,
+  long preemptedMemorySeconds, long preemptedVcoresSeconds) {
 ApplicationResourceUsageReport report =
 Records.newRecord(ApplicationResourceUsageReport.class);
 report.setNumUsedContainers(numUsedContainers);
@@ -48,6 +49,8 @@ public abstract class ApplicationResourceUsageReport {
 report.setVcoreSeconds(vcoreSeconds);
 report.setQueueUsagePercentage(queueUsagePerc);
 report.setClusterUsagePercentage(clusterUsagePerc);
+report.setPreemptedMemorySeconds(preemptedMemorySeconds);
+report.setPreemptedVcoreSeconds(preemptedVcoresSeconds);
 return report;
   }
 
@@ -188,4 +191,42 @@ public abstract class ApplicationResourceUsageReport {
   @Private
   @Unstable
   public abstract void setClusterUsagePercentage(float