Repository: hadoop Updated Branches: refs/heads/branch-2.8 3fa2e8658 -> c328ab47f
Revert "YARN-7728. Expose container preemptions related information in Capacity Scheduler queue metrics. Contributed by Eric Payne." This reverts commit 3fa2e8658740feb310c75729c34bdf0c9942e2df. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c328ab47 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c328ab47 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c328ab47 Branch: refs/heads/branch-2.8 Commit: c328ab47fa76a248a31c71b189b4c35340c043ac Parents: 3fa2e86 Author: Jason Lowe <[email protected]> Authored: Thu Jan 25 10:58:33 2018 -0600 Committer: Jason Lowe <[email protected]> Committed: Thu Jan 25 10:58:33 2018 -0600 ---------------------------------------------------------------------- .../resourcemanager/scheduler/QueueMetrics.java | 18 ------------------ .../scheduler/capacity/CapacityScheduler.java | 19 ------------------- .../TestCapacitySchedulerSurgicalPreemption.java | 4 ---- 3 files changed, 41 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c328ab47/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java index 97961da..a7f907b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java @@ -68,10 +68,6 @@ public class QueueMetrics implements MetricsSource { MutableCounterLong aggregateOffSwitchContainersAllocated; @Metric("Aggregate # of preempted containers") MutableCounterLong aggregateContainersPreempted; - @Metric("Aggregate # of preempted memory seconds") MutableCounterLong - aggregateMemoryMBSecondsPreempted; - @Metric("Aggregate # of preempted vcore seconds") MutableCounterLong - aggregateVcoreSecondsPreempted; @Metric("# of active users") MutableGaugeInt activeUsers; @Metric("# of active applications") MutableGaugeInt activeApplications; @Metric("App Attempt First Container Allocation Delay") @@ -525,20 +521,6 @@ public class QueueMetrics implements MetricsSource { } } - public void updatePreemptedMemoryMBSeconds(long mbSeconds) { - aggregateMemoryMBSecondsPreempted.incr(mbSeconds); - if (parent != null) { - parent.updatePreemptedMemoryMBSeconds(mbSeconds); - } - } - - public void updatePreemptedVcoreSeconds(long vcoreSeconds) { - aggregateVcoreSecondsPreempted.incr(vcoreSeconds); - if (parent != null) { - parent.updatePreemptedVcoreSeconds(vcoreSeconds); - } - } - public void reserveResource(String partition, String user, Resource res) { if(partition == null || partition.equals(RMNodeLabelsManager.NO_LABEL)) { reserveResource(user, res); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c328ab47/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index cc9f93c..c7254b3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -21,7 +21,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.time.DateUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate; @@ -36,7 +35,6 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.NodeId; @@ -1611,23 +1609,6 @@ public class CapacityScheduler extends LeafQueue queue = (LeafQueue)application.getQueue(); queue.completedContainer(clusterResource, application, node, rmContainer, containerStatus, event, null, true); - if (ContainerExitStatus.PREEMPTED == containerStatus.getExitStatus()) { - updateQueuePreemptionMetrics(queue, rmContainer); - } - } - - private void updateQueuePreemptionMetrics( - CSQueue queue, RMContainer rmc) { - QueueMetrics qMetrics = queue.getMetrics(); - long usedMillis = rmc.getFinishTime() - rmc.getCreationTime(); - Resource containerResource = rmc.getAllocatedResource(); - qMetrics.preemptContainer(); - long mbSeconds = (containerResource.getMemorySize() * usedMillis) - / DateUtils.MILLIS_PER_SECOND; - long vcSeconds = (containerResource.getVirtualCores() * usedMillis) - / DateUtils.MILLIS_PER_SECOND; - qMetrics.updatePreemptedMemoryMBSeconds(mbSeconds); - qMetrics.updatePreemptedVcoreSeconds(vcSeconds); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/c328ab47/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java index df0c0cf..908e0e8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java @@ -150,10 +150,6 @@ public class TestCapacitySchedulerSurgicalPreemption waitNumberOfLiveContainersOnNodeFromApp(cs.getNode(rmNode2.getNodeID()), am1.getApplicationAttemptId(), 16); - // Ensure preemption metrics were recored. - Assert.assertEquals("Number of preempted containers incorrectly recorded:", - 4, cs.getQueue("root").getMetrics().getAggregatePreemptedContainers()); - rm1.close(); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
