This is an automated email from the ASF dual-hosted git repository.
wwei pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 50094d7 YARN-9300. Lazy preemption should trigger an update on queue
preemption metrics for CapacityScheduler. Contributed by Tao Yang.
50094d7 is described below
commit 50094d7fefafaf0e88798b43cdce291ae067e3fd
Author: Weiwei Yang <[email protected]>
AuthorDate: Sun Feb 24 22:17:29 2019 +0800
YARN-9300. Lazy preemption should trigger an update on queue preemption
metrics for CapacityScheduler. Contributed by Tao Yang.
---
.../scheduler/capacity/CapacityScheduler.java | 22 ----------------------
.../scheduler/capacity/LeafQueue.java | 22 ++++++++++++++++++++++
.../TestCapacitySchedulerLazyPreemption.java | 11 ++++++++---
3 files changed, 30 insertions(+), 25 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 618ee20..8ecc2f0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -35,7 +35,6 @@ import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.time.DateUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
@@ -48,7 +47,6 @@ import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.ExecutionType;
@@ -2098,26 +2096,6 @@ public class CapacityScheduler extends
LeafQueue queue = (LeafQueue) application.getQueue();
queue.completedContainer(getClusterResource(), application, node,
rmContainer, containerStatus, event, null, true);
- if (ContainerExitStatus.PREEMPTED == containerStatus.getExitStatus()) {
- updateQueuePreemptionMetrics(queue, rmContainer);
- }
- }
-
- private void updateQueuePreemptionMetrics(
- CSQueue queue, RMContainer rmc) {
- QueueMetrics qMetrics = queue.getMetrics();
- final long usedMillis = rmc.getFinishTime() - rmc.getCreationTime();
- final long usedSeconds = usedMillis / DateUtils.MILLIS_PER_SECOND;
- Resource containerResource = rmc.getAllocatedResource();
- qMetrics.preemptContainer();
- long mbSeconds = (containerResource.getMemorySize() * usedMillis)
- / DateUtils.MILLIS_PER_SECOND;
- long vcSeconds = (containerResource.getVirtualCores() * usedMillis)
- / DateUtils.MILLIS_PER_SECOND;
- qMetrics.updatePreemptedMemoryMBSeconds(mbSeconds);
- qMetrics.updatePreemptedVcoreSeconds(vcSeconds);
- qMetrics.updatePreemptedSecondsForCustomResources(containerResource,
- usedSeconds);
}
@Lock(Lock.NoLock.class)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 30ae615..333d1f1 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -24,6 +24,7 @@ import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.time.DateUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -1701,6 +1702,12 @@ public class LeafQueue extends AbstractCSQueue {
// Notify PreemptionManager
csContext.getPreemptionManager().removeKillableContainer(
new KillableContainer(rmContainer, node.getPartition(), queueName));
+
+ // Update preemption metrics if exit status is PREEMPTED
+ if (containerStatus != null
+ && ContainerExitStatus.PREEMPTED == containerStatus.getExitStatus()) {
+ updateQueuePreemptionMetrics(rmContainer);
+ }
}
void allocateResource(Resource clusterResource,
@@ -2217,4 +2224,19 @@ public class LeafQueue extends AbstractCSQueue {
public long getDefaultApplicationLifetime() {
return defaultApplicationLifetime;
}
+
+ private void updateQueuePreemptionMetrics(RMContainer rmc) {
+ final long usedMillis = rmc.getFinishTime() - rmc.getCreationTime();
+ final long usedSeconds = usedMillis / DateUtils.MILLIS_PER_SECOND;
+ Resource containerResource = rmc.getAllocatedResource();
+ metrics.preemptContainer();
+ long mbSeconds = (containerResource.getMemorySize() * usedMillis)
+ / DateUtils.MILLIS_PER_SECOND;
+ long vcSeconds = (containerResource.getVirtualCores() * usedMillis)
+ / DateUtils.MILLIS_PER_SECOND;
+ metrics.updatePreemptedMemoryMBSeconds(mbSeconds);
+ metrics.updatePreemptedVcoreSeconds(vcSeconds);
+ metrics.updatePreemptedSecondsForCustomResources(containerResource,
+ usedSeconds);
+ }
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerLazyPreemption.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerLazyPreemption.java
index a4c7d61..a5d3539 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerLazyPreemption.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerLazyPreemption.java
@@ -25,7 +25,6 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
-import
org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingEditPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingMonitor;
import
org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingMonitorManager;
import
org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy;
@@ -48,8 +47,6 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Set;
-import static org.mockito.Mockito.mock;
-
public class TestCapacitySchedulerLazyPreemption
extends CapacitySchedulerPreemptionTestBase {
@Override
@@ -156,6 +153,14 @@ public class TestCapacitySchedulerLazyPreemption
Assert.assertEquals(6, schedulerApp1.getLiveContainers().size());
Assert.assertEquals(2, schedulerApp2.getLiveContainers().size());
+ // Ensure preemption metrics were recored.
+ Assert.assertEquals(
+ "Number of preempted containers incorrectly recorded:", 1,
+ cs.getQueue("a").getMetrics().getAggregatePreemptedContainers());
+ Assert.assertEquals(
+ "Number of preempted containers incorrectly recorded:", 1,
+ cs.getRootQueue().getMetrics().getAggregatePreemptedContainers());
+
rm1.close();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
