Comments to document the changes to FSPreemptionThread
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/450f956a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/450f956a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/450f956a Branch: refs/heads/fs-preemption Commit: 450f956a3f1b948b544d0cfd837f3af0fc9837e5 Parents: e4eec25 Author: Karthik Kambatla <ka...@apache.org> Authored: Wed Jun 15 09:14:32 2016 -0700 Committer: Karthik Kambatla <ka...@apache.org> Committed: Wed Jun 15 09:14:32 2016 -0700 ---------------------------------------------------------------------- .../scheduler/fair/FSPreemptionThread.java | 40 +++++++++++++++----- 1 file changed, 31 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/450f956a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java index 0e99b64..766fd5a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java @@ -74,39 +74,61 @@ public class FSPreemptionThread extends Thread { } /** - * Returns a non-null PremptionContext if it finds a node that can - * accommodate a request from this app. Also, reserves the node for this app. + * Given an app, identify containers to preempt to satisfy the app's next + * resource request. + * + * @param starvedApp + * @return */ - private List<RMContainer> identifyContainersToPreempt(FSAppAttempt starvedApp) { - List<RMContainer> containers = new ArrayList<>(); + private List<RMContainer> identifyContainersToPreempt(FSAppAttempt + starvedApp) { + List<RMContainer> containers = new ArrayList<>(); // return value + + // Find the nodes that match the next resource request ResourceRequest request = starvedApp.getNextResourceRequest(); + // TODO (KK): Should we check other resource requests if we can't match + // the first one? + Resource requestCapability = request.getCapability(); - List<FSSchedulerNode> nodes = + List<FSSchedulerNode> potentialNodes = scheduler.getNodeTracker().getNodes(request.getResourceName()); + + // From the potential nodes, pick a node that has enough containers + // from apps over their fairshare FSSchedulerNode targetNode = null; - Resource potential = Resources.clone(Resources.none()); - for (FSSchedulerNode node : nodes) { + for (FSSchedulerNode node : potentialNodes) { containers.clear(); - potential = Resources.clone(Resources.none()); + Resource potential = Resources.clone(Resources.none()); for (RMContainer container : node.getCopiedListOfRunningContainers()) { Resource containerResource = container.getAllocatedResource(); FSAppAttempt app = scheduler.getSchedulerApp(container.getApplicationAttemptId()); + + // Check if the app's allocation will be over its fairshare even + // after preempting this container if (Resources.fitsIn(containerResource, Resources.subtract(app.getResourceUsage(), app.getFairShare()))) { Resources.addTo(potential, containerResource); } + + // TODO (KK): Should we go through other app reservations if the + // containers alone are not enough to meet the starvedApp's requirements + + // Check if we have already identified enough containers if (Resources.fitsIn(requestCapability, potential)) { break; } } + + // Set targetNode if this node has enough containers to preempt if (Resources.fitsIn(requestCapability, potential)) { targetNode = node; break; } } - if (Resources.fitsIn(requestCapability, potential)) { + if (targetNode != null) { + // Reserve resources on the target node so it doesn't go to other nodes starvedApp.reserve(targetNode, requestCapability); return containers; } else { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
