hadoop git commit: YARN-6775. CapacityScheduler: Improvements to assignContainers, avoid unnecessary canAssignToUser/Queue calls. (Nathan Roberts via wangda)

2017-07-19 Thread wangda 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 6ed569df2 -> a39617df6


YARN-6775. CapacityScheduler: Improvements to assignContainers, avoid 
unnecessary canAssignToUser/Queue calls. (Nathan Roberts via wangda)

Change-Id: I5951f0997547de7d2e4a30b4ad87ab0a59b3066a


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a39617df
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a39617df
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a39617df

Branch: refs/heads/branch-2.8
Commit: a39617df63e758629123da50d0690f3c13657eca
Parents: 6ed569d
Author: Wangda Tan 
Authored: Wed Jul 19 15:29:45 2017 -0700
Committer: Wangda Tan 
Committed: Wed Jul 19 15:29:45 2017 -0700

--
 .../scheduler/capacity/LeafQueue.java   |  93 
 .../capacity/TestCapacityScheduler.java | 145 +++
 .../scheduler/capacity/TestLeafQueue.java   |  10 +-
 .../scheduler/capacity/TestReservations.java|   6 +-
 4 files changed, 221 insertions(+), 33 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a39617df/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 7258aee..bd3bdff 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -956,25 +956,56 @@ public class LeafQueue extends AbstractCSQueue {
 return CSAssignment.NULL_ASSIGNMENT;
   }
 
+  Map userLimits = new HashMap<>();
+  boolean needAssignToQueueCheck = true;
   for (Iterator assignmentIterator =
orderingPolicy.getAssignmentIterator(); assignmentIterator
.hasNext(); ) {
 FiCaSchedulerApp application = assignmentIterator.next();
 
 // Check queue max-capacity limit
-if (!super.canAssignToThisQueue(clusterResource, node.getPartition(),
-currentResourceLimits, application.getCurrentReservation(),
-schedulingMode)) {
-  return CSAssignment.NULL_ASSIGNMENT;
+Resource appReserved = application.getCurrentReservation();
+if (needAssignToQueueCheck) {
+  if (!super.canAssignToThisQueue(clusterResource, node.getPartition(),
+  currentResourceLimits, appReserved, schedulingMode)) {
+return CSAssignment.NULL_ASSIGNMENT;
+  }
+  // If there was no reservation and canAssignToThisQueue returned
+  // true, there is no reason to check further.
+  if (!this.reservationsContinueLooking
+  || appReserved.equals(Resources.none()) || !node.getPartition()
+  .equals(CommonNodeLabelsManager.NO_LABEL)) {
+needAssignToQueueCheck = false;
+  }
 }
 
+CachedUserLimit cul = userLimits.get(application.getUser());
+Resource cachedUserLimit = null;
+if (cul != null) {
+  cachedUserLimit = cul.userLimit;
+}
 Resource userLimit =
 computeUserLimitAndSetHeadroom(application, clusterResource,
-node.getPartition(), schedulingMode);
+node.getPartition(), schedulingMode, cachedUserLimit);
+if (cul == null) {
+  cul = new CachedUserLimit(userLimit);
+  userLimits.put(application.getUser(), cul);
+}
 
 // Check user limit
-if (!canAssignToUser(clusterResource, application.getUser(), userLimit,
-application, node.getPartition(), currentResourceLimits)) {
+boolean userAssignable = true;
+if (!cul.canAssign && Resources.fitsIn(appReserved, cul.reservation)) {
+  userAssignable = false;
+} else {
+  userAssignable =
+  canAssignToUser(clusterResource, application.getUser(), 
userLimit,
+  appReserved, node.getPartition(), currentResourceLimits);
+  if (!userAssignable && Resources.fitsIn(cul.reservation,

[06/50] [abbrv] hadoop git commit: YARN-6775. CapacityScheduler: Improvements to assignContainers, avoid unnecessary canAssignToUser/Queue calls. (Nathan Roberts via wangda)

2017-07-19 Thread xyao 
YARN-6775. CapacityScheduler: Improvements to assignContainers, avoid 
unnecessary canAssignToUser/Queue calls. (Nathan Roberts via wangda)

Change-Id: I84ccd54200ccbaae23018ef320028e42b4c3509a


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/945c0958
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/945c0958
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/945c0958

Branch: refs/heads/HDFS-7240
Commit: 945c0958bb8df3dd9d5f1467f1216d2e6b0ee3d8
Parents: b61ab85
Author: Wangda Tan 
Authored: Thu Jul 13 10:30:15 2017 -0700
Committer: Wangda Tan 
Committed: Thu Jul 13 10:30:15 2017 -0700

--
 .../scheduler/activities/ActivitiesLogger.java  |  33 +++--
 .../scheduler/capacity/LeafQueue.java   |  83 ---
 .../capacity/TestCapacityScheduler.java | 146 ++-
 .../scheduler/capacity/TestLeafQueue.java   |  10 +-
 4 files changed, 231 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/945c0958/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesLogger.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesLogger.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesLogger.java
index 3f8ed55..12aff02 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesLogger.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesLogger.java
@@ -63,9 +63,14 @@ public class ActivitiesLogger {
 SchedulerApplicationAttempt application, Priority priority,
 String diagnostic) {
   String type = "app";
-  recordActivity(activitiesManager, node, application.getQueueName(),
-  application.getApplicationId().toString(), priority,
-  ActivityState.REJECTED, diagnostic, type);
+  if (activitiesManager == null) {
+return;
+  }
+  if (activitiesManager.shouldRecordThisNode(node.getNodeID())) {
+recordActivity(activitiesManager, node, application.getQueueName(),
+application.getApplicationId().toString(), priority,
+ActivityState.REJECTED, diagnostic, type);
+  }
   finishSkippedAppAllocationRecording(activitiesManager,
   application.getApplicationId(), ActivityState.REJECTED, diagnostic);
 }
@@ -203,8 +208,13 @@ public class ActivitiesLogger {
 public static void recordQueueActivity(ActivitiesManager activitiesManager,
 SchedulerNode node, String parentQueueName, String queueName,
 ActivityState state, String diagnostic) {
-  recordActivity(activitiesManager, node, parentQueueName, queueName, null,
-  state, diagnostic, null);
+  if (activitiesManager == null) {
+return;
+  }
+  if (activitiesManager.shouldRecordThisNode(node.getNodeID())) {
+recordActivity(activitiesManager, node, parentQueueName, queueName,
+null, state, diagnostic, null);
+  }
 }
   }
 
@@ -266,13 +276,10 @@ public class ActivitiesLogger {
   private static void recordActivity(ActivitiesManager activitiesManager,
   SchedulerNode node, String parentName, String childName,
   Priority priority, ActivityState state, String diagnostic, String type) {
-if (activitiesManager == null) {
-  return;
-}
-if (activitiesManager.shouldRecordThisNode(node.getNodeID())) {
-  activitiesManager.addSchedulingActivityForNode(node.getNodeID(),
-  parentName, childName, priority != null ? priority.toString() : null,
-  state, diagnostic, type);
-}
+
+activitiesManager.addSchedulingActivityForNode(node.getNodeID(), 
parentName,
+childName, priority != null ? priority.toString() : null, state,
+diagnostic, type);
+
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/945c0958/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
--
diff --git

hadoop git commit: YARN-6775. CapacityScheduler: Improvements to assignContainers, avoid unnecessary canAssignToUser/Queue calls. (Nathan Roberts via wangda)

2017-07-19 Thread wangda 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 1d629c95a -> 5070c9ba3


YARN-6775. CapacityScheduler: Improvements to assignContainers, avoid 
unnecessary canAssignToUser/Queue calls. (Nathan Roberts via wangda)

Change-Id: Iaf9bb7e5ed3aa1300abdccf87ae6fcbddbd25e3e


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5070c9ba
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5070c9ba
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5070c9ba

Branch: refs/heads/branch-2
Commit: 5070c9ba39d1f9c6b46ce9df299584ea3ed5f506
Parents: 1d629c9
Author: Wangda Tan 
Authored: Wed Jul 19 15:25:21 2017 -0700
Committer: Wangda Tan 
Committed: Wed Jul 19 15:25:21 2017 -0700

--
 .../scheduler/activities/ActivitiesLogger.java  |  33 +++--
 .../scheduler/capacity/LeafQueue.java   |  83 ---
 .../capacity/TestCapacityScheduler.java | 145 +++
 .../scheduler/capacity/TestLeafQueue.java   |  10 +-
 4 files changed, 231 insertions(+), 40 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5070c9ba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesLogger.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesLogger.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesLogger.java
index 3f8ed55..12aff02 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesLogger.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesLogger.java
@@ -63,9 +63,14 @@ public class ActivitiesLogger {
 SchedulerApplicationAttempt application, Priority priority,
 String diagnostic) {
   String type = "app";
-  recordActivity(activitiesManager, node, application.getQueueName(),
-  application.getApplicationId().toString(), priority,
-  ActivityState.REJECTED, diagnostic, type);
+  if (activitiesManager == null) {
+return;
+  }
+  if (activitiesManager.shouldRecordThisNode(node.getNodeID())) {
+recordActivity(activitiesManager, node, application.getQueueName(),
+application.getApplicationId().toString(), priority,
+ActivityState.REJECTED, diagnostic, type);
+  }
   finishSkippedAppAllocationRecording(activitiesManager,
   application.getApplicationId(), ActivityState.REJECTED, diagnostic);
 }
@@ -203,8 +208,13 @@ public class ActivitiesLogger {
 public static void recordQueueActivity(ActivitiesManager activitiesManager,
 SchedulerNode node, String parentQueueName, String queueName,
 ActivityState state, String diagnostic) {
-  recordActivity(activitiesManager, node, parentQueueName, queueName, null,
-  state, diagnostic, null);
+  if (activitiesManager == null) {
+return;
+  }
+  if (activitiesManager.shouldRecordThisNode(node.getNodeID())) {
+recordActivity(activitiesManager, node, parentQueueName, queueName,
+null, state, diagnostic, null);
+  }
 }
   }
 
@@ -266,13 +276,10 @@ public class ActivitiesLogger {
   private static void recordActivity(ActivitiesManager activitiesManager,
   SchedulerNode node, String parentName, String childName,
   Priority priority, ActivityState state, String diagnostic, String type) {
-if (activitiesManager == null) {
-  return;
-}
-if (activitiesManager.shouldRecordThisNode(node.getNodeID())) {
-  activitiesManager.addSchedulingActivityForNode(node.getNodeID(),
-  parentName, childName, priority != null ? priority.toString() : null,
-  state, diagnostic, type);
-}
+
+activitiesManager.addSchedulingActivityForNode(node.getNodeID(), 
parentName,
+childName, priority != null ? priority.toString() : null, state,
+diagnostic, type);
+
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5070c9ba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java

hadoop git commit: YARN-6775. CapacityScheduler: Improvements to assignContainers, avoid unnecessary canAssignToUser/Queue calls. (Nathan Roberts via wangda)

2017-07-13 Thread wangda 
Repository: hadoop
Updated Branches:
  refs/heads/trunk b61ab8573 -> 945c0958b


YARN-6775. CapacityScheduler: Improvements to assignContainers, avoid 
unnecessary canAssignToUser/Queue calls. (Nathan Roberts via wangda)

Change-Id: I84ccd54200ccbaae23018ef320028e42b4c3509a


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/945c0958
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/945c0958
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/945c0958

Branch: refs/heads/trunk
Commit: 945c0958bb8df3dd9d5f1467f1216d2e6b0ee3d8
Parents: b61ab85
Author: Wangda Tan 
Authored: Thu Jul 13 10:30:15 2017 -0700
Committer: Wangda Tan 
Committed: Thu Jul 13 10:30:15 2017 -0700

--
 .../scheduler/activities/ActivitiesLogger.java  |  33 +++--
 .../scheduler/capacity/LeafQueue.java   |  83 ---
 .../capacity/TestCapacityScheduler.java | 146 ++-
 .../scheduler/capacity/TestLeafQueue.java   |  10 +-
 4 files changed, 231 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/945c0958/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesLogger.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesLogger.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesLogger.java
index 3f8ed55..12aff02 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesLogger.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesLogger.java
@@ -63,9 +63,14 @@ public class ActivitiesLogger {
 SchedulerApplicationAttempt application, Priority priority,
 String diagnostic) {
   String type = "app";
-  recordActivity(activitiesManager, node, application.getQueueName(),
-  application.getApplicationId().toString(), priority,
-  ActivityState.REJECTED, diagnostic, type);
+  if (activitiesManager == null) {
+return;
+  }
+  if (activitiesManager.shouldRecordThisNode(node.getNodeID())) {
+recordActivity(activitiesManager, node, application.getQueueName(),
+application.getApplicationId().toString(), priority,
+ActivityState.REJECTED, diagnostic, type);
+  }
   finishSkippedAppAllocationRecording(activitiesManager,
   application.getApplicationId(), ActivityState.REJECTED, diagnostic);
 }
@@ -203,8 +208,13 @@ public class ActivitiesLogger {
 public static void recordQueueActivity(ActivitiesManager activitiesManager,
 SchedulerNode node, String parentQueueName, String queueName,
 ActivityState state, String diagnostic) {
-  recordActivity(activitiesManager, node, parentQueueName, queueName, null,
-  state, diagnostic, null);
+  if (activitiesManager == null) {
+return;
+  }
+  if (activitiesManager.shouldRecordThisNode(node.getNodeID())) {
+recordActivity(activitiesManager, node, parentQueueName, queueName,
+null, state, diagnostic, null);
+  }
 }
   }
 
@@ -266,13 +276,10 @@ public class ActivitiesLogger {
   private static void recordActivity(ActivitiesManager activitiesManager,
   SchedulerNode node, String parentName, String childName,
   Priority priority, ActivityState state, String diagnostic, String type) {
-if (activitiesManager == null) {
-  return;
-}
-if (activitiesManager.shouldRecordThisNode(node.getNodeID())) {
-  activitiesManager.addSchedulingActivityForNode(node.getNodeID(),
-  parentName, childName, priority != null ? priority.toString() : null,
-  state, diagnostic, type);
-}
+
+activitiesManager.addSchedulingActivityForNode(node.getNodeID(), 
parentName,
+childName, priority != null ? priority.toString() : null, state,
+diagnostic, type);
+
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/945c0958/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java