subject:"\[42\/50\] \[abbrv\] hadoop git commit\: YARN\-1015. FS should watch node resource utilization and allocate opportunistic containers if appropriate."

[42/50] [abbrv] hadoop git commit: YARN-1015. FS should watch node resource utilization and allocate opportunistic containers if appropriate.

2018-04-10 Thread haibochen

YARN-1015. FS should watch node resource utilization and allocate opportunistic 
containers if appropriate.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/82ef338d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/82ef338d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/82ef338d

Branch: refs/heads/YARN-1011
Commit: 82ef338d65f957ff58ca725c4ea52f504167f29b
Parents: b237095
Author: Haibo Chen 
Authored: Fri Nov 17 07:47:32 2017 -0800
Committer: Haibo Chen 
Committed: Mon Apr 9 17:13:04 2018 -0700

--
 .../sls/scheduler/FairSchedulerMetrics.java |   4 +-
 .../hadoop/yarn/conf/YarnConfiguration.java |   5 +
 .../src/main/resources/yarn-default.xml |  13 +
 .../scheduler/SchedulerNode.java|  48 ++
 .../scheduler/fair/FSAppAttempt.java| 166 ---
 .../scheduler/fair/FSLeafQueue.java |  51 +-
 .../scheduler/fair/FSParentQueue.java   |  36 +-
 .../resourcemanager/scheduler/fair/FSQueue.java |  39 +-
 .../scheduler/fair/FairScheduler.java   |  97 ++--
 .../fair/FairSchedulerConfiguration.java|   5 +
 .../scheduler/fair/Schedulable.java |  17 +-
 .../DominantResourceFairnessPolicy.java |   8 +-
 .../fair/policies/FairSharePolicy.java  |   4 +-
 .../webapp/dao/FairSchedulerQueueInfo.java  |   2 +-
 .../yarn/server/resourcemanager/MockNodes.java  |  60 ++-
 .../TestWorkPreservingRMRestart.java|   2 +-
 .../scheduler/fair/FakeSchedulable.java |   9 +-
 .../scheduler/fair/TestAppRunnability.java  |   9 +-
 .../scheduler/fair/TestFSAppAttempt.java|   4 +-
 .../scheduler/fair/TestFSLeafQueue.java |   4 +-
 .../scheduler/fair/TestFSSchedulerNode.java |   4 +-
 .../scheduler/fair/TestFairScheduler.java   | 468 +--
 .../scheduler/fair/TestSchedulingPolicy.java|  10 +-
 23 files changed, 861 insertions(+), 204 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef338d/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
index a5aee74..1f4e7c7 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
@@ -75,7 +75,7 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
   case DEMAND:
 return schedulable.getDemand().getMemorySize();
   case USAGE:
-return schedulable.getResourceUsage().getMemorySize();
+return schedulable.getGuaranteedResourceUsage().getMemorySize();
   case MINSHARE:
 return schedulable.getMinShare().getMemorySize();
   case MAXSHARE:
@@ -96,7 +96,7 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
   case DEMAND:
 return schedulable.getDemand().getVirtualCores();
   case USAGE:
-return schedulable.getResourceUsage().getVirtualCores();
+return schedulable.getGuaranteedResourceUsage().getVirtualCores();
   case MINSHARE:
 return schedulable.getMinShare().getVirtualCores();
   case MAXSHARE:

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef338d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index fdbbdbf..4c1c6d5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -270,6 +270,11 @@ public class YarnConfiguration extends Configuration {
 
   public static final String APP_NAME_PLACEMENT_RULE = "app-name";
 
+  public static final String RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED =
+  RM_PREFIX + "scheduler.oversubscription.enabled";
+  public static final boolean DEFAULT_RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED
+  = false;
+
   /** Enable Resource Manager webapp ui actions */
   public static final String RM_WEBAPP_UI_ACTIONS_ENABLED =
 RM_PREFIX + "webapp.ui-actions.enabled";

http://git-wip-us.apach

[42/50] [abbrv] hadoop git commit: YARN-1015. FS should watch node resource utilization and allocate opportunistic containers if appropriate.

2018-01-28 Thread haibochen

YARN-1015. FS should watch node resource utilization and allocate opportunistic 
containers if appropriate.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bdf3e661
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bdf3e661
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bdf3e661

Branch: refs/heads/YARN-1011
Commit: bdf3e661b7558020c284442f24ade677406b19e8
Parents: eba8436
Author: Haibo Chen 
Authored: Fri Nov 17 07:47:32 2017 -0800
Committer: Haibo Chen 
Committed: Sun Jan 28 18:08:11 2018 -0800

--
 .../sls/scheduler/FairSchedulerMetrics.java |   4 +-
 .../hadoop/yarn/conf/YarnConfiguration.java |   5 +
 .../src/main/resources/yarn-default.xml |  13 +
 .../scheduler/SchedulerNode.java|  48 ++
 .../scheduler/fair/FSAppAttempt.java| 166 ---
 .../scheduler/fair/FSLeafQueue.java |  51 +-
 .../scheduler/fair/FSParentQueue.java   |  36 +-
 .../resourcemanager/scheduler/fair/FSQueue.java |  39 +-
 .../scheduler/fair/FairScheduler.java   |  97 ++--
 .../fair/FairSchedulerConfiguration.java|   5 +
 .../scheduler/fair/Schedulable.java |  17 +-
 .../DominantResourceFairnessPolicy.java |   8 +-
 .../fair/policies/FairSharePolicy.java  |   4 +-
 .../webapp/dao/FairSchedulerQueueInfo.java  |   2 +-
 .../yarn/server/resourcemanager/MockNodes.java  |  60 ++-
 .../TestWorkPreservingRMRestart.java|   2 +-
 .../scheduler/fair/FakeSchedulable.java |   9 +-
 .../scheduler/fair/TestAppRunnability.java  |   9 +-
 .../scheduler/fair/TestFSAppAttempt.java|   4 +-
 .../scheduler/fair/TestFSLeafQueue.java |   4 +-
 .../scheduler/fair/TestFSSchedulerNode.java |   4 +-
 .../scheduler/fair/TestFairScheduler.java   | 468 +--
 .../scheduler/fair/TestSchedulingPolicy.java|  10 +-
 23 files changed, 861 insertions(+), 204 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdf3e661/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
index a5aee74..1f4e7c7 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
@@ -75,7 +75,7 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
   case DEMAND:
 return schedulable.getDemand().getMemorySize();
   case USAGE:
-return schedulable.getResourceUsage().getMemorySize();
+return schedulable.getGuaranteedResourceUsage().getMemorySize();
   case MINSHARE:
 return schedulable.getMinShare().getMemorySize();
   case MAXSHARE:
@@ -96,7 +96,7 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
   case DEMAND:
 return schedulable.getDemand().getVirtualCores();
   case USAGE:
-return schedulable.getResourceUsage().getVirtualCores();
+return schedulable.getGuaranteedResourceUsage().getVirtualCores();
   case MINSHARE:
 return schedulable.getMinShare().getVirtualCores();
   case MAXSHARE:

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdf3e661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index ed8404c..15f6480 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -268,6 +268,11 @@ public class YarnConfiguration extends Configuration {
   /** UserGroupMappingPlacementRule configuration string. */
   public static final String USER_GROUP_PLACEMENT_RULE = "user-group";
 
+  public static final String RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED =
+  RM_PREFIX + "scheduler.oversubscription.enabled";
+  public static final boolean DEFAULT_RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED
+  = false;
+
   /** Enable Resource Manager webapp ui actions */
   public static final String RM_WEBAPP_UI_ACTIONS_ENABLED =

[42/50] [abbrv] hadoop git commit: YARN-1015. FS should watch node resource utilization and allocate opportunistic containers if appropriate.

[42/50] [abbrv] hadoop git commit: YARN-1015. FS should watch node resource utilization and allocate opportunistic containers if appropriate.

2 matches

Site Navigation

Mail list logo

Footer information