[01/50] [abbrv] hadoop git commit: YARN-4743. FairSharePolicy breaks TimSort assumption. (Zephyr Guo and Yufei Gu via kasha) [Forced Update!]

[rakeshr]

Repository: hadoop
Updated Branches:
  refs/heads/HDFS-10285 f705de356 -> 9ca592892 (forced update)


YARN-4743. FairSharePolicy breaks TimSort assumption. (Zephyr Guo and Yufei Gu 
via kasha)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4df8ed63
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4df8ed63
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4df8ed63

Branch: refs/heads/HDFS-10285
Commit: 4df8ed63ed93f2542e4b48f521b0cc6624ab59c1
Parents: b2c4f24
Author: Karthik Kambatla 
Authored: Thu Oct 27 17:42:44 2016 -0700
Committer: Karthik Kambatla 
Committed: Thu Oct 27 17:45:48 2016 -0700

--
 .../fair/policies/FairSharePolicy.java  |  31 ++-
 .../scheduler/fair/TestSchedulingPolicy.java| 228 +++
 2 files changed, 254 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4df8ed63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
index 6aa8405..f120f0f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
@@ -63,7 +63,11 @@ public class FairSharePolicy extends SchedulingPolicy {
* 
* Schedulables above their min share are compared by (runningTasks / 
weight).
* If all weights are equal, slots are given to the job with the fewest 
tasks;
-   * otherwise, jobs with more weight get proportionally more slots.
+   * otherwise, jobs with more weight get proportionally more slots. If weight
+   * equals to 0, we can't compare Schedulables by (resource usage/weight).
+   * There are two situations: 1)All weights equal to 0, slots are given
+   * to one with less resource usage. 2)Only one of weight equals to 0, slots
+   * are given to the one with non-zero weight.
*/
   private static class FairShareComparator implements Comparator,
   Serializable {
@@ -74,6 +78,7 @@ public class FairSharePolicy extends SchedulingPolicy {
 public int compare(Schedulable s1, Schedulable s2) {
   double minShareRatio1, minShareRatio2;
   double useToWeightRatio1, useToWeightRatio2;
+  double weight1, weight2;
   Resource minShare1 = Resources.min(RESOURCE_CALCULATOR, null,
   s1.getMinShare(), s1.getDemand());
   Resource minShare2 = Resources.min(RESOURCE_CALCULATOR, null,
@@ -86,10 +91,26 @@ public class FairSharePolicy extends SchedulingPolicy {
   / Resources.max(RESOURCE_CALCULATOR, null, minShare1, 
ONE).getMemorySize();
   minShareRatio2 = (double) s2.getResourceUsage().getMemorySize()
   / Resources.max(RESOURCE_CALCULATOR, null, minShare2, 
ONE).getMemorySize();
-  useToWeightRatio1 = s1.getResourceUsage().getMemorySize() /
-  s1.getWeights().getWeight(ResourceType.MEMORY);
-  useToWeightRatio2 = s2.getResourceUsage().getMemorySize() /
-  s2.getWeights().getWeight(ResourceType.MEMORY);
+
+  weight1 = s1.getWeights().getWeight(ResourceType.MEMORY);
+  weight2 = s2.getWeights().getWeight(ResourceType.MEMORY);
+  if (weight1 > 0.0 && weight2 > 0.0) {
+useToWeightRatio1 = s1.getResourceUsage().getMemorySize() / weight1;
+useToWeightRatio2 = s2.getResourceUsage().getMemorySize() / weight2;
+  } else { // Either weight1 or weight2 equals to 0
+if (weight1 == weight2) {
+  // If they have same weight, just compare usage
+  useToWeightRatio1 = s1.getResourceUsage().getMemorySize();
+  useToWeightRatio2 = s2.getResourceUsage().getMemorySize();
+} else {
+  // By setting useToWeightRatios to negative weights, we give the
+  // zero-weight one less priority, so the non-zero weight one will
+  // be given slots.
+  useToWeightRatio1 = -weight1;
+  useToWeightRatio2 = -weight2;
+}
+  }
+
   int res = 0;
   if (s1Needy && !s2Needy)
 res 

[01/50] [abbrv] hadoop git commit: YARN-4743. FairSharePolicy breaks TimSort assumption. (Zephyr Guo and Yufei Gu via kasha) [Forced Update!]

[sunilg]

Repository: hadoop
Updated Branches:
  refs/heads/YARN-3368 07ae9db73 -> 97143e966 (forced update)


YARN-4743. FairSharePolicy breaks TimSort assumption. (Zephyr Guo and Yufei Gu 
via kasha)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4df8ed63
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4df8ed63
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4df8ed63

Branch: refs/heads/YARN-3368
Commit: 4df8ed63ed93f2542e4b48f521b0cc6624ab59c1
Parents: b2c4f24
Author: Karthik Kambatla 
Authored: Thu Oct 27 17:42:44 2016 -0700
Committer: Karthik Kambatla 
Committed: Thu Oct 27 17:45:48 2016 -0700

--
 .../fair/policies/FairSharePolicy.java  |  31 ++-
 .../scheduler/fair/TestSchedulingPolicy.java| 228 +++
 2 files changed, 254 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4df8ed63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
index 6aa8405..f120f0f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
@@ -63,7 +63,11 @@ public class FairSharePolicy extends SchedulingPolicy {
* 
* Schedulables above their min share are compared by (runningTasks / 
weight).
* If all weights are equal, slots are given to the job with the fewest 
tasks;
-   * otherwise, jobs with more weight get proportionally more slots.
+   * otherwise, jobs with more weight get proportionally more slots. If weight
+   * equals to 0, we can't compare Schedulables by (resource usage/weight).
+   * There are two situations: 1)All weights equal to 0, slots are given
+   * to one with less resource usage. 2)Only one of weight equals to 0, slots
+   * are given to the one with non-zero weight.
*/
   private static class FairShareComparator implements Comparator,
   Serializable {
@@ -74,6 +78,7 @@ public class FairSharePolicy extends SchedulingPolicy {
 public int compare(Schedulable s1, Schedulable s2) {
   double minShareRatio1, minShareRatio2;
   double useToWeightRatio1, useToWeightRatio2;
+  double weight1, weight2;
   Resource minShare1 = Resources.min(RESOURCE_CALCULATOR, null,
   s1.getMinShare(), s1.getDemand());
   Resource minShare2 = Resources.min(RESOURCE_CALCULATOR, null,
@@ -86,10 +91,26 @@ public class FairSharePolicy extends SchedulingPolicy {
   / Resources.max(RESOURCE_CALCULATOR, null, minShare1, 
ONE).getMemorySize();
   minShareRatio2 = (double) s2.getResourceUsage().getMemorySize()
   / Resources.max(RESOURCE_CALCULATOR, null, minShare2, 
ONE).getMemorySize();
-  useToWeightRatio1 = s1.getResourceUsage().getMemorySize() /
-  s1.getWeights().getWeight(ResourceType.MEMORY);
-  useToWeightRatio2 = s2.getResourceUsage().getMemorySize() /
-  s2.getWeights().getWeight(ResourceType.MEMORY);
+
+  weight1 = s1.getWeights().getWeight(ResourceType.MEMORY);
+  weight2 = s2.getWeights().getWeight(ResourceType.MEMORY);
+  if (weight1 > 0.0 && weight2 > 0.0) {
+useToWeightRatio1 = s1.getResourceUsage().getMemorySize() / weight1;
+useToWeightRatio2 = s2.getResourceUsage().getMemorySize() / weight2;
+  } else { // Either weight1 or weight2 equals to 0
+if (weight1 == weight2) {
+  // If they have same weight, just compare usage
+  useToWeightRatio1 = s1.getResourceUsage().getMemorySize();
+  useToWeightRatio2 = s2.getResourceUsage().getMemorySize();
+} else {
+  // By setting useToWeightRatios to negative weights, we give the
+  // zero-weight one less priority, so the non-zero weight one will
+  // be given slots.
+  useToWeightRatio1 = -weight1;
+  useToWeightRatio2 = -weight2;
+}
+  }
+
   int res = 0;
   if (s1Needy && !s2Needy)
 res = 

hadoop git commit: YARN-4743. FairSharePolicy breaks TimSort assumption. (Zephyr Guo and Yufei Gu via kasha)

[kasha]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 334fd9e83 -> 950bfed1d


YARN-4743. FairSharePolicy breaks TimSort assumption. (Zephyr Guo and Yufei Gu 
via kasha)

(cherry picked from commit 4df8ed63ed93f2542e4b48f521b0cc6624ab59c1)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/950bfed1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/950bfed1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/950bfed1

Branch: refs/heads/branch-2
Commit: 950bfed1d31f07b989a3659121c29a0a80dc4a49
Parents: 334fd9e
Author: Karthik Kambatla 
Authored: Thu Oct 27 17:42:44 2016 -0700
Committer: Karthik Kambatla 
Committed: Thu Oct 27 17:48:24 2016 -0700

--
 .../fair/policies/FairSharePolicy.java  |  31 ++-
 .../scheduler/fair/TestSchedulingPolicy.java| 228 +++
 2 files changed, 254 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/950bfed1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
index 6aa8405..f120f0f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
@@ -63,7 +63,11 @@ public class FairSharePolicy extends SchedulingPolicy {
* 
* Schedulables above their min share are compared by (runningTasks / 
weight).
* If all weights are equal, slots are given to the job with the fewest 
tasks;
-   * otherwise, jobs with more weight get proportionally more slots.
+   * otherwise, jobs with more weight get proportionally more slots. If weight
+   * equals to 0, we can't compare Schedulables by (resource usage/weight).
+   * There are two situations: 1)All weights equal to 0, slots are given
+   * to one with less resource usage. 2)Only one of weight equals to 0, slots
+   * are given to the one with non-zero weight.
*/
   private static class FairShareComparator implements Comparator,
   Serializable {
@@ -74,6 +78,7 @@ public class FairSharePolicy extends SchedulingPolicy {
 public int compare(Schedulable s1, Schedulable s2) {
   double minShareRatio1, minShareRatio2;
   double useToWeightRatio1, useToWeightRatio2;
+  double weight1, weight2;
   Resource minShare1 = Resources.min(RESOURCE_CALCULATOR, null,
   s1.getMinShare(), s1.getDemand());
   Resource minShare2 = Resources.min(RESOURCE_CALCULATOR, null,
@@ -86,10 +91,26 @@ public class FairSharePolicy extends SchedulingPolicy {
   / Resources.max(RESOURCE_CALCULATOR, null, minShare1, 
ONE).getMemorySize();
   minShareRatio2 = (double) s2.getResourceUsage().getMemorySize()
   / Resources.max(RESOURCE_CALCULATOR, null, minShare2, 
ONE).getMemorySize();
-  useToWeightRatio1 = s1.getResourceUsage().getMemorySize() /
-  s1.getWeights().getWeight(ResourceType.MEMORY);
-  useToWeightRatio2 = s2.getResourceUsage().getMemorySize() /
-  s2.getWeights().getWeight(ResourceType.MEMORY);
+
+  weight1 = s1.getWeights().getWeight(ResourceType.MEMORY);
+  weight2 = s2.getWeights().getWeight(ResourceType.MEMORY);
+  if (weight1 > 0.0 && weight2 > 0.0) {
+useToWeightRatio1 = s1.getResourceUsage().getMemorySize() / weight1;
+useToWeightRatio2 = s2.getResourceUsage().getMemorySize() / weight2;
+  } else { // Either weight1 or weight2 equals to 0
+if (weight1 == weight2) {
+  // If they have same weight, just compare usage
+  useToWeightRatio1 = s1.getResourceUsage().getMemorySize();
+  useToWeightRatio2 = s2.getResourceUsage().getMemorySize();
+} else {
+  // By setting useToWeightRatios to negative weights, we give the
+  // zero-weight one less priority, so the non-zero weight one will
+  // be given slots.
+  useToWeightRatio1 = -weight1;
+  useToWeightRatio2 = -weight2;
+}
+  }
+
   int res 

hadoop git commit: YARN-4743. FairSharePolicy breaks TimSort assumption. (Zephyr Guo and Yufei Gu via kasha)

[kasha]

Repository: hadoop
Updated Branches:
  refs/heads/trunk b2c4f24c3 -> 4df8ed63e


YARN-4743. FairSharePolicy breaks TimSort assumption. (Zephyr Guo and Yufei Gu 
via kasha)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4df8ed63
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4df8ed63
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4df8ed63

Branch: refs/heads/trunk
Commit: 4df8ed63ed93f2542e4b48f521b0cc6624ab59c1
Parents: b2c4f24
Author: Karthik Kambatla 
Authored: Thu Oct 27 17:42:44 2016 -0700
Committer: Karthik Kambatla 
Committed: Thu Oct 27 17:45:48 2016 -0700

--
 .../fair/policies/FairSharePolicy.java  |  31 ++-
 .../scheduler/fair/TestSchedulingPolicy.java| 228 +++
 2 files changed, 254 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4df8ed63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
index 6aa8405..f120f0f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
@@ -63,7 +63,11 @@ public class FairSharePolicy extends SchedulingPolicy {
* 
* Schedulables above their min share are compared by (runningTasks / 
weight).
* If all weights are equal, slots are given to the job with the fewest 
tasks;
-   * otherwise, jobs with more weight get proportionally more slots.
+   * otherwise, jobs with more weight get proportionally more slots. If weight
+   * equals to 0, we can't compare Schedulables by (resource usage/weight).
+   * There are two situations: 1)All weights equal to 0, slots are given
+   * to one with less resource usage. 2)Only one of weight equals to 0, slots
+   * are given to the one with non-zero weight.
*/
   private static class FairShareComparator implements Comparator,
   Serializable {
@@ -74,6 +78,7 @@ public class FairSharePolicy extends SchedulingPolicy {
 public int compare(Schedulable s1, Schedulable s2) {
   double minShareRatio1, minShareRatio2;
   double useToWeightRatio1, useToWeightRatio2;
+  double weight1, weight2;
   Resource minShare1 = Resources.min(RESOURCE_CALCULATOR, null,
   s1.getMinShare(), s1.getDemand());
   Resource minShare2 = Resources.min(RESOURCE_CALCULATOR, null,
@@ -86,10 +91,26 @@ public class FairSharePolicy extends SchedulingPolicy {
   / Resources.max(RESOURCE_CALCULATOR, null, minShare1, 
ONE).getMemorySize();
   minShareRatio2 = (double) s2.getResourceUsage().getMemorySize()
   / Resources.max(RESOURCE_CALCULATOR, null, minShare2, 
ONE).getMemorySize();
-  useToWeightRatio1 = s1.getResourceUsage().getMemorySize() /
-  s1.getWeights().getWeight(ResourceType.MEMORY);
-  useToWeightRatio2 = s2.getResourceUsage().getMemorySize() /
-  s2.getWeights().getWeight(ResourceType.MEMORY);
+
+  weight1 = s1.getWeights().getWeight(ResourceType.MEMORY);
+  weight2 = s2.getWeights().getWeight(ResourceType.MEMORY);
+  if (weight1 > 0.0 && weight2 > 0.0) {
+useToWeightRatio1 = s1.getResourceUsage().getMemorySize() / weight1;
+useToWeightRatio2 = s2.getResourceUsage().getMemorySize() / weight2;
+  } else { // Either weight1 or weight2 equals to 0
+if (weight1 == weight2) {
+  // If they have same weight, just compare usage
+  useToWeightRatio1 = s1.getResourceUsage().getMemorySize();
+  useToWeightRatio2 = s2.getResourceUsage().getMemorySize();
+} else {
+  // By setting useToWeightRatios to negative weights, we give the
+  // zero-weight one less priority, so the non-zero weight one will
+  // be given slots.
+  useToWeightRatio1 = -weight1;
+  useToWeightRatio2 = -weight2;
+}
+  }
+
   int res = 0;
   if (s1Needy && !s2Needy)
 res = -1;