[09/50] hadoop git commit: YARN-8436. FSParentQueue: Comparison method violates its general contract. (Wilfred Spiegelenburg via Haibo Chen)

2018-07-24 Thread inigoiri 
YARN-8436. FSParentQueue: Comparison method violates its general contract. 
(Wilfred Spiegelenburg via Haibo Chen)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/25648847
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/25648847
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/25648847

Branch: refs/heads/HADOOP-15461
Commit: 2564884757fbf4df7718f814cc448f7f23dad875
Parents: 45d9568
Author: Haibo Chen 
Authored: Thu Jul 19 13:21:57 2018 -0700
Committer: Haibo Chen 
Committed: Thu Jul 19 13:22:31 2018 -0700

--
 .../scheduler/fair/FSParentQueue.java   | 30 +++-
 .../scheduler/fair/FakeSchedulable.java |  4 +
 .../TestDominantResourceFairnessPolicy.java | 77 
 3 files changed, 93 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/25648847/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
index 26c5630..d5df549 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
@@ -20,8 +20,8 @@ package 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
 
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
+import java.util.TreeSet;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -188,25 +188,19 @@ public class FSParentQueue extends FSQueue {
   return assigned;
 }
 
-// Hold the write lock when sorting childQueues
-writeLock.lock();
-try {
-  Collections.sort(childQueues, policy.getComparator());
-} finally {
-  writeLock.unlock();
-}
-
-/*
- * We are releasing the lock between the sort and iteration of the
- * "sorted" list. There could be changes to the list here:
- * 1. Add a child queue to the end of the list, this doesn't affect
- * container assignment.
- * 2. Remove a child queue, this is probably good to take care of so we
- * don't assign to a queue that is going to be removed shortly.
- */
+// Sort the queues while holding a read lock on this parent only.
+// The individual entries are not locked and can change which means that
+// the collection of childQueues can not be sorted by calling Sort().
+// Locking each childqueue to prevent changes would have a large
+// performance impact.
+// We do not have to handle the queue removal case as a queue must be
+// empty before removal. Assigning an application to a queue and removal of
+// that queue both need the scheduler lock.
+TreeSet sortedChildQueues = new TreeSet<>(policy.getComparator());
 readLock.lock();
 try {
-  for (FSQueue child : childQueues) {
+  sortedChildQueues.addAll(childQueues);
+  for (FSQueue child : sortedChildQueues) {
 assigned = child.assignContainer(node);
 if (!Resources.equals(assigned, Resources.none())) {
   break;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/25648847/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
index 03332b2..01eec73 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
+++

[45/50] hadoop git commit: YARN-8436. FSParentQueue: Comparison method violates its general contract. (Wilfred Spiegelenburg via Haibo Chen)

2018-07-20 Thread zhangduo 
YARN-8436. FSParentQueue: Comparison method violates its general contract. 
(Wilfred Spiegelenburg via Haibo Chen)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/25648847
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/25648847
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/25648847

Branch: refs/heads/HDFS-13572
Commit: 2564884757fbf4df7718f814cc448f7f23dad875
Parents: 45d9568
Author: Haibo Chen 
Authored: Thu Jul 19 13:21:57 2018 -0700
Committer: Haibo Chen 
Committed: Thu Jul 19 13:22:31 2018 -0700

--
 .../scheduler/fair/FSParentQueue.java   | 30 +++-
 .../scheduler/fair/FakeSchedulable.java |  4 +
 .../TestDominantResourceFairnessPolicy.java | 77 
 3 files changed, 93 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/25648847/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
index 26c5630..d5df549 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
@@ -20,8 +20,8 @@ package 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
 
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
+import java.util.TreeSet;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -188,25 +188,19 @@ public class FSParentQueue extends FSQueue {
   return assigned;
 }
 
-// Hold the write lock when sorting childQueues
-writeLock.lock();
-try {
-  Collections.sort(childQueues, policy.getComparator());
-} finally {
-  writeLock.unlock();
-}
-
-/*
- * We are releasing the lock between the sort and iteration of the
- * "sorted" list. There could be changes to the list here:
- * 1. Add a child queue to the end of the list, this doesn't affect
- * container assignment.
- * 2. Remove a child queue, this is probably good to take care of so we
- * don't assign to a queue that is going to be removed shortly.
- */
+// Sort the queues while holding a read lock on this parent only.
+// The individual entries are not locked and can change which means that
+// the collection of childQueues can not be sorted by calling Sort().
+// Locking each childqueue to prevent changes would have a large
+// performance impact.
+// We do not have to handle the queue removal case as a queue must be
+// empty before removal. Assigning an application to a queue and removal of
+// that queue both need the scheduler lock.
+TreeSet sortedChildQueues = new TreeSet<>(policy.getComparator());
 readLock.lock();
 try {
-  for (FSQueue child : childQueues) {
+  sortedChildQueues.addAll(childQueues);
+  for (FSQueue child : sortedChildQueues) {
 assigned = child.assignContainer(node);
 if (!Resources.equals(assigned, Resources.none())) {
   break;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/25648847/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
index 03332b2..01eec73 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
+++

hadoop git commit: YARN-8436. FSParentQueue: Comparison method violates its general contract. (Wilfred Spiegelenburg via Haibo Chen)

2018-07-19 Thread haibochen 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 45d9568aa -> 256488475


YARN-8436. FSParentQueue: Comparison method violates its general contract. 
(Wilfred Spiegelenburg via Haibo Chen)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/25648847
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/25648847
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/25648847

Branch: refs/heads/trunk
Commit: 2564884757fbf4df7718f814cc448f7f23dad875
Parents: 45d9568
Author: Haibo Chen 
Authored: Thu Jul 19 13:21:57 2018 -0700
Committer: Haibo Chen 
Committed: Thu Jul 19 13:22:31 2018 -0700

--
 .../scheduler/fair/FSParentQueue.java   | 30 +++-
 .../scheduler/fair/FakeSchedulable.java |  4 +
 .../TestDominantResourceFairnessPolicy.java | 77 
 3 files changed, 93 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/25648847/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
index 26c5630..d5df549 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
@@ -20,8 +20,8 @@ package 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
 
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
+import java.util.TreeSet;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -188,25 +188,19 @@ public class FSParentQueue extends FSQueue {
   return assigned;
 }
 
-// Hold the write lock when sorting childQueues
-writeLock.lock();
-try {
-  Collections.sort(childQueues, policy.getComparator());
-} finally {
-  writeLock.unlock();
-}
-
-/*
- * We are releasing the lock between the sort and iteration of the
- * "sorted" list. There could be changes to the list here:
- * 1. Add a child queue to the end of the list, this doesn't affect
- * container assignment.
- * 2. Remove a child queue, this is probably good to take care of so we
- * don't assign to a queue that is going to be removed shortly.
- */
+// Sort the queues while holding a read lock on this parent only.
+// The individual entries are not locked and can change which means that
+// the collection of childQueues can not be sorted by calling Sort().
+// Locking each childqueue to prevent changes would have a large
+// performance impact.
+// We do not have to handle the queue removal case as a queue must be
+// empty before removal. Assigning an application to a queue and removal of
+// that queue both need the scheduler lock.
+TreeSet sortedChildQueues = new TreeSet<>(policy.getComparator());
 readLock.lock();
 try {
-  for (FSQueue child : childQueues) {
+  sortedChildQueues.addAll(childQueues);
+  for (FSQueue child : sortedChildQueues) {
 assigned = child.assignContainer(node);
 if (!Resources.equals(assigned, Resources.none())) {
   break;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/25648847/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
index 03332b2..01eec73 100644
---