[09/18] hadoop git commit: YARN-6322: Disable queue refresh when configuration mutation is enabled. Contributed by Jonathan Hung

2017-10-09 Thread xgong 
YARN-6322: Disable queue refresh when configuration mutation is enabled. 
Contributed by Jonathan Hung


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0b546420
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0b546420
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0b546420

Branch: refs/heads/branch-3.0
Commit: 0b546420da3c7ae708d361ddf52a3cf8d24494b9
Parents: 793476e
Author: Xuan 
Authored: Tue Aug 1 08:48:04 2017 -0700
Committer: Jonathan Hung 
Committed: Mon Oct 9 14:54:55 2017 -0700

--
 .../server/resourcemanager/AdminService.java| 12 +-
 .../scheduler/MutableConfScheduler.java | 12 --
 .../scheduler/MutableConfigurationProvider.java |  4 +++-
 .../scheduler/capacity/CapacityScheduler.java   |  9 ++--
 .../conf/MutableCSConfigurationProvider.java| 11 +-
 .../resourcemanager/TestRMAdminService.java | 23 
 .../TestMutableCSConfigurationProvider.java | 14 +++-
 7 files changed, 67 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b546420/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
index 3457ae3..fd9e849 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
@@ -29,6 +29,7 @@ import java.util.Set;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.ha.HAServiceProtocol;
@@ -92,6 +93,8 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSyst
 import 
org.apache.hadoop.yarn.server.resourcemanager.resource.DynamicResourceConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import 
org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import 
org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -384,6 +387,12 @@ public class AdminService extends CompositeService 
implements
 RefreshQueuesResponse response =
 recordFactory.newRecordInstance(RefreshQueuesResponse.class);
 try {
+  ResourceScheduler scheduler = rm.getRMContext().getScheduler();
+  if (scheduler instanceof MutableConfScheduler
+  && ((MutableConfScheduler) scheduler).isConfigurationMutable()) {
+throw new IOException("Scheduler configuration is mutable. " +
+operation + " is not allowed in this scenario.");
+  }
   refreshQueues();
   RMAuditLogger.logSuccess(user.getShortUserName(), operation,
   "AdminService");
@@ -393,7 +402,8 @@ public class AdminService extends CompositeService 
implements
 }
   }
 
-  private void refreshQueues() throws IOException, YarnException {
+  @Private
+  public void refreshQueues() throws IOException, YarnException {
 rm.getRMContext().getScheduler().reinitialize(getConfig(),
 this.rm.getRMContext());
 // refresh the reservation system

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b546420/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java

[09/18] hadoop git commit: YARN-6322: Disable queue refresh when configuration mutation is enabled. Contributed by Jonathan Hung

2017-10-09 Thread xgong 
YARN-6322: Disable queue refresh when configuration mutation is enabled. 
Contributed by Jonathan Hung


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3a31c388
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3a31c388
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3a31c388

Branch: refs/heads/branch-2
Commit: 3a31c38885965c9f906464904981d15ac8db9bb1
Parents: cd58f5d
Author: Xuan 
Authored: Tue Aug 1 08:48:04 2017 -0700
Committer: Jonathan Hung 
Committed: Mon Oct 9 11:12:07 2017 -0700

--
 .../server/resourcemanager/AdminService.java| 12 +-
 .../scheduler/MutableConfScheduler.java | 12 --
 .../scheduler/MutableConfigurationProvider.java |  4 +++-
 .../scheduler/capacity/CapacityScheduler.java   |  9 ++--
 .../conf/MutableCSConfigurationProvider.java| 11 +-
 .../resourcemanager/TestRMAdminService.java | 23 
 .../TestMutableCSConfigurationProvider.java | 14 +++-
 7 files changed, 67 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a31c388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
index 3457ae3..fd9e849 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
@@ -29,6 +29,7 @@ import java.util.Set;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.ha.HAServiceProtocol;
@@ -92,6 +93,8 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSyst
 import 
org.apache.hadoop.yarn.server.resourcemanager.resource.DynamicResourceConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import 
org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import 
org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -384,6 +387,12 @@ public class AdminService extends CompositeService 
implements
 RefreshQueuesResponse response =
 recordFactory.newRecordInstance(RefreshQueuesResponse.class);
 try {
+  ResourceScheduler scheduler = rm.getRMContext().getScheduler();
+  if (scheduler instanceof MutableConfScheduler
+  && ((MutableConfScheduler) scheduler).isConfigurationMutable()) {
+throw new IOException("Scheduler configuration is mutable. " +
+operation + " is not allowed in this scenario.");
+  }
   refreshQueues();
   RMAuditLogger.logSuccess(user.getShortUserName(), operation,
   "AdminService");
@@ -393,7 +402,8 @@ public class AdminService extends CompositeService 
implements
 }
   }
 
-  private void refreshQueues() throws IOException, YarnException {
+  @Private
+  public void refreshQueues() throws IOException, YarnException {
 rm.getRMContext().getScheduler().reinitialize(getConfig(),
 this.rm.getRMContext());
 // refresh the reservation system

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a31c388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java