YARN-6575. Support global configuration mutation in MutableConfProvider. (Jonathan Hung via Xuan Gong)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e566fd8b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e566fd8b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e566fd8b Branch: refs/heads/trunk Commit: e566fd8b58a6570593b21597aa5294d7b37cb388 Parents: a4e6253 Author: Xuan <xg...@apache.org> Authored: Mon Jun 5 16:30:38 2017 -0700 Committer: Jonathan Hung <jh...@linkedin.com> Committed: Mon Oct 9 11:11:23 2017 -0700 ---------------------------------------------------------------------- .../ConfigurationMutationACLPolicy.java | 4 +- .../DefaultConfigurationMutationACLPolicy.java | 4 +- .../scheduler/MutableConfScheduler.java | 4 +- .../scheduler/MutableConfigurationProvider.java | 4 +- .../scheduler/capacity/CapacityScheduler.java | 4 +- .../conf/MutableCSConfigurationProvider.java | 10 +- ...ueueAdminConfigurationMutationACLPolicy.java | 22 +++- .../resourcemanager/webapp/RMWebServices.java | 4 +- .../webapp/dao/QueueConfigsUpdateInfo.java | 60 ----------- .../webapp/dao/SchedConfUpdateInfo.java | 69 +++++++++++++ .../TestConfigurationMutationACLPolicies.java | 28 ++++- .../TestMutableCSConfigurationProvider.java | 10 +- .../TestRMWebServicesConfigurationMutation.java | 101 +++++++++++++------ 13 files changed, 205 insertions(+), 119 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e566fd8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java index 724487b..3a388fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java @@ -21,7 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo; /** * Interface for determining whether configuration mutations are allowed. @@ -41,7 +41,7 @@ public interface ConfigurationMutationACLPolicy { * @param confUpdate configurations to be updated * @return whether provided mutation is allowed or not */ - boolean isMutationAllowed(UserGroupInformation user, QueueConfigsUpdateInfo + boolean isMutationAllowed(UserGroupInformation user, SchedConfUpdateInfo confUpdate); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e566fd8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java index 680c3b8..6648668 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java @@ -22,7 +22,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.security.YarnAuthorizationProvider; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo; /** * Default configuration mutation ACL policy. Checks if user is YARN admin. @@ -39,7 +39,7 @@ public class DefaultConfigurationMutationACLPolicy implements @Override public boolean isMutationAllowed(UserGroupInformation user, - QueueConfigsUpdateInfo confUpdate) { + SchedConfUpdateInfo confUpdate) { return authorizer.isAdmin(user); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e566fd8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java index 93a935e..027d944 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java @@ -19,7 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo; import java.io.IOException; @@ -36,7 +36,7 @@ public interface MutableConfScheduler extends ResourceScheduler { * @throws IOException if update is invalid */ void updateConfiguration(UserGroupInformation user, - QueueConfigsUpdateInfo confUpdate) throws IOException; + SchedConfUpdateInfo confUpdate) throws IOException; /** * Get the scheduler configuration. http://git-wip-us.apache.org/repos/asf/hadoop/blob/e566fd8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java index f04c128..6b8306c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java @@ -19,7 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo; import java.io.IOException; @@ -34,7 +34,7 @@ public interface MutableConfigurationProvider { * @param confUpdate Key-value pairs for configurations to be updated. * @throws IOException if scheduler could not be reinitialized */ - void mutateConfiguration(UserGroupInformation user, QueueConfigsUpdateInfo + void mutateConfiguration(UserGroupInformation user, SchedConfUpdateInfo confUpdate) throws IOException; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e566fd8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 4f89f7e..4d367e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -137,7 +137,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.Placeme import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimplePlacementSet; import org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo; import org.apache.hadoop.yarn.server.utils.Lock; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; @@ -2617,7 +2617,7 @@ public class CapacityScheduler extends @Override public void updateConfiguration(UserGroupInformation user, - QueueConfigsUpdateInfo confUpdate) throws IOException { + SchedConfUpdateInfo confUpdate) throws IOException { if (csConfProvider instanceof MutableConfigurationProvider) { ((MutableConfigurationProvider) csConfProvider).mutateConfiguration( user, confUpdate); http://git-wip-us.apache.org/repos/asf/hadoop/blob/e566fd8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java index 8b879b0..eb97260 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java @@ -32,7 +32,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo; import java.io.IOException; import java.util.ArrayList; @@ -98,7 +98,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, @Override public void mutateConfiguration(UserGroupInformation user, - QueueConfigsUpdateInfo confUpdate) throws IOException { + SchedConfUpdateInfo confUpdate) throws IOException { if (!aclMutationPolicy.isMutationAllowed(user, confUpdate)) { throw new AccessControlException("User is not admin of all modified" + " queues."); @@ -126,7 +126,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, private Map<String, String> constructKeyValueConfUpdate( - QueueConfigsUpdateInfo mutationInfo) throws IOException { + SchedConfUpdateInfo mutationInfo) throws IOException { CapacityScheduler cs = (CapacityScheduler) rmContext.getScheduler(); CapacitySchedulerConfiguration proposedConf = new CapacitySchedulerConfiguration(cs.getConfiguration(), false); @@ -140,6 +140,10 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, for (QueueConfigInfo updateQueueInfo : mutationInfo.getUpdateQueueInfo()) { updateQueue(updateQueueInfo, proposedConf, confUpdate); } + for (Map.Entry<String, String> global : mutationInfo.getGlobalParams() + .entrySet()) { + confUpdate.put(global.getKey(), global.getValue()); + } return confUpdate; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e566fd8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java index 1f94c1c..0a82d50 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java @@ -22,15 +22,17 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueInfo; +import org.apache.hadoop.yarn.security.YarnAuthorizationProvider; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo; import java.io.IOException; import java.util.HashSet; +import java.util.Map; import java.util.Set; /** @@ -40,16 +42,29 @@ import java.util.Set; public class QueueAdminConfigurationMutationACLPolicy implements ConfigurationMutationACLPolicy { + private Configuration conf; private RMContext rmContext; + private YarnAuthorizationProvider authorizer; @Override - public void init(Configuration conf, RMContext context) { + public void init(Configuration config, RMContext context) { + this.conf = config; this.rmContext = context; + this.authorizer = YarnAuthorizationProvider.getInstance(conf); } @Override public boolean isMutationAllowed(UserGroupInformation user, - QueueConfigsUpdateInfo confUpdate) { + SchedConfUpdateInfo confUpdate) { + // If there are global config changes, check if user is admin. + Map<String, String> globalParams = confUpdate.getGlobalParams(); + if (globalParams != null && globalParams.size() != 0) { + if (!authorizer.isAdmin(user)) { + return false; + } + } + + // Check if user is admin of all modified queues. Set<String> queues = new HashSet<>(); for (QueueConfigInfo addQueueInfo : confUpdate.getAddQueueInfo()) { queues.add(addQueueInfo.getQueue()); @@ -71,7 +86,6 @@ public class QueueAdminConfigurationMutationACLPolicy implements // Queue is not found, do nothing. } String parentPath = queuePath; - // TODO: handle global config change. while (queueInfo == null) { // We are adding a queue (whose parent we are possibly also adding). // Check ACL of lowest parent queue which already exists. http://git-wip-us.apache.org/repos/asf/hadoop/blob/e566fd8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index d95465b..b97e212 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -2459,11 +2459,11 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol { } @PUT - @Path("/queues") + @Path("/sched-conf") @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) - public Response updateSchedulerConfiguration(QueueConfigsUpdateInfo + public Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo, @Context HttpServletRequest hsr) throws AuthorizationException, InterruptedException { init(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/e566fd8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java deleted file mode 100644 index 644ec90..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; - -import java.util.ArrayList; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlRootElement; - -/** - * Information for making scheduler configuration changes (supports adding, - * removing, or updating a queue). - */ -@XmlRootElement(name = "schedConf") -@XmlAccessorType(XmlAccessType.FIELD) -public class QueueConfigsUpdateInfo { - - @XmlElement(name = "add") - private ArrayList<QueueConfigInfo> addQueueInfo = new ArrayList<>(); - - @XmlElement(name = "remove") - private ArrayList<String> removeQueueInfo = new ArrayList<>(); - - @XmlElement(name = "update") - private ArrayList<QueueConfigInfo> updateQueueInfo = new ArrayList<>(); - - public QueueConfigsUpdateInfo() { - // JAXB needs this - } - - public ArrayList<QueueConfigInfo> getAddQueueInfo() { - return addQueueInfo; - } - - public ArrayList<String> getRemoveQueueInfo() { - return removeQueueInfo; - } - - public ArrayList<QueueConfigInfo> getUpdateQueueInfo() { - return updateQueueInfo; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e566fd8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java new file mode 100644 index 0000000..b7c585e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.ArrayList; +import java.util.HashMap; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlElementWrapper; +import javax.xml.bind.annotation.XmlRootElement; + +/** + * Information for making scheduler configuration changes (supports adding, + * removing, or updating a queue, as well as global scheduler conf changes). + */ +@XmlRootElement(name = "schedConf") +@XmlAccessorType(XmlAccessType.FIELD) +public class SchedConfUpdateInfo { + + @XmlElement(name = "add-queue") + private ArrayList<QueueConfigInfo> addQueueInfo = new ArrayList<>(); + + @XmlElement(name = "remove-queue") + private ArrayList<String> removeQueueInfo = new ArrayList<>(); + + @XmlElement(name = "update-queue") + private ArrayList<QueueConfigInfo> updateQueueInfo = new ArrayList<>(); + + private HashMap<String, String> global = new HashMap<>(); + + public SchedConfUpdateInfo() { + // JAXB needs this + } + + public ArrayList<QueueConfigInfo> getAddQueueInfo() { + return addQueueInfo; + } + + public ArrayList<String> getRemoveQueueInfo() { + return removeQueueInfo; + } + + public ArrayList<QueueConfigInfo> getUpdateQueueInfo() { + return updateQueueInfo; + } + + @XmlElementWrapper(name = "global-updates") + public HashMap<String, String> getGlobalParams() { + return global; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e566fd8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java index 4016dcf..0f5a3d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java @@ -26,7 +26,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.QueueAdminConfigurationMutationACLPolicy; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo; import org.junit.Before; import org.junit.Test; @@ -77,6 +77,7 @@ public class TestConfigurationMutationACLPolicies { .thenReturn(false); when(scheduler.getQueue(eq(queueName))).thenReturn(queue); } + @Test public void testDefaultPolicy() { Configuration conf = new Configuration(); @@ -98,7 +99,7 @@ public class TestConfigurationMutationACLPolicies { ConfigurationMutationACLPolicy.class); policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf); policy.init(conf, rmContext); - QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); QueueConfigInfo configInfo = new QueueConfigInfo("root.a", EMPTY_MAP); updateInfo.getUpdateQueueInfo().add(configInfo); assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo)); @@ -114,7 +115,7 @@ public class TestConfigurationMutationACLPolicies { policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf); policy.init(conf, rmContext); // Add root.b.b1. Should check ACL of root.b queue. - QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); QueueConfigInfo configInfo = new QueueConfigInfo("root.b.b2", EMPTY_MAP); updateInfo.getAddQueueInfo().add(configInfo); assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo)); @@ -130,7 +131,7 @@ public class TestConfigurationMutationACLPolicies { policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf); policy.init(conf, rmContext); // Add root.b.b1.b11. Should check ACL of root.b queue. - QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); QueueConfigInfo configInfo = new QueueConfigInfo("root.b.b2.b21", EMPTY_MAP); updateInfo.getAddQueueInfo().add(configInfo); assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo)); @@ -146,9 +147,26 @@ public class TestConfigurationMutationACLPolicies { policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf); policy.init(conf, rmContext); // Remove root.b.b1. - QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); updateInfo.getRemoveQueueInfo().add("root.b.b1"); assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo)); assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo)); } + + @Test + public void testQueueAdminPolicyGlobal() { + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.YARN_ADMIN_ACL, GOOD_USER.getShortUserName()); + conf.setClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS, + QueueAdminConfigurationMutationACLPolicy.class, + ConfigurationMutationACLPolicy.class); + policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf); + policy.init(conf, rmContext); + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); + assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo)); + assertTrue(policy.isMutationAllowed(BAD_USER, updateInfo)); + updateInfo.getGlobalParams().put("globalKey", "globalValue"); + assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo)); + assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo)); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e566fd8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java index 13229b1..3216781 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java @@ -24,7 +24,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo; import org.junit.Before; import org.junit.Test; @@ -47,8 +47,8 @@ public class TestMutableCSConfigurationProvider { private MutableCSConfigurationProvider confProvider; private RMContext rmContext; - private QueueConfigsUpdateInfo goodUpdate; - private QueueConfigsUpdateInfo badUpdate; + private SchedConfUpdateInfo goodUpdate; + private SchedConfUpdateInfo badUpdate; private CapacityScheduler cs; private static final UserGroupInformation TEST_USER = UserGroupInformation @@ -62,14 +62,14 @@ public class TestMutableCSConfigurationProvider { when(cs.getConfiguration()).thenReturn( new CapacitySchedulerConfiguration()); confProvider = new MutableCSConfigurationProvider(rmContext); - goodUpdate = new QueueConfigsUpdateInfo(); + goodUpdate = new SchedConfUpdateInfo(); Map<String, String> goodUpdateMap = new HashMap<>(); goodUpdateMap.put("goodKey", "goodVal"); QueueConfigInfo goodUpdateInfo = new QueueConfigInfo("root.a", goodUpdateMap); goodUpdate.getUpdateQueueInfo().add(goodUpdateInfo); - badUpdate = new QueueConfigsUpdateInfo(); + badUpdate = new SchedConfUpdateInfo(); Map<String, String> badUpdateMap = new HashMap<>(); badUpdateMap.put("badKey", "badVal"); QueueConfigInfo badUpdateInfo = new http://git-wip-us.apache.org/repos/asf/hadoop/blob/e566fd8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java index d149055..5fbe36f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java @@ -36,7 +36,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.GuiceServletConfig; import org.apache.hadoop.yarn.webapp.JerseyTestBase; @@ -162,7 +162,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { ClientResponse response; // Add parent queue root.d with two children d1 and d2. - QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); Map<String, String> d1Capacity = new HashMap<>(); d1Capacity.put(CapacitySchedulerConfiguration.CAPACITY, "25"); d1Capacity.put(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY, "25"); @@ -181,9 +181,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { updateInfo.getAddQueueInfo().add(d); response = r.path("ws").path("v1").path("cluster") - .path("queues").queryParam("user.name", userName) + .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class), + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); @@ -205,7 +205,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { ClientResponse response; // Add root.d with capacity 25, reducing root.b capacity from 75 to 50. - QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); Map<String, String> dCapacity = new HashMap<>(); dCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "25"); Map<String, String> bCapacity = new HashMap<>(); @@ -216,9 +216,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { updateInfo.getUpdateQueueInfo().add(b); response = r.path("ws").path("v1").path("cluster") - .path("queues").queryParam("user.name", userName) + .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class), + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); @@ -238,13 +238,13 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { stopQueue("root.a.a2"); // Remove root.a.a2 - QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); updateInfo.getRemoveQueueInfo().add("root.a.a2"); response = r.path("ws").path("v1").path("cluster") - .path("queues").queryParam("user.name", userName) + .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class), + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); @@ -263,13 +263,13 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { stopQueue("root.c", "root.c.c1"); // Remove root.c (parent queue) - QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); updateInfo.getRemoveQueueInfo().add("root.c"); response = r.path("ws").path("v1").path("cluster") - .path("queues").queryParam("user.name", userName) + .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class), + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); @@ -288,7 +288,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { stopQueue("root.a", "root.a.a1", "root.a.a2"); // Remove root.a (parent queue) with capacity 25 - QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); updateInfo.getRemoveQueueInfo().add("root.a"); // Set root.b capacity to 100 @@ -298,9 +298,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { updateInfo.getUpdateQueueInfo().add(b); response = r.path("ws").path("v1").path("cluster") - .path("queues").queryParam("user.name", userName) + .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class), + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); @@ -320,7 +320,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { stopQueue("root.b", "root.c", "root.c.c1"); // Remove root.b and root.c - QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); updateInfo.getRemoveQueueInfo().add("root.b"); updateInfo.getRemoveQueueInfo().add("root.c"); Map<String, String> aCapacity = new HashMap<>(); @@ -330,9 +330,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { updateInfo.getUpdateQueueInfo().add(configInfo); response = r.path("ws").path("v1").path("cluster") - .path("queues").queryParam("user.name", userName) + .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class), + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); @@ -348,7 +348,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { ClientResponse response; // Set state of queues to STOPPED. - QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); Map<String, String> stoppedParam = new HashMap<>(); stoppedParam.put(CapacitySchedulerConfiguration.STATE, QueueState.STOPPED.toString()); @@ -358,9 +358,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { } response = r.path("ws").path("v1").path("cluster") - .path("queues").queryParam("user.name", userName) + .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class), + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); @@ -378,7 +378,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { ClientResponse response; // Update config value. - QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); Map<String, String> updateParam = new HashMap<>(); updateParam.put(CapacitySchedulerConfiguration.MAXIMUM_AM_RESOURCE_SUFFIX, "0.2"); @@ -393,9 +393,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { 0.001f); response = r.path("ws").path("v1").path("cluster") - .path("queues").queryParam("user.name", userName) + .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class), + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); @@ -411,9 +411,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { updateInfo.getUpdateQueueInfo().add(aUpdateInfo); response = r.path("ws").path("v1").path("cluster") - .path("queues").queryParam("user.name", userName) + .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class), + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); @@ -431,7 +431,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { ClientResponse response; // Update root.a and root.b capacity to 50. - QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); Map<String, String> updateParam = new HashMap<>(); updateParam.put(CapacitySchedulerConfiguration.CAPACITY, "50"); QueueConfigInfo aUpdateInfo = new QueueConfigInfo("root.a", updateParam); @@ -441,9 +441,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { response = r.path("ws").path("v1").path("cluster") - .path("queues").queryParam("user.name", userName) + .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class), + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); @@ -453,6 +453,47 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity("root.b"), 0.01f); } + @Test + public void testGlobalConfChange() throws Exception { + WebResource r = resource(); + + ClientResponse response; + + // Set maximum-applications to 30000. + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); + updateInfo.getGlobalParams().put(CapacitySchedulerConfiguration.PREFIX + + "maximum-applications", "30000"); + + response = + r.path("ws").path("v1").path("cluster") + .path("sched-conf").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), + MediaType.APPLICATION_JSON) + .put(ClientResponse.class); + assertEquals(Status.OK.getStatusCode(), response.getStatus()); + CapacitySchedulerConfiguration newCSConf = + ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); + assertEquals(30000, newCSConf.getMaximumSystemApplications()); + + updateInfo.getGlobalParams().put(CapacitySchedulerConfiguration.PREFIX + + "maximum-applications", null); + // Unset maximum-applications. Should be set to default. + response = + r.path("ws").path("v1").path("cluster") + .path("sched-conf").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), + MediaType.APPLICATION_JSON) + .put(ClientResponse.class); + assertEquals(Status.OK.getStatusCode(), response.getStatus()); + newCSConf = + ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); + assertEquals(CapacitySchedulerConfiguration + .DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS, + newCSConf.getMaximumSystemApplications()); + } + @Override @After public void tearDown() throws Exception { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org