YARN-8559. Expose mutable-conf scheduler's configuration in RM /scheduler-conf endpoint. Contributed by Weiwei Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d352f167 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d352f167 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d352f167 Branch: refs/heads/HDFS-12090 Commit: d352f167ebb865a6486afbbdac8e2a5e97a7bbad Parents: cd04e95 Author: Weiwei Yang <[email protected]> Authored: Thu Aug 9 23:46:53 2018 +0800 Committer: Weiwei Yang <[email protected]> Committed: Thu Aug 9 23:46:53 2018 +0800 ---------------------------------------------------------------------- .../scheduler/MutableConfigurationProvider.java | 7 ++ .../conf/MutableCSConfigurationProvider.java | 5 ++ .../resourcemanager/webapp/RMWebServices.java | 34 +++++++++ .../resourcemanager/webapp/dao/ConfInfo.java | 72 ++++++++++++++++++++ .../TestRMWebServicesConfigurationMutation.java | 40 +++++++++++ .../src/site/markdown/ResourceManagerRest.md | 40 +++++++++++ 6 files changed, 198 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d352f167/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java index 2b9b25a..6e56f3d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; @@ -59,6 +60,12 @@ public interface MutableConfigurationProvider { void confirmPendingMutation(boolean isValid) throws Exception; /** + * Returns scheduler configuration cached in this provider. + * @return scheduler configuration. + */ + Configuration getConfiguration(); + + /** * Closes the configuration provider, releasing any required resources. * @throws IOException on failure to close */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/d352f167/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java index 9c3bf9d..51de437 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java @@ -126,6 +126,11 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, } @Override + public Configuration getConfiguration() { + return new Configuration(schedConf); + } + + @Override public ConfigurationMutationACLPolicy getAclMutationPolicy() { return aclMutationPolicy; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d352f167/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 7752fa2..0117376 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -188,6 +188,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ConfInfo; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.server.webapp.WebServices; @@ -2373,6 +2374,39 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol { } @GET + @Path(RMWSConsts.SCHEDULER_CONF) + @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + public Response getSchedulerConfiguration(@Context HttpServletRequest hsr) + throws AuthorizationException { + // Only admin user is allowed to read scheduler conf, + // in order to avoid leaking sensitive info, such as ACLs + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); + initForWritableEndpoints(callerUGI, true); + + ResourceScheduler scheduler = rm.getResourceScheduler(); + if (scheduler instanceof MutableConfScheduler + && ((MutableConfScheduler) scheduler).isConfigurationMutable()) { + MutableConfigurationProvider mutableConfigurationProvider = + ((MutableConfScheduler) scheduler).getMutableConfProvider(); + // We load the cached configuration from configuration store, + // this should be the conf properties used by the scheduler. + Configuration schedulerConf = mutableConfigurationProvider + .getConfiguration(); + return Response.status(Status.OK) + .entity(new ConfInfo(schedulerConf)) + .build(); + } else { + return Response.status(Status.BAD_REQUEST).entity( + "This API only supports to retrieve scheduler configuration" + + " from a mutable-conf scheduler, underneath scheduler " + + scheduler.getClass().getSimpleName() + + " is not an instance of MutableConfScheduler") + .build(); + } + } + + @GET @Path(RMWSConsts.CHECK_USER_ACCESS_TO_QUEUE) @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) http://git-wip-us.apache.org/repos/asf/hadoop/blob/d352f167/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ConfInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ConfInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ConfInfo.java new file mode 100644 index 0000000..bfc450b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ConfInfo.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import org.apache.hadoop.conf.Configuration; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; + +@XmlRootElement(name = "configuration") +@XmlAccessorType(XmlAccessType.FIELD) +public class ConfInfo { + + protected ArrayList<ConfItem> property = new ArrayList<>(); + + public ConfInfo() { + } // JAXB needs this + + public ConfInfo(Configuration conf) { + conf.forEach(entry -> + add(new ConfItem(entry.getKey(), entry.getValue()))); + } + + public void add(ConfItem confItem) { + property.add(confItem); + } + + public ArrayList<ConfItem> getItems() { + return property; + } + + @XmlAccessorType(XmlAccessType.FIELD) + public static class ConfItem { + + private String name; + private String value; + + public ConfItem() { + // JAXB needs this + } + + public ConfItem(String name, String value){ + this.name = name; + this.value = value; + } + + public String getKey() { + return name; + } + + public String getValue() { + return value; + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d352f167/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java index 99b5648..3e2542c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java @@ -39,6 +39,9 @@ import org.apache.hadoop.yarn.webapp.JerseyTestBase; import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo; import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -56,6 +59,7 @@ import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertNotNull; /** * Test scheduler configuration mutation via REST API. @@ -157,8 +161,40 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { .contextPath("jersey-guice-filter").servletPath("/").build()); } + private CapacitySchedulerConfiguration getSchedulerConf() + throws JSONException { + WebResource r = resource(); + ClientResponse response = + r.path("ws").path("v1").path("cluster") + .queryParam("user.name", userName).path("scheduler-conf") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(Status.OK.getStatusCode(), response.getStatus()); + JSONObject json = response.getEntity(JSONObject.class); + JSONArray items = (JSONArray) json.get("property"); + CapacitySchedulerConfiguration parsedConf = + new CapacitySchedulerConfiguration(); + for (int i=0; i<items.length(); i++) { + JSONObject obj = (JSONObject) items.get(i); + parsedConf.set(obj.get("name").toString(), + obj.get("value").toString()); + } + return parsedConf; + } + + @Test + public void testGetSchedulerConf() throws Exception { + CapacitySchedulerConfiguration orgConf = getSchedulerConf(); + assertNotNull(orgConf); + assertEquals(3, orgConf.getQueues("root").length); + } + @Test public void testAddNestedQueue() throws Exception { + CapacitySchedulerConfiguration orgConf = getSchedulerConf(); + assertNotNull(orgConf); + assertEquals(3, orgConf.getQueues("root").length); + WebResource r = resource(); ClientResponse response; @@ -198,6 +234,10 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { 0.01f); assertEquals(75.0f, newCSConf.getNonLabeledQueueCapacity("root.d.d2"), 0.01f); + + CapacitySchedulerConfiguration newConf = getSchedulerConf(); + assertNotNull(newConf); + assertEquals(4, newConf.getQueues("root").length); } @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/d352f167/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md index 24c2319..041af4c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md @@ -4964,6 +4964,7 @@ Please note that this feature is currently in the alpha stage and is subject to ### HTTP Operations Supported + * GET * PUT ### Elements of the *sched-conf* object @@ -4975,6 +4976,45 @@ Please note that this feature is currently in the alpha stage and is subject to | remove-queue | string | Full path name of a queue to remove | | global-updates | map | Map of key value pairs to update scheduler's global configuration | +### GET Request Examples + +Get requests are used to retrieve the scheduler's configuration that is currently loaded into scheduler's context. + +**XML response** + +HTTP Request: + + Accept: application/xml + Content-Type: application/xml + GET http://rm-http-address:port/ws/v1/cluster/scheduler-conf + +Response Header: + + TTP/1.1 200 OK + Content-Type: application/xml; charset=utf-8 + Transfer-Encoding: chunked + +Response Body: + + +```xml + <?xml version="1.0" encoding="UTF-8" standalone="yes"?> + <configuration> + <property> + <name>yarn.scheduler.capacity.root.queues</name> + <value>default</value> + </property> + <property> + <name>yarn.scheduler.capacity.maximum-applications</name> + <value>10000</value> + </property> + <property> + <name>yarn.scheduler.capacity.root.default.capacity</name> + <value>100</value> + </property> + </configuration> +``` + ### PUT Request Examples Put requests are used to modify the scheduler configuration. A successful mutation results in a 200 response. A malformed request or one which resulted in an invalid scheduler configuration results in a 400 response. --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
