Author: vinodkv
Date: Tue Mar 4 23:35:20 2014
New Revision: 1574252
URL: http://svn.apache.org/r1574252
Log:
YARN-1766. Fixed a bug in ResourceManager to use configuration loaded from the
configuration-provider when booting up. Contributed by Xuan Gong.
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1574252&r1=1574251&r2=1574252&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Tue Mar 4 23:35:20 2014
@@ -391,6 +391,9 @@ Release 2.4.0 - UNRELEASED
YARN-1729. Made TimelineWebServices deserialize the string primary- and
secondary-filters param into the JSON-compatible object. (Billie Rinaldi
via
zjshen)
+
+ YARN-1766. Fixed a bug in ResourceManager to use configuration loaded from
the
+ configuration-provider when booting up. (Xuan Gong via vinodkv)
Release 2.3.1 - UNRELEASED
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1574252&r1=1574251&r2=1574252&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
Tue Mar 4 23:35:20 2014
@@ -36,6 +36,7 @@ import org.apache.hadoop.metrics2.source
import org.apache.hadoop.security.Groups;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.service.Service;
@@ -181,7 +182,6 @@ public class ResourceManager extends Com
@Override
protected void serviceInit(Configuration conf) throws Exception {
- validateConfigs(conf);
this.conf = conf;
this.rmContext = new RMContextImpl();
@@ -190,13 +190,6 @@ public class ResourceManager extends Com
this.configurationProvider.init(this.conf);
rmContext.setConfigurationProvider(configurationProvider);
- // load yarn-site.xml
- InputStream yarnSiteXMLInputStream =
- this.configurationProvider.getConfigurationInputStream(this.conf,
- YarnConfiguration.YARN_SITE_CONFIGURATION_FILE);
- if (yarnSiteXMLInputStream != null) {
- this.conf.addResource(yarnSiteXMLInputStream);
- }
// load core-site.xml
InputStream coreSiteXMLInputStream =
this.configurationProvider.getConfigurationInputStream(this.conf,
@@ -209,6 +202,19 @@ public class ResourceManager extends Com
Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(this.conf)
.refresh();
+ // Do refreshSuperUserGroupsConfiguration with loaded core-site.xml
+ ProxyUsers.refreshSuperUserGroupsConfiguration(this.conf);
+
+ // load yarn-site.xml
+ InputStream yarnSiteXMLInputStream =
+ this.configurationProvider.getConfigurationInputStream(this.conf,
+ YarnConfiguration.YARN_SITE_CONFIGURATION_FILE);
+ if (yarnSiteXMLInputStream != null) {
+ this.conf.addResource(yarnSiteXMLInputStream);
+ }
+
+ validateConfigs(this.conf);
+
// register the handlers for all AlwaysOn services using setupDispatcher().
rmDispatcher = setupDispatcher();
addIfService(rmDispatcher);
@@ -218,15 +224,15 @@ public class ResourceManager extends Com
addService(adminService);
rmContext.setRMAdminService(adminService);
- this.rmContext.setHAEnabled(HAUtil.isHAEnabled(conf));
+ this.rmContext.setHAEnabled(HAUtil.isHAEnabled(this.conf));
if (this.rmContext.isHAEnabled()) {
- HAUtil.verifyAndSetConfiguration(conf);
+ HAUtil.verifyAndSetConfiguration(this.conf);
}
createAndInitActiveServices();
- webAppAddress = WebAppUtils.getRMWebAppURLWithoutScheme(conf);
+ webAppAddress = WebAppUtils.getRMWebAppURLWithoutScheme(this.conf);
- super.serviceInit(conf);
+ super.serviceInit(this.conf);
}
protected QueueACLsManager createQueueACLsManager(ResourceScheduler
scheduler,
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java?rev=1574252&r1=1574251&r2=1574252&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
Tue Mar 4 23:35:20 2014
@@ -590,6 +590,140 @@ public class TestRMAdminService {
}
+ @Test
+ public void testRMInitialsWithFileSystemBasedConfigurationProvider()
+ throws Exception {
+ configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
+ "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
+
+ // upload configurations
+ final File excludeHostsFile = new File(tmpDir.toString(), "excludeHosts");
+ if (excludeHostsFile.exists()) {
+ excludeHostsFile.delete();
+ }
+ if (!excludeHostsFile.createNewFile()) {
+ Assert.fail("Can not create " + "excludeHosts");
+ }
+ PrintWriter fileWriter = new PrintWriter(excludeHostsFile);
+ fileWriter.write("0.0.0.0:123");
+ fileWriter.close();
+ uploadToRemoteFileSystem(new Path(excludeHostsFile.getAbsolutePath()));
+
+ YarnConfiguration yarnConf = new YarnConfiguration();
+ yarnConf.set(YarnConfiguration.YARN_ADMIN_ACL, "world:anyone:rwcda");
+ yarnConf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, this.workingPath
+ + "/excludeHosts");
+ uploadConfiguration(yarnConf, "yarn-site.xml");
+
+ CapacitySchedulerConfiguration csConf =
+ new CapacitySchedulerConfiguration();
+ csConf.set("yarn.scheduler.capacity.maximum-applications", "5000");
+ uploadConfiguration(csConf, "capacity-scheduler.xml");
+
+ String aclsString = "alice,bob users,wheel";
+ Configuration newConf = new Configuration();
+ newConf.set("security.applicationclient.protocol.acl", aclsString);
+ uploadConfiguration(newConf, "hadoop-policy.xml");
+
+ Configuration conf = new Configuration();
+ conf.setBoolean(
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, true);
+ conf.set("hadoop.proxyuser.test.groups", "test_groups");
+ conf.set("hadoop.proxyuser.test.hosts", "test_hosts");
+ conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
+ MockUnixGroupsMapping.class,
+ GroupMappingServiceProvider.class);
+ uploadConfiguration(conf, "core-site.xml");
+
+ // update the groups
+ MockUnixGroupsMapping.updateGroups();
+
+ ResourceManager resourceManager = null;
+ try {
+ try {
+ resourceManager = new ResourceManager();
+ resourceManager.init(configuration);
+ resourceManager.start();
+ } catch (Exception ex) {
+ fail("Should not get any exceptions");
+ }
+
+ // validate values for excludeHosts
+ Set<String> excludeHosts =
+ resourceManager.getRMContext().getNodesListManager()
+ .getHostsReader().getExcludedHosts();
+ Assert.assertTrue(excludeHosts.size() == 1);
+ Assert.assertTrue(excludeHosts.contains("0.0.0.0:123"));
+
+ // validate values for admin-acls
+ String aclStringAfter =
+ resourceManager.adminService.getAccessControlList()
+ .getAclString().trim();
+ Assert.assertEquals(aclStringAfter, "world:anyone:rwcda");
+
+ // validate values for queue configuration
+ CapacityScheduler cs =
+ (CapacityScheduler) resourceManager.getRMContext().getScheduler();
+ int maxAppsAfter = cs.getConfiguration().getMaximumSystemApplications();
+ Assert.assertEquals(maxAppsAfter, 5000);
+
+ // verify service Acls for AdminService
+ ServiceAuthorizationManager adminServiceServiceManager =
+ resourceManager.adminService.getServer()
+ .getServiceAuthorizationManager();
+ verifyServiceACLsRefresh(adminServiceServiceManager,
+ org.apache.hadoop.yarn.api.ApplicationClientProtocolPB.class,
+ aclsString);
+
+ // verify service ACLs for ClientRMService
+ ServiceAuthorizationManager clientRMServiceServiceManager =
+ resourceManager.getRMContext().getClientRMService().getServer()
+ .getServiceAuthorizationManager();
+ verifyServiceACLsRefresh(clientRMServiceServiceManager,
+ org.apache.hadoop.yarn.api.ApplicationClientProtocolPB.class,
+ aclsString);
+
+ // verify service ACLs for ApplicationMasterService
+ ServiceAuthorizationManager appMasterService =
+ resourceManager.getRMContext().getApplicationMasterService()
+ .getServer().getServiceAuthorizationManager();
+ verifyServiceACLsRefresh(appMasterService,
+ org.apache.hadoop.yarn.api.ApplicationClientProtocolPB.class,
+ aclsString);
+
+ // verify service ACLs for ResourceTrackerService
+ ServiceAuthorizationManager RTService =
+ resourceManager.getRMContext().getResourceTrackerService()
+ .getServer().getServiceAuthorizationManager();
+ verifyServiceACLsRefresh(RTService,
+ org.apache.hadoop.yarn.api.ApplicationClientProtocolPB.class,
+ aclsString);
+
+ // verify ProxyUsers and ProxyHosts
+ Assert.assertTrue(ProxyUsers.getProxyGroups()
+ .get("hadoop.proxyuser.test.groups").size() == 1);
+ Assert.assertTrue(ProxyUsers.getProxyGroups()
+ .get("hadoop.proxyuser.test.groups").contains("test_groups"));
+
+ Assert.assertTrue(ProxyUsers.getProxyHosts()
+ .get("hadoop.proxyuser.test.hosts").size() == 1);
+ Assert.assertTrue(ProxyUsers.getProxyHosts()
+ .get("hadoop.proxyuser.test.hosts").contains("test_hosts"));
+
+ // verify UserToGroupsMappings
+ List<String> groupAfter =
+ Groups.getUserToGroupsMappingService(configuration).getGroups(
+ UserGroupInformation.getCurrentUser().getUserName());
+ Assert.assertTrue(groupAfter.contains("test_group_D")
+ && groupAfter.contains("test_group_E")
+ && groupAfter.contains("test_group_F") && groupAfter.size() == 3);
+ } finally {
+ if (resourceManager != null) {
+ resourceManager.stop();
+ }
+ }
+ }
+
private String writeConfigurationXML(Configuration conf, String confXMLName)
throws IOException {
DataOutputStream output = null;
