Author: vinodkv
Date: Mon Mar 3 19:42:24 2014
New Revision: 1573697
URL: http://svn.apache.org/r1573697
Log:
YARN-1758. Fixed ResourceManager to not mandate the presence of site specific
configuration files and thus fix failures in downstream tests. Contributed by
Xuan Gong.
svn merge --ignore-ancestry -c 1573695 ../../trunk/
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1573697&r1=1573696&r2=1573697&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Mon Mar 3
19:42:24 2014
@@ -353,6 +353,10 @@ Release 2.4.0 - UNRELEASED
YARN-1760. TestRMAdminService assumes CapacityScheduler. (kasha)
+ YARN-1758. Fixed ResourceManager to not mandate the presence of site
specific
+ configuration files and thus fix failures in downstream tests. (Xuan Gong
via
+ vinodkv)
+
Release 2.3.1 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java?rev=1573697&r1=1573696&r2=1573697&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java
Mon Mar 3 19:42:24 2014
@@ -54,13 +54,14 @@ public class FileSystemBasedConfiguratio
if (YarnConfiguration.RM_CONFIGURATION_FILES.contains(name)) {
filePath = new Path(this.configDir, name);
if (!fs.exists(filePath)) {
- throw new YarnException("Can not find Configuration: " + name + " in "
- + configDir);
+ LOG.info(filePath + " not found");
+ return null;
}
} else {
filePath = new Path(name);
if (!fs.exists(filePath)) {
- throw new YarnException("Can not find file: " + name);
+ LOG.info(filePath + " not found");
+ return null;
}
}
return fs.open(filePath);
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java?rev=1573697&r1=1573696&r2=1573697&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
Mon Mar 3 19:42:24 2014
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import java.io.IOException;
+import java.io.InputStream;
import java.net.InetSocketAddress;
import java.util.Map;
import java.util.Set;
@@ -538,8 +539,11 @@ public class AdminService extends Compos
private synchronized Configuration getConfiguration(Configuration conf,
String confFileName) throws YarnException, IOException {
- conf.addResource(this.rmContext.getConfigurationProvider()
- .getConfigurationInputStream(conf, confFileName));
+ InputStream confFileInputStream = this.rmContext.getConfigurationProvider()
+ .getConfigurationInputStream(conf, confFileName);
+ if (confFileInputStream != null) {
+ conf.addResource(confFileInputStream);
+ }
return conf;
}
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java?rev=1573697&r1=1573696&r2=1573697&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
Mon Mar 3 19:42:24 2014
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import java.io.IOException;
+import java.io.InputStream;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
@@ -142,9 +143,13 @@ public class ApplicationMasterService ex
if (conf.getBoolean(
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
false)) {
- conf.addResource(this.rmContext.getConfigurationProvider()
- .getConfigurationInputStream(conf,
- YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE));
+ InputStream inputStream =
+ this.rmContext.getConfigurationProvider()
+ .getConfigurationInputStream(conf,
+ YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE);
+ if (inputStream != null) {
+ conf.addResource(inputStream);
+ }
refreshServiceAcls(conf, RMPolicyProvider.getInstance());
}
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java?rev=1573697&r1=1573696&r2=1573697&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
Mon Mar 3 19:42:24 2014
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import java.io.IOException;
+import java.io.InputStream;
import java.net.InetSocketAddress;
import java.security.AccessControlException;
import java.util.ArrayList;
@@ -169,9 +170,13 @@ public class ClientRMService extends Abs
if (conf.getBoolean(
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
false)) {
- conf.addResource(this.rmContext.getConfigurationProvider()
- .getConfigurationInputStream(conf,
- YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE));
+ InputStream inputStream =
+ this.rmContext.getConfigurationProvider()
+ .getConfigurationInputStream(conf,
+ YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE);
+ if (inputStream != null) {
+ conf.addResource(inputStream);
+ }
refreshServiceAcls(conf, RMPolicyProvider.getInstance());
}
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1573697&r1=1573696&r2=1573697&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
Mon Mar 3 19:42:24 2014
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import java.io.IOException;
+import java.io.InputStream;
import java.net.InetSocketAddress;
import java.util.List;
import java.util.concurrent.BlockingQueue;
@@ -190,13 +191,20 @@ public class ResourceManager extends Com
rmContext.setConfigurationProvider(configurationProvider);
// load yarn-site.xml
- this.conf.addResource(this.configurationProvider
- .getConfigurationInputStream(this.conf,
- YarnConfiguration.YARN_SITE_CONFIGURATION_FILE));
+ InputStream yarnSiteXMLInputStream =
+ this.configurationProvider.getConfigurationInputStream(this.conf,
+ YarnConfiguration.YARN_SITE_CONFIGURATION_FILE);
+ if (yarnSiteXMLInputStream != null) {
+ this.conf.addResource(yarnSiteXMLInputStream);
+ }
// load core-site.xml
- this.conf.addResource(this.configurationProvider
- .getConfigurationInputStream(this.conf,
- YarnConfiguration.CORE_SITE_CONFIGURATION_FILE));
+ InputStream coreSiteXMLInputStream =
+ this.configurationProvider.getConfigurationInputStream(this.conf,
+ YarnConfiguration.CORE_SITE_CONFIGURATION_FILE);
+ if (coreSiteXMLInputStream != null) {
+ this.conf.addResource(coreSiteXMLInputStream);
+ }
+
// Do refreshUserToGroupsMappings with loaded core-site.xml
Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(this.conf)
.refresh();
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java?rev=1573697&r1=1573696&r2=1573697&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
Mon Mar 3 19:42:24 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import java.io.IOException;
+import java.io.InputStream;
import java.net.InetSocketAddress;
import org.apache.commons.logging.Log;
@@ -163,9 +164,13 @@ public class ResourceTrackerService exte
if (conf.getBoolean(
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
false)) {
- conf.addResource(this.rmContext.getConfigurationProvider()
- .getConfigurationInputStream(conf,
- YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE));
+ InputStream inputStream =
+ this.rmContext.getConfigurationProvider()
+ .getConfigurationInputStream(conf,
+ YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE);
+ if (inputStream != null) {
+ conf.addResource(inputStream);
+ }
refreshServiceAcls(conf, RMPolicyProvider.getInstance());
}
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1573697&r1=1573696&r2=1573697&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
Mon Mar 3 19:42:24 2014
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import java.io.IOException;
+import java.io.InputStream;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
@@ -1024,10 +1025,15 @@ public class CapacityScheduler extends A
private CapacitySchedulerConfiguration loadCapacitySchedulerConfiguration(
Configuration configuration) throws IOException {
try {
- configuration.addResource(this.rmContext.getConfigurationProvider()
- .getConfigurationInputStream(configuration,
- YarnConfiguration.CS_CONFIGURATION_FILE));
- return new CapacitySchedulerConfiguration(configuration, false);
+ InputStream CSInputStream =
+ this.rmContext.getConfigurationProvider()
+ .getConfigurationInputStream(configuration,
+ YarnConfiguration.CS_CONFIGURATION_FILE);
+ if (CSInputStream != null) {
+ configuration.addResource(CSInputStream);
+ return new CapacitySchedulerConfiguration(configuration, false);
+ }
+ return new CapacitySchedulerConfiguration(configuration, true);
} catch (Exception e) {
throw new IOException(e);
}
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java?rev=1573697&r1=1573696&r2=1573697&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
Mon Mar 3 19:42:24 2014
@@ -126,14 +126,6 @@ public class TestRMAdminService {
throws IOException, YarnException {
configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
"org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
- try {
- rm = new MockRM(configuration);
- rm.init(configuration);
- rm.start();
- fail("Should throw an exception");
- } catch(Exception ex) {
- // Expect exception here
- }
//upload default configurations
uploadDefaultConfiguration();
@@ -180,14 +172,6 @@ public class TestRMAdminService {
throws IOException, YarnException {
configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
"org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
- try {
- rm = new MockRM(configuration);
- rm.init(configuration);
- rm.start();
- fail("Should throw an exception");
- } catch(Exception ex) {
- // Expect exception here
- }
//upload default configurations
uploadDefaultConfiguration();
@@ -246,14 +230,6 @@ public class TestRMAdminService {
"org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
ResourceManager resourceManager = null;
try {
- try {
- resourceManager = new ResourceManager();
- resourceManager.init(configuration);
- resourceManager.start();
- fail("Should throw an exception");
- } catch (Exception ex) {
- // expect to get an exception here
- }
//upload default configurations
uploadDefaultConfiguration();
@@ -350,14 +326,6 @@ public class TestRMAdminService {
throws IOException, YarnException {
configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
"org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
- try {
- rm = new MockRM(configuration);
- rm.init(configuration);
- rm.start();
- fail("Should throw an exception");
- } catch(Exception ex) {
- // Expect exception here
- }
//upload default configurations
uploadDefaultConfiguration();
@@ -408,14 +376,6 @@ public class TestRMAdminService {
throws IOException, YarnException {
configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
"org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
- try {
- rm = new MockRM(configuration);
- rm.init(configuration);
- rm.start();
- fail("Should throw an exception");
- } catch (Exception ex) {
- // Expect exception here
- }
String user = UserGroupInformation.getCurrentUser().getUserName();
List<String> groupWithInit =
@@ -484,14 +444,6 @@ public class TestRMAdminService {
throws IOException, YarnException {
configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
"org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
- try {
- rm = new MockRM(configuration);
- rm.init(configuration);
- rm.start();
- fail("Should throw an exception");
- } catch (Exception ex) {
- // Expect exception here
- }
// upload default configurations
uploadDefaultConfiguration();
@@ -617,6 +569,27 @@ public class TestRMAdminService {
}
}
+ @Test
+ public void testRMStartsWithoutConfigurationFilesProvided() {
+ // enable FileSystemBasedConfigurationProvider without uploading
+ // any configuration files into Remote File System.
+ configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
+ "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
+
+ // The configurationProvider will return NULL instead of
+ // throwing out Exceptions, if there are no configuration files provided.
+ // RM will not load the remote Configuration files,
+ // and should start successfully.
+ try {
+ rm = new MockRM(configuration);
+ rm.init(configuration);
+ rm.start();
+ } catch (Exception ex) {
+ fail("Should not get any exceptions");
+ }
+
+ }
+
private String writeConfigurationXML(Configuration conf, String confXMLName)
throws IOException {
DataOutputStream output = null;
