Author: szetszwo
Date: Mon Mar 3 23:51:58 2014
New Revision: 1573821
URL: http://svn.apache.org/r1573821
Log:
Merge r1569890 through r1573813 from trunk.
Added:
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestKillApplicationWithRMHA.java
- copied unchanged from r1573813,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestKillApplicationWithRMHA.java
Modified:
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/LICENSE.txt
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/NOTICE.txt
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/CHANGES.txt?rev=1573821&r1=1573820&r2=1573821&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/CHANGES.txt Mon Mar 3
23:51:58 2014
@@ -243,6 +243,12 @@ Release 2.4.0 - UNRELEASED
YARN-1528. Allow setting auth for ZK connections. (kasha)
+ YARN-1704. Modified LICENSE and NOTICE files to reflect newly used levelDB
+ related libraries. (Billie Rinaldi via vinodkv)
+
+ YARN-1765. Added test cases to verify that killApplication API works across
+ ResourceManager failover. (Xuan Gong via vinodkv)
+
OPTIMIZATIONS
BUG FIXES
@@ -368,6 +374,13 @@ Release 2.4.0 - UNRELEASED
YARN-1760. TestRMAdminService assumes CapacityScheduler. (kasha)
+ YARN-1758. Fixed ResourceManager to not mandate the presence of site
specific
+ configuration files and thus fix failures in downstream tests. (Xuan Gong
via
+ vinodkv)
+
+ YARN-1748. Excluded core-site.xml from hadoop-yarn-server-tests package's
jar
+ and thus avoid breaking downstream tests. (Sravya Tirukkovalur via vinodkv)
+
Release 2.3.1 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/LICENSE.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/LICENSE.txt?rev=1573821&r1=1573820&r2=1573821&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/LICENSE.txt (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/LICENSE.txt Mon Mar 3
23:51:58 2014
@@ -242,3 +242,100 @@ For the org.apache.hadoop.util.bloom.* c
* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
+
+The binary distribution of this product bundles binaries of leveldbjni
+(https://github.com/fusesource/leveldbjni), which is available under the
+following license:
+
+Copyright (c) 2011 FuseSource Corp. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of FuseSource Corp. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The binary distribution of this product bundles binaries of leveldb
+(http://code.google.com/p/leveldb/), which is available under the following
+license:
+
+Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The binary distribution of this product bundles binaries of snappy
+(http://code.google.com/p/snappy/), which is available under the following
+license:
+
+Copyright 2011, Google Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/NOTICE.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/NOTICE.txt?rev=1573821&r1=1573820&r2=1573821&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/NOTICE.txt (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/NOTICE.txt Mon Mar 3
23:51:58 2014
@@ -1,2 +1,17 @@
This product includes software developed by The Apache Software
Foundation (http://www.apache.org/).
+
+The binary distribution of this product bundles binaries of
+org.iq80.leveldb:leveldb-api (https://github.com/dain/leveldb), which has the
+following notices:
+* Copyright 2011 Dain Sundstrom <[email protected]>
+* Copyright 2011 FuseSource Corp. http://fusesource.com
+
+The binary distribution of this product bundles binaries of
+org.fusesource.hawtjni:hawtjni-runtime (https://github.com/fusesource/hawtjni),
+which has the following notices:
+* This product includes software developed by FuseSource Corp.
+ http://fusesource.com
+* This product includes software developed at
+ Progress Software Corporation and/or its subsidiaries or affiliates.
+* This product includes software developed by IBM Corporation and others.
Modified:
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java?rev=1573821&r1=1573820&r2=1573821&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java
Mon Mar 3 23:51:58 2014
@@ -54,13 +54,14 @@ public class FileSystemBasedConfiguratio
if (YarnConfiguration.RM_CONFIGURATION_FILES.contains(name)) {
filePath = new Path(this.configDir, name);
if (!fs.exists(filePath)) {
- throw new YarnException("Can not find Configuration: " + name + " in "
- + configDir);
+ LOG.info(filePath + " not found");
+ return null;
}
} else {
filePath = new Path(name);
if (!fs.exists(filePath)) {
- throw new YarnException("Can not find file: " + name);
+ LOG.info(filePath + " not found");
+ return null;
}
}
return fs.open(filePath);
Modified:
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java?rev=1573821&r1=1573820&r2=1573821&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
Mon Mar 3 23:51:58 2014
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import java.io.IOException;
+import java.io.InputStream;
import java.net.InetSocketAddress;
import java.util.Map;
import java.util.Set;
@@ -538,8 +539,11 @@ public class AdminService extends Compos
private synchronized Configuration getConfiguration(Configuration conf,
String confFileName) throws YarnException, IOException {
- conf.addResource(this.rmContext.getConfigurationProvider()
- .getConfigurationInputStream(conf, confFileName));
+ InputStream confFileInputStream = this.rmContext.getConfigurationProvider()
+ .getConfigurationInputStream(conf, confFileName);
+ if (confFileInputStream != null) {
+ conf.addResource(confFileInputStream);
+ }
return conf;
}
Modified:
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java?rev=1573821&r1=1573820&r2=1573821&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
Mon Mar 3 23:51:58 2014
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import java.io.IOException;
+import java.io.InputStream;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
@@ -142,9 +143,13 @@ public class ApplicationMasterService ex
if (conf.getBoolean(
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
false)) {
- conf.addResource(this.rmContext.getConfigurationProvider()
- .getConfigurationInputStream(conf,
- YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE));
+ InputStream inputStream =
+ this.rmContext.getConfigurationProvider()
+ .getConfigurationInputStream(conf,
+ YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE);
+ if (inputStream != null) {
+ conf.addResource(inputStream);
+ }
refreshServiceAcls(conf, RMPolicyProvider.getInstance());
}
Modified:
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java?rev=1573821&r1=1573820&r2=1573821&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
Mon Mar 3 23:51:58 2014
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import java.io.IOException;
+import java.io.InputStream;
import java.net.InetSocketAddress;
import java.security.AccessControlException;
import java.util.ArrayList;
@@ -170,9 +171,13 @@ public class ClientRMService extends Abs
if (conf.getBoolean(
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
false)) {
- conf.addResource(this.rmContext.getConfigurationProvider()
- .getConfigurationInputStream(conf,
- YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE));
+ InputStream inputStream =
+ this.rmContext.getConfigurationProvider()
+ .getConfigurationInputStream(conf,
+ YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE);
+ if (inputStream != null) {
+ conf.addResource(inputStream);
+ }
refreshServiceAcls(conf, RMPolicyProvider.getInstance());
}
Modified:
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1573821&r1=1573820&r2=1573821&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
Mon Mar 3 23:51:58 2014
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import java.io.IOException;
+import java.io.InputStream;
import java.net.InetSocketAddress;
import java.util.List;
import java.util.concurrent.BlockingQueue;
@@ -190,13 +191,20 @@ public class ResourceManager extends Com
rmContext.setConfigurationProvider(configurationProvider);
// load yarn-site.xml
- this.conf.addResource(this.configurationProvider
- .getConfigurationInputStream(this.conf,
- YarnConfiguration.YARN_SITE_CONFIGURATION_FILE));
+ InputStream yarnSiteXMLInputStream =
+ this.configurationProvider.getConfigurationInputStream(this.conf,
+ YarnConfiguration.YARN_SITE_CONFIGURATION_FILE);
+ if (yarnSiteXMLInputStream != null) {
+ this.conf.addResource(yarnSiteXMLInputStream);
+ }
// load core-site.xml
- this.conf.addResource(this.configurationProvider
- .getConfigurationInputStream(this.conf,
- YarnConfiguration.CORE_SITE_CONFIGURATION_FILE));
+ InputStream coreSiteXMLInputStream =
+ this.configurationProvider.getConfigurationInputStream(this.conf,
+ YarnConfiguration.CORE_SITE_CONFIGURATION_FILE);
+ if (coreSiteXMLInputStream != null) {
+ this.conf.addResource(coreSiteXMLInputStream);
+ }
+
// Do refreshUserToGroupsMappings with loaded core-site.xml
Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(this.conf)
.refresh();
Modified:
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java?rev=1573821&r1=1573820&r2=1573821&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
Mon Mar 3 23:51:58 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import java.io.IOException;
+import java.io.InputStream;
import java.net.InetSocketAddress;
import org.apache.commons.logging.Log;
@@ -163,9 +164,13 @@ public class ResourceTrackerService exte
if (conf.getBoolean(
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
false)) {
- conf.addResource(this.rmContext.getConfigurationProvider()
- .getConfigurationInputStream(conf,
- YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE));
+ InputStream inputStream =
+ this.rmContext.getConfigurationProvider()
+ .getConfigurationInputStream(conf,
+ YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE);
+ if (inputStream != null) {
+ conf.addResource(inputStream);
+ }
refreshServiceAcls(conf, RMPolicyProvider.getInstance());
}
Modified:
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1573821&r1=1573820&r2=1573821&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
Mon Mar 3 23:51:58 2014
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import java.io.IOException;
+import java.io.InputStream;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
@@ -1024,10 +1025,15 @@ public class CapacityScheduler extends A
private CapacitySchedulerConfiguration loadCapacitySchedulerConfiguration(
Configuration configuration) throws IOException {
try {
- configuration.addResource(this.rmContext.getConfigurationProvider()
- .getConfigurationInputStream(configuration,
- YarnConfiguration.CS_CONFIGURATION_FILE));
- return new CapacitySchedulerConfiguration(configuration, false);
+ InputStream CSInputStream =
+ this.rmContext.getConfigurationProvider()
+ .getConfigurationInputStream(configuration,
+ YarnConfiguration.CS_CONFIGURATION_FILE);
+ if (CSInputStream != null) {
+ configuration.addResource(CSInputStream);
+ return new CapacitySchedulerConfiguration(configuration, false);
+ }
+ return new CapacitySchedulerConfiguration(configuration, true);
} catch (Exception e) {
throw new IOException(e);
}
Modified:
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java?rev=1573821&r1=1573820&r2=1573821&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
Mon Mar 3 23:51:58 2014
@@ -68,9 +68,16 @@ public class TestRMAdminService {
private Path workingPath;
private Path tmpDir;
+ static {
+ YarnConfiguration.addDefaultResource(
+ YarnConfiguration.CS_CONFIGURATION_FILE);
+ }
+
@Before
public void setup() throws IOException {
configuration = new YarnConfiguration();
+ configuration.set(YarnConfiguration.RM_SCHEDULER,
+ CapacityScheduler.class.getCanonicalName());
fs = FileSystem.get(configuration);
workingPath =
new Path(new File("target", this.getClass().getSimpleName()
@@ -94,16 +101,9 @@ public class TestRMAdminService {
fs.delete(tmpDir, true);
}
- private void useCapacityScheduler() {
- configuration.set(YarnConfiguration.RM_SCHEDULER,
- CapacityScheduler.class.getCanonicalName());
- configuration.addResource(YarnConfiguration.CS_CONFIGURATION_FILE);
- }
-
@Test
public void testAdminRefreshQueuesWithLocalConfigurationProvider()
throws IOException, YarnException {
- useCapacityScheduler();
rm = new MockRM(configuration);
rm.init(configuration);
rm.start();
@@ -126,15 +126,6 @@ public class TestRMAdminService {
throws IOException, YarnException {
configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
"org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
- useCapacityScheduler();
- try {
- rm = new MockRM(configuration);
- rm.init(configuration);
- rm.start();
- fail("Should throw an exception");
- } catch(Exception ex) {
- // Expect exception here
- }
//upload default configurations
uploadDefaultConfiguration();
@@ -181,14 +172,6 @@ public class TestRMAdminService {
throws IOException, YarnException {
configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
"org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
- try {
- rm = new MockRM(configuration);
- rm.init(configuration);
- rm.start();
- fail("Should throw an exception");
- } catch(Exception ex) {
- // Expect exception here
- }
//upload default configurations
uploadDefaultConfiguration();
@@ -247,14 +230,6 @@ public class TestRMAdminService {
"org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
ResourceManager resourceManager = null;
try {
- try {
- resourceManager = new ResourceManager();
- resourceManager.init(configuration);
- resourceManager.start();
- fail("Should throw an exception");
- } catch (Exception ex) {
- // expect to get an exception here
- }
//upload default configurations
uploadDefaultConfiguration();
@@ -351,14 +326,6 @@ public class TestRMAdminService {
throws IOException, YarnException {
configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
"org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
- try {
- rm = new MockRM(configuration);
- rm.init(configuration);
- rm.start();
- fail("Should throw an exception");
- } catch(Exception ex) {
- // Expect exception here
- }
//upload default configurations
uploadDefaultConfiguration();
@@ -409,14 +376,6 @@ public class TestRMAdminService {
throws IOException, YarnException {
configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
"org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
- try {
- rm = new MockRM(configuration);
- rm.init(configuration);
- rm.start();
- fail("Should throw an exception");
- } catch (Exception ex) {
- // Expect exception here
- }
String user = UserGroupInformation.getCurrentUser().getUserName();
List<String> groupWithInit =
@@ -485,14 +444,6 @@ public class TestRMAdminService {
throws IOException, YarnException {
configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
"org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
- try {
- rm = new MockRM(configuration);
- rm.init(configuration);
- rm.start();
- fail("Should throw an exception");
- } catch (Exception ex) {
- // Expect exception here
- }
// upload default configurations
uploadDefaultConfiguration();
@@ -618,6 +569,27 @@ public class TestRMAdminService {
}
}
+ @Test
+ public void testRMStartsWithoutConfigurationFilesProvided() {
+ // enable FileSystemBasedConfigurationProvider without uploading
+ // any configuration files into Remote File System.
+ configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
+ "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
+
+ // The configurationProvider will return NULL instead of
+ // throwing out Exceptions, if there are no configuration files provided.
+ // RM will not load the remote Configuration files,
+ // and should start successfully.
+ try {
+ rm = new MockRM(configuration);
+ rm.init(configuration);
+ rm.start();
+ } catch (Exception ex) {
+ fail("Should not get any exceptions");
+ }
+
+ }
+
private String writeConfigurationXML(Configuration conf, String confXMLName)
throws IOException {
DataOutputStream output = null;
Modified:
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml?rev=1573821&r1=1573820&r2=1573821&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
Mon Mar 3 23:51:58 2014
@@ -138,6 +138,11 @@
<goals>
<goal>test-jar</goal>
</goals>
+ <configuration>
+ <excludes>
+ <exclude>**/core-site.xml</exclude>
+ </excludes>
+ </configuration>
<phase>test-compile</phase>
</execution>
</executions>
