Author: vinodkv
Date: Mon Jul 29 22:23:58 2013
New Revision: 1508217
URL: http://svn.apache.org/r1508217
Log:
YARN-961. Changed ContainerManager to enforce Token auth irrespective of
security. Contributed by Omkar Vinit Joshi.
svn merge --ignore-ancestry -c 1508216 ../../trunk/
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1508217&r1=1508216&r2=1508217&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Mon Jul 29
22:23:58 2013
@@ -750,6 +750,12 @@ Release 2.1.0-beta - 2013-07-02
YARN-245. Fixed NodeManager to handle duplicate responses from
ResourceManager. (Mayank Bansal via vinodkv)
+ YARN-932. TestResourceLocalizationService.testLocalizationInit can fail on
+ JDK7. (Karthik Kambatla via Sandy Ryza)
+
+ YARN-961. Changed ContainerManager to enforce Token auth irrespective of
+ security. (Omkar Vinit Joshi via vinodkv)
+
BREAKDOWN OF HADOOP-8562/YARN-191 SUBTASKS AND RELATED JIRAS
YARN-158. Yarn creating package-info.java must not depend on sh.
@@ -815,9 +821,6 @@ Release 2.1.0-beta - 2013-07-02
YARN-909. Disable TestLinuxContainerExecutorWithMocks on Windows. (Chuan
Liu
via cnauroth)
- YARN-932. TestResourceLocalizationService.testLocalizationInit can fail on
- JDK7. (Karthik Kambatla via Sandy Ryza)
-
Release 2.0.5-alpha - 06/06/2013
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java?rev=1508217&r1=1508216&r2=1508217&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
Mon Jul 29 22:23:58 2013
@@ -40,6 +40,7 @@ import org.apache.hadoop.io.DataInputByt
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SaslRpcServer;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.PolicyProvider;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
@@ -230,6 +231,13 @@ public class ContainerManagerImpl extend
// Enqueue user dirs in deletion context
Configuration conf = getConfig();
+ Configuration serverConf = new Configuration(conf);
+
+ // always enforce it to be token-based.
+ serverConf.set(
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ SaslRpcServer.AuthMethod.TOKEN.toString());
+
YarnRPC rpc = YarnRPC.create(conf);
InetSocketAddress initialAddress = conf.getSocketAddr(
@@ -238,8 +246,8 @@ public class ContainerManagerImpl extend
YarnConfiguration.DEFAULT_NM_PORT);
server =
- rpc.getServer(ContainerManagementProtocol.class, this, initialAddress,
conf,
- this.context.getNMTokenSecretManager(),
+ rpc.getServer(ContainerManagementProtocol.class, this, initialAddress,
+ serverConf, this.context.getNMTokenSecretManager(),
conf.getInt(YarnConfiguration.NM_CONTAINER_MGR_THREAD_COUNT,
YarnConfiguration.DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT));
@@ -249,7 +257,7 @@ public class ContainerManagerImpl extend
false)) {
refreshServiceAcls(conf, new NMPolicyProvider());
}
-
+
LOG.info("Blocking new container-requests as container manager rpc" +
" server is still starting.");
this.setBlockNewContainerRequests(true);
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java?rev=1508217&r1=1508216&r2=1508217&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
Mon Jul 29 22:23:58 2013
@@ -33,6 +33,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.security.SaslRpcServer;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.PolicyProvider;
import org.apache.hadoop.security.token.TokenIdentifier;
@@ -119,12 +120,11 @@ public class ApplicationMasterService ex
YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
Configuration serverConf = conf;
- if (!UserGroupInformation.isSecurityEnabled()) {
- // If the auth is not-simple, enforce it to be token-based.
- serverConf = new Configuration(conf);
-
serverConf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
- UserGroupInformation.AuthenticationMethod.TOKEN.toString());
- }
+ // If the auth is not-simple, enforce it to be token-based.
+ serverConf = new Configuration(conf);
+ serverConf.set(
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ SaslRpcServer.AuthMethod.TOKEN.toString());
this.server =
rpc.getServer(ApplicationMasterProtocol.class, this,
masterServiceAddress,
serverConf, this.rmContext.getAMRMTokenSecretManager(),
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java?rev=1508217&r1=1508216&r2=1508217&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java
Mon Jul 29 22:23:58 2013
@@ -253,15 +253,14 @@ public class TestAMAuthorization {
} catch (Exception e) {
// Because there are no tokens, the request should be rejected as the
// server side will assume we are trying simple auth.
- String availableAuthMethods;
+ String expectedMessage = "";
if (UserGroupInformation.isSecurityEnabled()) {
- availableAuthMethods = "[TOKEN, KERBEROS]";
+ expectedMessage = "Client cannot authenticate via:[TOKEN]";
} else {
- availableAuthMethods = "[TOKEN]";
+ expectedMessage =
+ "SIMPLE authentication is not enabled. Available:[TOKEN]";
}
- Assert.assertTrue(e.getCause().getMessage().contains(
- "SIMPLE authentication is not enabled. "
- + "Available:" + availableAuthMethods));
+ Assert.assertTrue(e.getCause().getMessage().contains(expectedMessage));
}
// TODO: Add validation of invalid authorization when there's more data in
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java?rev=1508217&r1=1508216&r2=1508217&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
Mon Jul 29 22:23:58 2013
@@ -24,6 +24,8 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.security.PrivilegedAction;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.List;
import junit.framework.Assert;
@@ -63,7 +65,11 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+@RunWith(Parameterized.class)
public class TestContainerManagerSecurity {
static Log LOG = LogFactory.getLog(TestContainerManagerSecurity.class);
@@ -71,28 +77,33 @@ public class TestContainerManagerSecurit
.getRecordFactory(null);
private static MiniYARNCluster yarnCluster;
- static final Configuration conf = new Configuration();
+ private Configuration conf;
- @Test (timeout = 1000000)
- public void testContainerManagerWithSecurityEnabled() throws Exception {
- conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ @Parameters
+ public static Collection<Object[]> configs() {
+ Configuration configurationWithoutSecurity = new Configuration();
+ configurationWithoutSecurity.set(
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"simple");
+
+ Configuration configurationWithSecurity = new Configuration();
+ configurationWithSecurity.set(
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"kerberos");
- testContainerManager();
+ return Arrays.asList(new Object[][] { { configurationWithoutSecurity },
+ { configurationWithSecurity } });
}
- @Test (timeout=1000000)
- public void testContainerManagerWithSecurityDisabled() throws Exception {
- conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
- "simple");
- testContainerManager();
+ public TestContainerManagerSecurity(Configuration conf) {
+ conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 100000L);
+ UserGroupInformation.setConfiguration(conf);
+ this.conf = conf;
}
- private void testContainerManager() throws Exception {
+ @Test (timeout = 1000000)
+ public void testContainerManager() throws Exception {
try {
yarnCluster = new MiniYARNCluster(TestContainerManagerSecurity.class
.getName(), 1, 1, 1);
- conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 100000L);
- UserGroupInformation.setConfiguration(conf);
yarnCluster.init(conf);
yarnCluster.start();
@@ -184,6 +195,18 @@ public class TestContainerManagerSecurit
} while (tempManager.getCurrentKey().getKeyId() == nmTokenSecretManagerRM
.getCurrentKey().getKeyId());
+ // Testing that NM rejects the requests when we don't send any token.
+ if (UserGroupInformation.isSecurityEnabled()) {
+ sb = new StringBuilder("Client cannot authenticate via:[TOKEN]");
+ } else {
+ sb =
+ new StringBuilder(
+ "SIMPLE authentication is not enabled. Available:[TOKEN]");
+ }
+ String errorMsg = testStartContainer(rpc, validAppAttemptId, validNode,
+ validContainerToken, null, true);
+ Assert.assertTrue(errorMsg.contains(sb.toString()));
+
org.apache.hadoop.yarn.api.records.Token invalidNMToken =
tempManager.createNMToken(validAppAttemptId, validNode, user);
sb = new StringBuilder("Given NMToken for application : ");
@@ -402,7 +425,9 @@ public class TestContainerManagerSecurit
UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
final InetSocketAddress addr =
NetUtils.createSocketAddr(nodeId.getHost(), nodeId.getPort());
- ugi.addToken(ConverterUtils.convertFromYarn(nmToken, addr));
+ if (nmToken != null) {
+ ugi.addToken(ConverterUtils.convertFromYarn(nmToken, addr));
+ }
proxy = ugi
.doAs(new PrivilegedAction<ContainerManagementProtocol>() {
