Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java?rev=1480440&r1=1480439&r2=1480440&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java Wed May 8 20:20:56 2013 @@ -404,7 +404,8 @@ public class TestRMRestart { } @Test - public void testTokenRestoredOnRMrestart() throws Exception { + public void testDelegationTokenRestoredInDelegationTokenRenewer() + throws Exception { Logger rootLogger = LogManager.getRootLogger(); rootLogger.setLevel(Level.DEBUG); ExitUtil.disableSystemExit(); @@ -423,7 +424,7 @@ public class TestRMRestart { Map<ApplicationId, ApplicationState> rmAppState = rmState.getApplicationState(); - MockRM rm1 = new MyMockRM(conf, memStore); + MockRM rm1 = new TestSecurityMockRM(conf, memStore); rm1.start(); HashSet<Token<RMDelegationTokenIdentifier>> tokenSet = @@ -461,21 +462,26 @@ public class TestRMRestart { ApplicationState appState = rmAppState.get(app.getApplicationId()); Assert.assertNotNull(appState); + // assert delegation tokens exist in rm1 DelegationTokenRenewr + Assert.assertEquals(tokenSet, rm1.getRMContext() + .getDelegationTokenRenewer().getDelegationTokens()); + // assert delegation tokens are saved DataOutputBuffer dob = new DataOutputBuffer(); ts.writeTokenStorageToStream(dob); ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); + securityTokens.rewind(); Assert.assertEquals(securityTokens, appState .getApplicationSubmissionContext().getAMContainerSpec() .getContainerTokens()); // start new RM - MockRM rm2 = new MyMockRM(conf, memStore); + MockRM rm2 = new TestSecurityMockRM(conf, memStore); rm2.start(); - // verify tokens are properly populated back to DelegationTokenRenewer - Assert.assertEquals(tokenSet, rm1.getRMContext() + // verify tokens are properly populated back to rm2 DelegationTokenRenewer + Assert.assertEquals(tokenSet, rm2.getRMContext() .getDelegationTokenRenewer().getDelegationTokens()); // stop the RM @@ -483,9 +489,92 @@ public class TestRMRestart { rm2.stop(); } - class MyMockRM extends MockRM { + @Test + public void testAppAttemptTokensRestoredOnRMRestart() throws Exception { + Logger rootLogger = LogManager.getRootLogger(); + rootLogger.setLevel(Level.DEBUG); + ExitUtil.disableSystemExit(); + + YarnConfiguration conf = new YarnConfiguration(); + conf.set(YarnConfiguration.RECOVERY_ENABLED, "true"); + conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName()); + conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2); + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, + "kerberos"); + UserGroupInformation.setConfiguration(conf); + + MemoryRMStateStore memStore = new MemoryRMStateStore(); + memStore.init(conf); + RMState rmState = memStore.getState(); + + Map<ApplicationId, ApplicationState> rmAppState = + rmState.getApplicationState(); + MockRM rm1 = new TestSecurityMockRM(conf, memStore); + rm1.start(); + MockNM nm1 = + new MockNM("0.0.0.0:4321", 15120, rm1.getResourceTrackerService()); + nm1.registerNode(); + + // submit an app + RMApp app1 = + rm1.submitApp(200, "name", "user", + new HashMap<ApplicationAccessType, String>(), "default"); + + // assert app info is saved + ApplicationState appState = rmAppState.get(app1.getApplicationId()); + Assert.assertNotNull(appState); + + // Allocate the AM + nm1.nodeHeartbeat(true); + RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); + ApplicationAttemptId attemptId1 = attempt1.getAppAttemptId(); + rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED); + + // assert attempt info is saved + ApplicationAttemptState attemptState = appState.getAttempt(attemptId1); + Assert.assertNotNull(attemptState); + Assert.assertEquals(BuilderUtils.newContainerId(attemptId1, 1), + attemptState.getMasterContainer().getId()); + + // the appToken and clientToken that are generated when RMAppAttempt is created, + HashSet<Token<?>> tokenSet = new HashSet<Token<?>>(); + tokenSet.add(attempt1.getApplicationToken()); + tokenSet.add(attempt1.getClientToken()); + + // assert application Token is saved + HashSet<Token<?>> savedTokens = new HashSet<Token<?>>(); + savedTokens.addAll(attemptState.getAppAttemptTokens().getAllTokens()); + Assert.assertEquals(tokenSet, savedTokens); + + // start new RM + MockRM rm2 = new TestSecurityMockRM(conf, memStore); + rm2.start(); + + RMApp loadedApp1 = + rm2.getRMContext().getRMApps().get(app1.getApplicationId()); + RMAppAttempt loadedAttempt1 = loadedApp1.getRMAppAttempt(attemptId1); + + // assert loaded attempt recovered attempt tokens + Assert.assertNotNull(loadedAttempt1); + savedTokens.clear(); + savedTokens.add(loadedAttempt1.getApplicationToken()); + savedTokens.add(loadedAttempt1.getClientToken()); + Assert.assertEquals(tokenSet, savedTokens); + + // assert clientToken is recovered back to api-versioned clientToken + Assert.assertEquals(attempt1.getClientToken(), + loadedAttempt1.getClientToken()); + + // Not testing ApplicationTokenSecretManager has the password populated back, + // that is needed in work-preserving restart + + rm1.stop(); + rm2.stop(); + } + + class TestSecurityMockRM extends MockRM { - public MyMockRM(Configuration conf, RMStateStore store) { + public TestSecurityMockRM(Configuration conf, RMStateStore store) { super(conf, store); }
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java?rev=1480440&r1=1480439&r2=1480440&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java Wed May 8 20:20:56 2013 @@ -18,14 +18,19 @@ package org.apache.hadoop.yarn.server.resourcemanager.recovery; -import static org.junit.Assert.*; -import static org.mockito.Mockito.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; +import java.util.List; import java.util.Map; -import org.junit.Test; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -34,6 +39,8 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -44,13 +51,18 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier; +import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptStoredEvent; +import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; +import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.junit.Test; public class TestRMStateStore { @@ -141,7 +153,7 @@ public class TestRMStateStore { ApplicationAttemptId attemptId = ConverterUtils.toApplicationAttemptId( "appattempt_1352994193343_0003_000001"); storeAttempt(testStore, attemptId, - "container_1352994193343_0003_01_000001", dispatcher); + "container_1352994193343_0003_01_000001", null, null, dispatcher); } @Override @@ -186,14 +198,17 @@ public class TestRMStateStore { } ContainerId storeAttempt(RMStateStore store, ApplicationAttemptId attemptId, - String containerIdStr, TestDispatcher dispatcher) - throws Exception { + String containerIdStr, Token<ApplicationTokenIdentifier> appToken, + Token<ClientTokenIdentifier> clientToken, TestDispatcher dispatcher) + throws Exception { Container container = new ContainerPBImpl(); container.setId(ConverterUtils.toContainerId(containerIdStr)); RMAppAttempt mockAttempt = mock(RMAppAttempt.class); when(mockAttempt.getAppAttemptId()).thenReturn(attemptId); when(mockAttempt.getMasterContainer()).thenReturn(container); + when(mockAttempt.getApplicationToken()).thenReturn(appToken); + when(mockAttempt.getClientToken()).thenReturn(clientToken); dispatcher.attemptId = attemptId; dispatcher.storedException = null; store.storeApplicationAttempt(mockAttempt); @@ -201,30 +216,58 @@ public class TestRMStateStore { return container.getId(); } + @SuppressWarnings("unchecked") void testRMStateStore(RMStateStoreHelper stateStoreHelper) throws Exception { long submitTime = System.currentTimeMillis(); + Configuration conf = new YarnConfiguration(); RMStateStore store = stateStoreHelper.getRMStateStore(); TestDispatcher dispatcher = new TestDispatcher(); store.setDispatcher(dispatcher); + ApplicationTokenSecretManager appTokenMgr = + new ApplicationTokenSecretManager(conf); + ClientToAMTokenSecretManagerInRM clientTokenMgr = + new ClientToAMTokenSecretManagerInRM(); + ApplicationAttemptId attemptId1 = ConverterUtils .toApplicationAttemptId("appattempt_1352994193343_0001_000001"); ApplicationId appId1 = attemptId1.getApplicationId(); storeApp(store, appId1, submitTime); + + // create application token1 for attempt1 + List<Token<?>> appAttemptToken1 = + generateTokens(attemptId1, appTokenMgr, clientTokenMgr, conf); + HashSet<Token<?>> attemptTokenSet1 = new HashSet<Token<?>>(); + attemptTokenSet1.addAll(appAttemptToken1); + ContainerId containerId1 = storeAttempt(store, attemptId1, - "container_1352994193343_0001_01_000001", dispatcher); + "container_1352994193343_0001_01_000001", + (Token<ApplicationTokenIdentifier>) (appAttemptToken1.get(0)), + (Token<ClientTokenIdentifier>)(appAttemptToken1.get(1)), + dispatcher); + String appAttemptIdStr2 = "appattempt_1352994193343_0001_000002"; ApplicationAttemptId attemptId2 = - ConverterUtils.toApplicationAttemptId(appAttemptIdStr2); + ConverterUtils.toApplicationAttemptId(appAttemptIdStr2); + + // create application token2 for attempt2 + List<Token<?>> appAttemptToken2 = + generateTokens(attemptId2, appTokenMgr, clientTokenMgr, conf); + HashSet<Token<?>> attemptTokenSet2 = new HashSet<Token<?>>(); + attemptTokenSet2.addAll(appAttemptToken2); + ContainerId containerId2 = storeAttempt(store, attemptId2, - "container_1352994193343_0001_02_000001", dispatcher); + "container_1352994193343_0001_02_000001", + (Token<ApplicationTokenIdentifier>) (appAttemptToken2.get(0)), + (Token<ClientTokenIdentifier>)(appAttemptToken2.get(1)), + dispatcher); ApplicationAttemptId attemptIdRemoved = ConverterUtils .toApplicationAttemptId("appattempt_1352994193343_0002_000001"); ApplicationId appIdRemoved = attemptIdRemoved.getApplicationId(); storeApp(store, appIdRemoved, submitTime); storeAttempt(store, attemptIdRemoved, - "container_1352994193343_0002_01_000001", dispatcher); + "container_1352994193343_0002_01_000001", null, null, dispatcher); RMApp mockRemovedApp = mock(RMApp.class); HashMap<ApplicationAttemptId, RMAppAttempt> attempts = @@ -268,12 +311,21 @@ public class TestRMStateStore { assertEquals(attemptId1, attemptState.getAttemptId()); // attempt1 container is loaded correctly assertEquals(containerId1, attemptState.getMasterContainer().getId()); + // attempt1 applicationToken is loaded correctly + HashSet<Token<?>> savedTokens = new HashSet<Token<?>>(); + savedTokens.addAll(attemptState.getAppAttemptTokens().getAllTokens()); + assertEquals(attemptTokenSet1, savedTokens); + attemptState = appState.getAttempt(attemptId2); // attempt2 is loaded correctly assertNotNull(attemptState); assertEquals(attemptId2, attemptState.getAttemptId()); // attempt2 container is loaded correctly assertEquals(containerId2, attemptState.getMasterContainer().getId()); + // attempt2 applicationToken is loaded correctly + savedTokens.clear(); + savedTokens.addAll(attemptState.getAppAttemptTokens().getAllTokens()); + assertEquals(attemptTokenSet2, savedTokens); // assert store is in expected state after everything is cleaned assertTrue(stateStoreHelper.isFinalStateValid()); @@ -281,4 +333,23 @@ public class TestRMStateStore { store.close(); } + private List<Token<?>> generateTokens(ApplicationAttemptId attemptId, + ApplicationTokenSecretManager appTokenMgr, + ClientToAMTokenSecretManagerInRM clientTokenMgr, Configuration conf) { + ApplicationTokenIdentifier appTokenId = + new ApplicationTokenIdentifier(attemptId); + Token<ApplicationTokenIdentifier> appToken = + new Token<ApplicationTokenIdentifier>(appTokenId, appTokenMgr); + appToken.setService(new Text("appToken service")); + + ClientTokenIdentifier clientTokenId = new ClientTokenIdentifier(attemptId); + clientTokenMgr.registerApplication(attemptId); + Token<ClientTokenIdentifier> clientToken = + new Token<ClientTokenIdentifier>(clientTokenId, clientTokenMgr); + clientToken.setService(new Text("clientToken service")); + List<Token<?>> tokenPair = new ArrayList<Token<?>>(); + tokenPair.add(0, appToken); + tokenPair.add(1, clientToken); + return tokenPair; + } } Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java?rev=1480440&r1=1480439&r2=1480440&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java Wed May 8 20:20:56 2013 @@ -537,6 +537,9 @@ public class TestContainerManagerSecurit "Unauthorized request to start container. " + "\nExpected containerId: " + tokenId.getContainerID() + " Found: " + newContainerId.toString())); + } catch (IOException e) { + LOG.info("Got IOException: ",e); + fail("IOException is not expected."); } } @@ -563,6 +566,9 @@ public class TestContainerManagerSecurit Assert.assertTrue(e.getMessage().contains( "\nExpected resource " + tokenId.getResource().toString() + " but found " + container.getResource().toString())); + } catch (IOException e) { + LOG.info("Got IOException: ",e); + fail("IOException is not expected."); } } @@ -591,6 +597,9 @@ public class TestContainerManagerSecurit Assert.assertTrue(e.getMessage().contains( "Expected user-name " + tokenId.getApplicationSubmitter() + " but found " + context.getUser())); + } catch (IOException e) { + LOG.info("Got IOException: ",e); + fail("IOException is not expected."); } } Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1480440&r1=1480439&r2=1480440&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm Wed May 8 20:20:56 2013 @@ -124,7 +124,7 @@ Hadoop MapReduce Next Generation - Fair * The smallest container size the scheduler can allocate, in MB of memory. - * <<<yarn.scheduler.fair.minimum-allocation-mb>>> + * <<<yarn.scheduler.fair.maximum-allocation-mb>>> * The largest container the scheduler can allocate, in MB of memory. @@ -177,7 +177,7 @@ Hadoop MapReduce Next Generation - Fair Allocation file format - The allocation file must be in XML format. The format contains three types of + The allocation file must be in XML format. The format contains four types of elements: * <<Queue elements>>, which represent queues. Each may contain the following
