Author: vinodkv
Date: Tue Nov 19 00:50:34 2013
New Revision: 1543269
URL: http://svn.apache.org/r1543269
Log:
YARN-709. Added tests to verify validity of delegation tokens and logging of
appsummary after RM restart. Contributed by Jian He.
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1543269&r1=1543268&r2=1543269&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Tue Nov 19 00:50:34 2013
@@ -100,6 +100,9 @@ Release 2.3.0 - UNRELEASED
YARN-1222. Make improvements in ZKRMStateStore for fencing (Karthik
Kambatla via bikas)
+ YARN-709. Added tests to verify validity of delegation tokens and logging
of
+ appsummary after RM restart. (Jian He via vinodkv)
+
OPTIMIZATIONS
BUG FIXES
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java?rev=1543269&r1=1543268&r2=1543269&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
Tue Nov 19 00:50:34 2013
@@ -54,6 +54,8 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* This class manages the list of applications for the resource manager.
*/
@@ -165,6 +167,11 @@ public class RMAppManager implements Eve
}
}
+ @VisibleForTesting
+ public void logApplicationSummary(ApplicationId appId) {
+ ApplicationSummary.logAppSummary(rmContext.getRMApps().get(appId));
+ }
+
protected synchronized void setCompletedAppsMax(int max) {
this.completedAppsMax = max;
}
@@ -351,8 +358,7 @@ public class RMAppManager implements Eve
case APP_COMPLETED:
{
finishApplication(applicationId);
- ApplicationSummary.logAppSummary(
- rmContext.getRMApps().get(applicationId));
+ logApplicationSummary(applicationId);
checkAppNumCompletedLimit();
}
break;
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java?rev=1543269&r1=1543268&r2=1543269&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
Tue Nov 19 00:50:34 2013
@@ -421,6 +421,10 @@ public class MockRM extends ResourceMana
return this.clientToAMSecretManager;
}
+ public RMAppManager getRMAppManager() {
+ return this.rmAppManager;
+ }
+
@Override
protected void startWepApp() {
// override to disable webapp
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java?rev=1543269&r1=1543268&r2=1543269&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
Tue Nov 19 00:50:34 2013
@@ -18,6 +18,11 @@
package org.apache.hadoop.yarn.server.resourcemanager;
+import static org.mockito.Matchers.isA;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
@@ -577,7 +582,14 @@ public class TestRMRestart {
rm1.waitForState(am2.getApplicationAttemptId(), RMAppAttemptState.KILLED);
// restart rm
- MockRM rm2 = new MockRM(conf, memStore);
+
+ MockRM rm2 = new MockRM(conf, memStore) {
+ @Override
+ protected RMAppManager createRMAppManager() {
+ return spy(super.createRMAppManager());
+ }
+ };
+
rm2.start();
GetApplicationsRequest request1 =
@@ -620,6 +632,10 @@ public class TestRMRestart {
rm2.getClientRMService().getApplications(request2);
List<ApplicationReport> appList2 = response2.getApplicationList();
Assert.assertTrue(3 == appList2.size());
+
+ // check application summary is logged for the completed apps after RM
restart.
+ verify(rm2.getRMAppManager(), times(3)).logApplicationSummary(
+ isA(ApplicationId.class));
}
private MockAM launchAM(RMApp app, MockRM rm, MockNM nm)
@@ -920,7 +936,6 @@ public class TestRMRestart {
@Test
public void testRMDelegationTokenRestoredOnRMRestart() throws Exception {
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
-
conf.set(
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"kerberos");
@@ -1063,6 +1078,43 @@ public class TestRMRestart {
rm2.stop();
}
+ // This is to test submit an application to the new RM with the old
delegation
+ // token got from previous RM.
+ @Test
+ public void testAppSubmissionWithOldDelegationTokenAfterRMRestart()
+ throws Exception {
+ conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ "kerberos");
+ conf.set(YarnConfiguration.RM_ADDRESS, "localhost:8032");
+ UserGroupInformation.setConfiguration(conf);
+ MemoryRMStateStore memStore = new MemoryRMStateStore();
+ memStore.init(conf);
+
+ MockRM rm1 = new TestSecurityMockRM(conf, memStore);
+ rm1.start();
+
+ GetDelegationTokenRequest request1 =
+ GetDelegationTokenRequest.newInstance("renewer1");
+ UserGroupInformation.getCurrentUser().setAuthenticationMethod(
+ AuthMethod.KERBEROS);
+ GetDelegationTokenResponse response1 =
+ rm1.getClientRMService().getDelegationToken(request1);
+ Token<RMDelegationTokenIdentifier> token1 =
+ ConverterUtils.convertFromYarn(response1.getRMDelegationToken(),
rmAddr);
+
+ // start new RM
+ MockRM rm2 = new TestSecurityMockRM(conf, memStore);
+ rm2.start();
+
+ // submit an app with the old delegation token got from previous RM.
+ Credentials ts = new Credentials();
+ ts.addToken(token1.getService(), token1);
+ RMApp app = rm2.submitApp(200, "name", "user",
+ new HashMap<ApplicationAccessType, String>(), false, "default", 1, ts);
+ rm2.waitForState(app.getApplicationId(), RMAppState.ACCEPTED);
+ }
+
@Test
public void testRMStateStoreDispatcherDrainedOnRMStop() throws Exception {
MemoryRMStateStore memStore = new MemoryRMStateStore() {
