Author: szetszwo
Date: Tue Jul 29 00:49:14 2014
New Revision: 1614234
URL: http://svn.apache.org/r1614234
Log:
Merge r1609845 through r1614231 from trunk.
Added:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/
- copied from r1614231,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMContainerTokenSecretManager.java
- copied unchanged from r1614231,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMContainerTokenSecretManager.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/AMRMTokenSecretManagerState.java
- copied unchanged from r1614231,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/AMRMTokenSecretManagerState.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/AMRMTokenSecretManagerStatePBImpl.java
- copied unchanged from r1614231,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/AMRMTokenSecretManagerStatePBImpl.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMAuthenticationHandler.java
- copied unchanged from r1614231,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMAuthenticationHandler.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/
- copied from r1614231,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokenAuthentication.java
- copied unchanged from r1614231,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokenAuthentication.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebappAuthentication.java
- copied unchanged from r1614231,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebappAuthentication.java
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceOnHA.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMTokenSecretManagerInNM.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/CHANGES.txt?rev=1614234&r1=1614233&r2=1614234&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/CHANGES.txt Tue Jul 29
00:49:14 2014
@@ -62,6 +62,15 @@ Release 2.6.0 - UNRELEASED
YARN-2295. Refactored DistributedShell to use public APIs of protocol
records.
(Li Lu via jianhe)
+ YARN-1342. Recover container tokens upon nodemanager restart. (Jason Lowe
via
+ devaraj)
+
+ YARN-2214. FairScheduler: preemptContainerPreCheck() in FSParentQueue
delays
+ convergence towards fairness. (Ashwin Shankar via kasha)
+
+ YARN-2211. Persist AMRMToken master key in RMStateStore for RM recovery.
+ (Xuan Gong via jianhe)
+
OPTIMIZATIONS
BUG FIXES
@@ -94,6 +103,11 @@ Release 2.6.0 - UNRELEASED
YARN-2313. Livelock can occur in FairScheduler when there are lots of
running apps (Tsuyoshi Ozawa via Sandy Ryza)
+ YARN-2147. client lacks delegation token exception details when
+ application submit fails (Chen He via jlowe)
+
+ YARN-1796. container-executor shouldn't require o-r permissions (atm)
+
Release 2.5.0 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -133,6 +147,9 @@ Release 2.5.0 - UNRELEASED
YARN-2233. Implemented ResourceManager web-services to create, renew and
cancel delegation tokens. (Varun Vasudev via vinodkv)
+ YARN-2247. Made RM web services authenticate users via kerberos and
delegation
+ token. (Varun Vasudev via zjshen)
+
IMPROVEMENTS
YARN-1479. Invalid NaN values in Hadoop REST API JSON response (Chen He via
@@ -300,6 +317,9 @@ Release 2.5.0 - UNRELEASED
YARN-1408 Preemption caused Invalid State Event: ACQUIRED at KILLED and
caused a task timeout for 30mins. (Sunil G via mayank)
+ YARN-2300. Improved the documentation of the sample requests for RM REST
API -
+ submitting an app. (Varun Vasudev via zjshen)
+
OPTIMIZATIONS
BUG FIXES
@@ -427,6 +447,11 @@ Release 2.5.0 - UNRELEASED
YARN-2319. Made the MiniKdc instance start/close before/after the class of
TestRMWebServicesDelegationTokens. (Wenwu Peng via zjshen)
+ YARN-2335. Annotate all hadoop-sls APIs as @Private. (Wei Yan via kasha)
+
+ YARN-1726. ResourceSchedulerWrapper broken due to AbstractYarnScheduler.
+ (Wei Yan via kasha)
+
Release 2.4.1 - 2014-06-23
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1614234&r1=1614233&r2=1614234&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
Tue Jul 29 00:49:14 2014
@@ -263,6 +263,17 @@ public class YarnConfiguration extends C
public static final String RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY =
RM_PREFIX + "webapp.spnego-keytab-file";
+ /**
+ * Flag to enable override of the default kerberos authentication filter with
+ * the RM authentication filter to allow authentication using delegation
+ * tokens(fallback to kerberos if the tokens are missing). Only applicable
+ * when the http authentication type is kerberos.
+ */
+ public static final String RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER = RM_PREFIX
+ + "webapp.delegation-token-auth-filter.enabled";
+ public static final boolean DEFAULT_RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER =
+ true;
+
/** How long to wait until a container is considered dead.*/
public static final String RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS =
RM_PREFIX + "rm.container-allocation.expiry-interval-ms";
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java?rev=1614234&r1=1614233&r2=1614234&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
Tue Jul 29 00:49:14 2014
@@ -267,6 +267,7 @@ public abstract class ProtocolHATestBase
protected void startHACluster(int numOfNMs, boolean overrideClientRMService,
boolean overrideRTS, boolean overrideApplicationMasterService)
throws Exception {
+ conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
cluster =
new MiniYARNClusterForHATesting(TestRMFailover.class.getName(), 2,
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceOnHA.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceOnHA.java?rev=1614234&r1=1614233&r2=1614234&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceOnHA.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceOnHA.java
Tue Jul 29 00:49:14 2014
@@ -54,11 +54,9 @@ public class TestApplicationMasterServic
amClient = ClientRMProxy
.createRMProxy(this.conf, ApplicationMasterProtocol.class);
- AMRMTokenIdentifier id =
- new AMRMTokenIdentifier(attemptId);
Token<AMRMTokenIdentifier> appToken =
- new Token<AMRMTokenIdentifier>(id, this.cluster.getResourceManager()
- .getRMContext().getAMRMTokenSecretManager());
+ this.cluster.getResourceManager().getRMContext()
+ .getAMRMTokenSecretManager().createAndGetAMRMToken(attemptId);
appToken.setService(new Text("appToken service"));
UserGroupInformation.setLoginUser(UserGroupInformation
.createRemoteUser(UserGroupInformation.getCurrentUser()
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1614234&r1=1614233&r2=1614234&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
Tue Jul 29 00:49:14 2014
@@ -195,6 +195,15 @@
</property>
<property>
+ <description>Flag to enable override of the default kerberos authentication
+ filter with the RM authentication filter to allow authentication using
+ delegation tokens(fallback to kerberos if the tokens are missing). Only
+ applicable when the http authentication type is kerberos.</description>
+
<name>yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
<description>How long to wait until a node manager is considered
dead.</description>
<name>yarn.nm.liveness-monitor.expiry-interval-ms</name>
<value>600000</value>
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java?rev=1614234&r1=1614233&r2=1614234&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java
Tue Jul 29 00:49:14 2014
@@ -43,7 +43,7 @@ public class BaseContainerTokenSecretMan
private static Log LOG = LogFactory
.getLog(BaseContainerTokenSecretManager.class);
- private int serialNo = new SecureRandom().nextInt();
+ protected int serialNo = new SecureRandom().nextInt();
protected final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
protected final Lock readLock = readWriteLock.readLock();
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java?rev=1614234&r1=1614233&r2=1614234&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
Tue Jul 29 00:49:14 2014
@@ -173,8 +173,8 @@ public class NodeManager extends Composi
NMContainerTokenSecretManager containerTokenSecretManager)
throws IOException {
if (nmStore.canRecover()) {
- nmTokenSecretManager.recover(nmStore.loadNMTokenState());
- // TODO: recover containerTokenSecretManager
+ nmTokenSecretManager.recover();
+ containerTokenSecretManager.recover();
}
}
@@ -190,7 +190,7 @@ public class NodeManager extends Composi
initAndStartRecoveryStore(conf);
NMContainerTokenSecretManager containerTokenSecretManager =
- new NMContainerTokenSecretManager(conf);
+ new NMContainerTokenSecretManager(conf, nmStore);
NMTokenSecretManagerInNM nmTokenSecretManager =
new NMTokenSecretManagerInNM(nmStore);
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java?rev=1614234&r1=1614233&r2=1614234&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
Tue Jul 29 00:49:14 2014
@@ -37,6 +37,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto;
@@ -90,6 +91,12 @@ public class NMLeveldbStateStoreService
NM_TOKENS_KEY_PREFIX + CURRENT_MASTER_KEY_SUFFIX;
private static final String NM_TOKENS_PREV_MASTER_KEY =
NM_TOKENS_KEY_PREFIX + PREV_MASTER_KEY_SUFFIX;
+ private static final String CONTAINER_TOKENS_KEY_PREFIX =
+ "ContainerTokens/";
+ private static final String CONTAINER_TOKENS_CURRENT_MASTER_KEY =
+ CONTAINER_TOKENS_KEY_PREFIX + CURRENT_MASTER_KEY_SUFFIX;
+ private static final String CONTAINER_TOKENS_PREV_MASTER_KEY =
+ CONTAINER_TOKENS_KEY_PREFIX + PREV_MASTER_KEY_SUFFIX;
private DB db;
@@ -141,7 +148,7 @@ public class NMLeveldbStateStoreService
key.substring(0, userEndPos+1)));
}
} catch (DBException e) {
- throw new IOException(e.getMessage(), e);
+ throw new IOException(e);
} finally {
if (iter != null) {
iter.close();
@@ -260,7 +267,7 @@ public class NMLeveldbStateStoreService
try {
db.put(bytes(key), proto.toByteArray());
} catch (DBException e) {
- throw new IOException(e.getMessage(), e);
+ throw new IOException(e);
}
}
@@ -283,7 +290,7 @@ public class NMLeveldbStateStoreService
batch.close();
}
} catch (DBException e) {
- throw new IOException(e.getMessage(), e);
+ throw new IOException(e);
}
}
@@ -306,7 +313,7 @@ public class NMLeveldbStateStoreService
batch.close();
}
} catch (DBException e) {
- throw new IOException(e.getMessage(), e);
+ throw new IOException(e);
}
}
@@ -355,7 +362,7 @@ public class NMLeveldbStateStoreService
DeletionServiceDeleteTaskProto.parseFrom(entry.getValue()));
}
} catch (DBException e) {
- throw new IOException(e.getMessage(), e);
+ throw new IOException(e);
} finally {
if (iter != null) {
iter.close();
@@ -371,7 +378,7 @@ public class NMLeveldbStateStoreService
try {
db.put(bytes(key), taskProto.toByteArray());
} catch (DBException e) {
- throw new IOException(e.getMessage(), e);
+ throw new IOException(e);
}
}
@@ -381,14 +388,14 @@ public class NMLeveldbStateStoreService
try {
db.delete(bytes(key));
} catch (DBException e) {
- throw new IOException(e.getMessage(), e);
+ throw new IOException(e);
}
}
@Override
- public RecoveredNMTokenState loadNMTokenState() throws IOException {
- RecoveredNMTokenState state = new RecoveredNMTokenState();
+ public RecoveredNMTokensState loadNMTokensState() throws IOException {
+ RecoveredNMTokensState state = new RecoveredNMTokensState();
state.applicationMasterKeys =
new HashMap<ApplicationAttemptId, MasterKey>();
LeveldbIterator iter = null;
@@ -420,7 +427,7 @@ public class NMLeveldbStateStoreService
}
}
} catch (DBException e) {
- throw new IOException(e.getMessage(), e);
+ throw new IOException(e);
} finally {
if (iter != null) {
iter.close();
@@ -454,7 +461,7 @@ public class NMLeveldbStateStoreService
try {
db.delete(bytes(key));
} catch (DBException e) {
- throw new IOException(e.getMessage(), e);
+ throw new IOException(e);
}
}
@@ -468,7 +475,91 @@ public class NMLeveldbStateStoreService
try {
db.put(bytes(dbKey), pb.getProto().toByteArray());
} catch (DBException e) {
- throw new IOException(e.getMessage(), e);
+ throw new IOException(e);
+ }
+ }
+
+
+ @Override
+ public RecoveredContainerTokensState loadContainerTokensState()
+ throws IOException {
+ RecoveredContainerTokensState state = new RecoveredContainerTokensState();
+ state.activeTokens = new HashMap<ContainerId, Long>();
+ LeveldbIterator iter = null;
+ try {
+ iter = new LeveldbIterator(db);
+ iter.seek(bytes(CONTAINER_TOKENS_KEY_PREFIX));
+ final int containerTokensKeyPrefixLength =
+ CONTAINER_TOKENS_KEY_PREFIX.length();
+ while (iter.hasNext()) {
+ Entry<byte[], byte[]> entry = iter.next();
+ String fullKey = asString(entry.getKey());
+ if (!fullKey.startsWith(CONTAINER_TOKENS_KEY_PREFIX)) {
+ break;
+ }
+ String key = fullKey.substring(containerTokensKeyPrefixLength);
+ if (key.equals(CURRENT_MASTER_KEY_SUFFIX)) {
+ state.currentMasterKey = parseMasterKey(entry.getValue());
+ } else if (key.equals(PREV_MASTER_KEY_SUFFIX)) {
+ state.previousMasterKey = parseMasterKey(entry.getValue());
+ } else if (key.startsWith(ConverterUtils.CONTAINER_PREFIX)) {
+ loadContainerToken(state, fullKey, key, entry.getValue());
+ }
+ }
+ } catch (DBException e) {
+ throw new IOException(e);
+ } finally {
+ if (iter != null) {
+ iter.close();
+ }
+ }
+ return state;
+ }
+
+ private static void loadContainerToken(RecoveredContainerTokensState state,
+ String key, String containerIdStr, byte[] value) throws IOException {
+ ContainerId containerId;
+ Long expTime;
+ try {
+ containerId = ConverterUtils.toContainerId(containerIdStr);
+ expTime = Long.parseLong(asString(value));
+ } catch (IllegalArgumentException e) {
+ throw new IOException("Bad container token state for " + key, e);
+ }
+ state.activeTokens.put(containerId, expTime);
+ }
+
+ @Override
+ public void storeContainerTokenCurrentMasterKey(MasterKey key)
+ throws IOException {
+ storeMasterKey(CONTAINER_TOKENS_CURRENT_MASTER_KEY, key);
+ }
+
+ @Override
+ public void storeContainerTokenPreviousMasterKey(MasterKey key)
+ throws IOException {
+ storeMasterKey(CONTAINER_TOKENS_PREV_MASTER_KEY, key);
+ }
+
+ @Override
+ public void storeContainerToken(ContainerId containerId, Long expTime)
+ throws IOException {
+ String key = CONTAINER_TOKENS_KEY_PREFIX + containerId;
+ try {
+ db.put(bytes(key), bytes(expTime.toString()));
+ } catch (DBException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public void removeContainerToken(ContainerId containerId)
+ throws IOException {
+ String key = CONTAINER_TOKENS_KEY_PREFIX + containerId;
+ try {
+ db.delete(bytes(key));
+ } catch (DBException e) {
+ throw new IOException(e);
}
}
@@ -554,7 +645,7 @@ public class NMLeveldbStateStoreService
try {
db.put(bytes(key), data);
} catch (DBException e) {
- throw new IOException(e.getMessage(), e);
+ throw new IOException(e);
}
}
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java?rev=1614234&r1=1614233&r2=1614234&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java
Tue Jul 29 00:49:14 2014
@@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto;
import
org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto;
import
org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto;
@@ -80,7 +81,7 @@ public class NMNullStateStoreService ext
}
@Override
- public RecoveredNMTokenState loadNMTokenState() throws IOException {
+ public RecoveredNMTokensState loadNMTokensState() throws IOException {
throw new UnsupportedOperationException(
"Recovery not supported by this state store");
}
@@ -106,6 +107,33 @@ public class NMNullStateStoreService ext
}
@Override
+ public RecoveredContainerTokensState loadContainerTokensState()
+ throws IOException {
+ throw new UnsupportedOperationException(
+ "Recovery not supported by this state store");
+ }
+
+ @Override
+ public void storeContainerTokenCurrentMasterKey(MasterKey key)
+ throws IOException {
+ }
+
+ @Override
+ public void storeContainerTokenPreviousMasterKey(MasterKey key)
+ throws IOException {
+ }
+
+ @Override
+ public void storeContainerToken(ContainerId containerId,
+ Long expirationTime) throws IOException {
+ }
+
+ @Override
+ public void removeContainerToken(ContainerId containerId)
+ throws IOException {
+ }
+
+ @Override
protected void initStorage(Configuration conf) throws IOException {
}
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java?rev=1614234&r1=1614233&r2=1614234&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java
Tue Jul 29 00:49:14 2014
@@ -31,6 +31,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto;
import
org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto;
import
org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto;
@@ -102,7 +103,7 @@ public abstract class NMStateStoreServic
}
}
- public static class RecoveredNMTokenState {
+ public static class RecoveredNMTokensState {
MasterKey currentMasterKey;
MasterKey previousMasterKey;
Map<ApplicationAttemptId, MasterKey> applicationMasterKeys;
@@ -120,6 +121,24 @@ public abstract class NMStateStoreServic
}
}
+ public static class RecoveredContainerTokensState {
+ MasterKey currentMasterKey;
+ MasterKey previousMasterKey;
+ Map<ContainerId, Long> activeTokens;
+
+ public MasterKey getCurrentMasterKey() {
+ return currentMasterKey;
+ }
+
+ public MasterKey getPreviousMasterKey() {
+ return previousMasterKey;
+ }
+
+ public Map<ContainerId, Long> getActiveTokens() {
+ return activeTokens;
+ }
+ }
+
/** Initialize the state storage */
@Override
public void serviceInit(Configuration conf) throws IOException {
@@ -193,7 +212,8 @@ public abstract class NMStateStoreServic
public abstract void removeDeletionTask(int taskId) throws IOException;
- public abstract RecoveredNMTokenState loadNMTokenState() throws IOException;
+ public abstract RecoveredNMTokensState loadNMTokensState()
+ throws IOException;
public abstract void storeNMTokenCurrentMasterKey(MasterKey key)
throws IOException;
@@ -208,6 +228,22 @@ public abstract class NMStateStoreServic
ApplicationAttemptId attempt) throws IOException;
+ public abstract RecoveredContainerTokensState loadContainerTokensState()
+ throws IOException;
+
+ public abstract void storeContainerTokenCurrentMasterKey(MasterKey key)
+ throws IOException;
+
+ public abstract void storeContainerTokenPreviousMasterKey(MasterKey key)
+ throws IOException;
+
+ public abstract void storeContainerToken(ContainerId containerId,
+ Long expirationTime) throws IOException;
+
+ public abstract void removeContainerToken(ContainerId containerId)
+ throws IOException;
+
+
protected abstract void initStorage(Configuration conf) throws IOException;
protected abstract void startStorage() throws IOException;
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java?rev=1614234&r1=1614233&r2=1614234&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java
Tue Jul 29 00:49:14 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.nodemanager.security;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -33,6 +34,9 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
+import
org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
+import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
+import
org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredContainerTokensState;
import org.apache.hadoop.yarn.server.security.BaseContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.security.MasterKeyData;
@@ -49,14 +53,74 @@ public class NMContainerTokenSecretManag
private MasterKeyData previousMasterKey;
private final TreeMap<Long, List<ContainerId>>
recentlyStartedContainerTracker;
-
+ private final NMStateStoreService stateStore;
private String nodeHostAddr;
public NMContainerTokenSecretManager(Configuration conf) {
+ this(conf, new NMNullStateStoreService());
+ }
+
+ public NMContainerTokenSecretManager(Configuration conf,
+ NMStateStoreService stateStore) {
super(conf);
recentlyStartedContainerTracker =
new TreeMap<Long, List<ContainerId>>();
+ this.stateStore = stateStore;
+ }
+
+ public synchronized void recover()
+ throws IOException {
+ RecoveredContainerTokensState state =
+ stateStore.loadContainerTokensState();
+ MasterKey key = state.getCurrentMasterKey();
+ if (key != null) {
+ super.currentMasterKey =
+ new MasterKeyData(key, createSecretKey(key.getBytes().array()));
+ }
+
+ key = state.getPreviousMasterKey();
+ if (key != null) {
+ previousMasterKey =
+ new MasterKeyData(key, createSecretKey(key.getBytes().array()));
+ }
+
+ // restore the serial number from the current master key
+ if (super.currentMasterKey != null) {
+ super.serialNo = super.currentMasterKey.getMasterKey().getKeyId() + 1;
+ }
+
+ for (Entry<ContainerId, Long> entry : state.getActiveTokens().entrySet()) {
+ ContainerId containerId = entry.getKey();
+ Long expTime = entry.getValue();
+ List<ContainerId> containerList =
+ recentlyStartedContainerTracker.get(expTime);
+ if (containerList == null) {
+ containerList = new ArrayList<ContainerId>();
+ recentlyStartedContainerTracker.put(expTime, containerList);
+ }
+ if (!containerList.contains(containerId)) {
+ containerList.add(containerId);
+ }
+ }
+ }
+
+ private void updateCurrentMasterKey(MasterKeyData key) {
+ super.currentMasterKey = key;
+ try {
+ stateStore.storeContainerTokenCurrentMasterKey(key.getMasterKey());
+ } catch (IOException e) {
+ LOG.error("Unable to update current master key in state store", e);
+ }
+ }
+
+ private void updatePreviousMasterKey(MasterKeyData key) {
+ previousMasterKey = key;
+ try {
+ stateStore.storeContainerTokenPreviousMasterKey(key.getMasterKey());
+ } catch (IOException e) {
+ LOG.error("Unable to update previous master key in state store", e);
+ }
}
/**
@@ -68,21 +132,16 @@ public class NMContainerTokenSecretManag
*/
@Private
public synchronized void setMasterKey(MasterKey masterKeyRecord) {
- LOG.info("Rolling master-key for container-tokens, got key with id "
- + masterKeyRecord.getKeyId());
- if (super.currentMasterKey == null) {
- super.currentMasterKey =
- new MasterKeyData(masterKeyRecord, createSecretKey(masterKeyRecord
- .getBytes().array()));
- } else {
- if (super.currentMasterKey.getMasterKey().getKeyId() != masterKeyRecord
- .getKeyId()) {
- // Update keys only if the key has changed.
- this.previousMasterKey = super.currentMasterKey;
- super.currentMasterKey =
- new MasterKeyData(masterKeyRecord, createSecretKey(masterKeyRecord
- .getBytes().array()));
+ // Update keys only if the key has changed.
+ if (super.currentMasterKey == null || super.currentMasterKey.getMasterKey()
+ .getKeyId() != masterKeyRecord.getKeyId()) {
+ LOG.info("Rolling master-key for container-tokens, got key with id "
+ + masterKeyRecord.getKeyId());
+ if (super.currentMasterKey != null) {
+ updatePreviousMasterKey(super.currentMasterKey);
}
+ updateCurrentMasterKey(new MasterKeyData(masterKeyRecord,
+ createSecretKey(masterKeyRecord.getBytes().array())));
}
}
@@ -137,14 +196,19 @@ public class NMContainerTokenSecretManag
removeAnyContainerTokenIfExpired();
+ ContainerId containerId = tokenId.getContainerID();
Long expTime = tokenId.getExpiryTimeStamp();
// We might have multiple containers with same expiration time.
if (!recentlyStartedContainerTracker.containsKey(expTime)) {
recentlyStartedContainerTracker
.put(expTime, new ArrayList<ContainerId>());
}
- recentlyStartedContainerTracker.get(expTime).add(tokenId.getContainerID());
-
+ recentlyStartedContainerTracker.get(expTime).add(containerId);
+ try {
+ stateStore.storeContainerToken(containerId, expTime);
+ } catch (IOException e) {
+ LOG.error("Unable to store token for container " + containerId, e);
+ }
}
protected synchronized void removeAnyContainerTokenIfExpired() {
@@ -155,6 +219,13 @@ public class NMContainerTokenSecretManag
while (containersI.hasNext()) {
Entry<Long, List<ContainerId>> containerEntry = containersI.next();
if (containerEntry.getKey() < currTime) {
+ for (ContainerId container : containerEntry.getValue()) {
+ try {
+ stateStore.removeContainerToken(container);
+ } catch (IOException e) {
+ LOG.error("Unable to remove token for container " + container, e);
+ }
+ }
containersI.remove();
} else {
break;
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java?rev=1614234&r1=1614233&r2=1614234&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java
Tue Jul 29 00:49:14 2014
@@ -34,7 +34,7 @@ import org.apache.hadoop.yarn.security.N
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import
org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
-import
org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredNMTokenState;
+import
org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredNMTokensState;
import org.apache.hadoop.yarn.server.security.BaseNMTokenSecretManager;
import org.apache.hadoop.yarn.server.security.MasterKeyData;
@@ -64,8 +64,9 @@ public class NMTokenSecretManagerInNM ex
this.stateStore = stateStore;
}
- public synchronized void recover(RecoveredNMTokenState state)
+ public synchronized void recover()
throws IOException {
+ RecoveredNMTokensState state = stateStore.loadNMTokensState();
MasterKey key = state.getCurrentMasterKey();
if (key != null) {
super.currentMasterKey =
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c?rev=1614234&r1=1614233&r2=1614234&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
Tue Jul 29 00:49:14 2014
@@ -111,16 +111,16 @@ int check_executor_permissions(char *exe
return -1;
}
- // check others do not have read/write/execute permissions
- if ((filestat.st_mode & S_IROTH) == S_IROTH || (filestat.st_mode & S_IWOTH)
- == S_IWOTH || (filestat.st_mode & S_IXOTH) == S_IXOTH) {
+ // check others do not have write/execute permissions
+ if ((filestat.st_mode & S_IWOTH) == S_IWOTH ||
+ (filestat.st_mode & S_IXOTH) == S_IXOTH) {
fprintf(LOGFILE,
- "The container-executor binary should not have read or write or"
- " execute for others.\n");
+ "The container-executor binary should not have write or execute "
+ "for others.\n");
return -1;
}
- // Binary should be setuid/setgid executable
+ // Binary should be setuid executable
if ((filestat.st_mode & S_ISUID) == 0) {
fprintf(LOGFILE, "The container-executor binary should be set setuid.\n");
return -1;
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java?rev=1614234&r1=1614233&r2=1614234&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java
Tue Jul 29 00:49:14 2014
@@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto;
import
org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto;
import
org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto;
@@ -36,7 +37,8 @@ import org.apache.hadoop.yarn.server.api
public class NMMemoryStateStoreService extends NMStateStoreService {
private Map<TrackerKey, TrackerState> trackerStates;
private Map<Integer, DeletionServiceDeleteTaskProto> deleteTasks;
- private RecoveredNMTokenState nmTokenState;
+ private RecoveredNMTokensState nmTokenState;
+ private RecoveredContainerTokensState containerTokenState;
public NMMemoryStateStoreService() {
super(NMMemoryStateStoreService.class.getName());
@@ -117,12 +119,13 @@ public class NMMemoryStateStoreService e
@Override
protected void initStorage(Configuration conf) {
- nmTokenState = new RecoveredNMTokenState();
+ nmTokenState = new RecoveredNMTokensState();
nmTokenState.applicationMasterKeys =
new HashMap<ApplicationAttemptId, MasterKey>();
+ containerTokenState = new RecoveredContainerTokensState();
+ containerTokenState.activeTokens = new HashMap<ContainerId, Long>();
trackerStates = new HashMap<TrackerKey, TrackerState>();
deleteTasks = new HashMap<Integer, DeletionServiceDeleteTaskProto>();
-
}
@Override
@@ -157,9 +160,9 @@ public class NMMemoryStateStoreService e
@Override
- public RecoveredNMTokenState loadNMTokenState() throws IOException {
+ public RecoveredNMTokensState loadNMTokensState() throws IOException {
// return a copy so caller can't modify our state
- RecoveredNMTokenState result = new RecoveredNMTokenState();
+ RecoveredNMTokensState result = new RecoveredNMTokensState();
result.currentMasterKey = nmTokenState.currentMasterKey;
result.previousMasterKey = nmTokenState.previousMasterKey;
result.applicationMasterKeys =
@@ -197,6 +200,48 @@ public class NMMemoryStateStoreService e
}
+ @Override
+ public RecoveredContainerTokensState loadContainerTokensState()
+ throws IOException {
+ // return a copy so caller can't modify our state
+ RecoveredContainerTokensState result =
+ new RecoveredContainerTokensState();
+ result.currentMasterKey = containerTokenState.currentMasterKey;
+ result.previousMasterKey = containerTokenState.previousMasterKey;
+ result.activeTokens =
+ new HashMap<ContainerId, Long>(containerTokenState.activeTokens);
+ return result;
+ }
+
+ @Override
+ public void storeContainerTokenCurrentMasterKey(MasterKey key)
+ throws IOException {
+ MasterKeyPBImpl keypb = (MasterKeyPBImpl) key;
+ containerTokenState.currentMasterKey =
+ new MasterKeyPBImpl(keypb.getProto());
+ }
+
+ @Override
+ public void storeContainerTokenPreviousMasterKey(MasterKey key)
+ throws IOException {
+ MasterKeyPBImpl keypb = (MasterKeyPBImpl) key;
+ containerTokenState.previousMasterKey =
+ new MasterKeyPBImpl(keypb.getProto());
+ }
+
+ @Override
+ public void storeContainerToken(ContainerId containerId,
+ Long expirationTime) throws IOException {
+ containerTokenState.activeTokens.put(containerId, expirationTime);
+ }
+
+ @Override
+ public void removeContainerToken(ContainerId containerId)
+ throws IOException {
+ containerTokenState.activeTokens.remove(containerId);
+ }
+
+
private static class TrackerState {
Map<Path, LocalResourceProto> inProgressMap =
new HashMap<Path, LocalResourceProto>();
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java?rev=1614234&r1=1614233&r2=1614234&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
Tue Jul 29 00:49:14 2014
@@ -27,11 +27,13 @@ import java.io.File;
import java.io.IOException;
import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.service.ServiceStateException;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
@@ -42,12 +44,15 @@ import org.apache.hadoop.yarn.proto.Yarn
import
org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import
org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.LocalResourceTrackerState;
+import
org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredContainerTokensState;
import
org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredDeletionServiceState;
import
org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredLocalizationState;
-import
org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredNMTokenState;
+import
org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredNMTokensState;
import
org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredUserResources;
import
org.apache.hadoop.yarn.server.nodemanager.recovery.records.NMDBSchemaVersion;
+import org.apache.hadoop.yarn.server.security.BaseContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.security.BaseNMTokenSecretManager;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.junit.After;
import org.junit.Assert;
@@ -502,7 +507,7 @@ public class TestNMLeveldbStateStoreServ
@Test
public void testNMTokenStorage() throws IOException {
// test empty when no state
- RecoveredNMTokenState state = stateStore.loadNMTokenState();
+ RecoveredNMTokensState state = stateStore.loadNMTokensState();
assertNull(state.getCurrentMasterKey());
assertNull(state.getPreviousMasterKey());
assertTrue(state.getApplicationMasterKeys().isEmpty());
@@ -512,7 +517,7 @@ public class TestNMLeveldbStateStoreServ
MasterKey currentKey = secretMgr.generateKey();
stateStore.storeNMTokenCurrentMasterKey(currentKey);
restartStateStore();
- state = stateStore.loadNMTokenState();
+ state = stateStore.loadNMTokensState();
assertEquals(currentKey, state.getCurrentMasterKey());
assertNull(state.getPreviousMasterKey());
assertTrue(state.getApplicationMasterKeys().isEmpty());
@@ -521,7 +526,7 @@ public class TestNMLeveldbStateStoreServ
MasterKey prevKey = secretMgr.generateKey();
stateStore.storeNMTokenPreviousMasterKey(prevKey);
restartStateStore();
- state = stateStore.loadNMTokenState();
+ state = stateStore.loadNMTokensState();
assertEquals(currentKey, state.getCurrentMasterKey());
assertEquals(prevKey, state.getPreviousMasterKey());
assertTrue(state.getApplicationMasterKeys().isEmpty());
@@ -536,7 +541,7 @@ public class TestNMLeveldbStateStoreServ
MasterKey attemptKey2 = secretMgr.generateKey();
stateStore.storeNMTokenApplicationMasterKey(attempt2, attemptKey2);
restartStateStore();
- state = stateStore.loadNMTokenState();
+ state = stateStore.loadNMTokensState();
assertEquals(currentKey, state.getCurrentMasterKey());
assertEquals(prevKey, state.getPreviousMasterKey());
Map<ApplicationAttemptId, MasterKey> loadedAppKeys =
@@ -558,7 +563,7 @@ public class TestNMLeveldbStateStoreServ
currentKey = secretMgr.generateKey();
stateStore.storeNMTokenCurrentMasterKey(currentKey);
restartStateStore();
- state = stateStore.loadNMTokenState();
+ state = stateStore.loadNMTokensState();
assertEquals(currentKey, state.getCurrentMasterKey());
assertEquals(prevKey, state.getPreviousMasterKey());
loadedAppKeys = state.getApplicationMasterKeys();
@@ -568,10 +573,89 @@ public class TestNMLeveldbStateStoreServ
assertEquals(attemptKey3, loadedAppKeys.get(attempt3));
}
+ @Test
+ public void testContainerTokenStorage() throws IOException {
+ // test empty when no state
+ RecoveredContainerTokensState state =
+ stateStore.loadContainerTokensState();
+ assertNull(state.getCurrentMasterKey());
+ assertNull(state.getPreviousMasterKey());
+ assertTrue(state.getActiveTokens().isEmpty());
+
+ // store a master key and verify recovered
+ ContainerTokenKeyGeneratorForTest keygen =
+ new ContainerTokenKeyGeneratorForTest(new YarnConfiguration());
+ MasterKey currentKey = keygen.generateKey();
+ stateStore.storeContainerTokenCurrentMasterKey(currentKey);
+ restartStateStore();
+ state = stateStore.loadContainerTokensState();
+ assertEquals(currentKey, state.getCurrentMasterKey());
+ assertNull(state.getPreviousMasterKey());
+ assertTrue(state.getActiveTokens().isEmpty());
+
+ // store a previous key and verify recovered
+ MasterKey prevKey = keygen.generateKey();
+ stateStore.storeContainerTokenPreviousMasterKey(prevKey);
+ restartStateStore();
+ state = stateStore.loadContainerTokensState();
+ assertEquals(currentKey, state.getCurrentMasterKey());
+ assertEquals(prevKey, state.getPreviousMasterKey());
+ assertTrue(state.getActiveTokens().isEmpty());
+
+ // store a few container tokens and verify recovered
+ ContainerId cid1 = BuilderUtils.newContainerId(1, 1, 1, 1);
+ Long expTime1 = 1234567890L;
+ ContainerId cid2 = BuilderUtils.newContainerId(2, 2, 2, 2);
+ Long expTime2 = 9876543210L;
+ stateStore.storeContainerToken(cid1, expTime1);
+ stateStore.storeContainerToken(cid2, expTime2);
+ restartStateStore();
+ state = stateStore.loadContainerTokensState();
+ assertEquals(currentKey, state.getCurrentMasterKey());
+ assertEquals(prevKey, state.getPreviousMasterKey());
+ Map<ContainerId, Long> loadedActiveTokens =
+ state.getActiveTokens();
+ assertEquals(2, loadedActiveTokens.size());
+ assertEquals(expTime1, loadedActiveTokens.get(cid1));
+ assertEquals(expTime2, loadedActiveTokens.get(cid2));
+
+ // add/update/remove tokens and verify recovered
+ ContainerId cid3 = BuilderUtils.newContainerId(3, 3, 3, 3);
+ Long expTime3 = 135798642L;
+ stateStore.storeContainerToken(cid3, expTime3);
+ stateStore.removeContainerToken(cid1);
+ expTime2 += 246897531L;
+ stateStore.storeContainerToken(cid2, expTime2);
+ prevKey = currentKey;
+ stateStore.storeContainerTokenPreviousMasterKey(prevKey);
+ currentKey = keygen.generateKey();
+ stateStore.storeContainerTokenCurrentMasterKey(currentKey);
+ restartStateStore();
+ state = stateStore.loadContainerTokensState();
+ assertEquals(currentKey, state.getCurrentMasterKey());
+ assertEquals(prevKey, state.getPreviousMasterKey());
+ loadedActiveTokens = state.getActiveTokens();
+ assertEquals(2, loadedActiveTokens.size());
+ assertNull(loadedActiveTokens.get(cid1));
+ assertEquals(expTime2, loadedActiveTokens.get(cid2));
+ assertEquals(expTime3, loadedActiveTokens.get(cid3));
+ }
+
private static class NMTokenSecretManagerForTest extends
BaseNMTokenSecretManager {
public MasterKey generateKey() {
return createNewMasterKey().getMasterKey();
}
}
+
+ private static class ContainerTokenKeyGeneratorForTest extends
+ BaseContainerTokenSecretManager {
+ public ContainerTokenKeyGeneratorForTest(Configuration conf) {
+ super(conf);
+ }
+
+ public MasterKey generateKey() {
+ return createNewMasterKey().getMasterKey();
+ }
+ }
}
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMTokenSecretManagerInNM.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMTokenSecretManagerInNM.java?rev=1614234&r1=1614233&r2=1614234&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMTokenSecretManagerInNM.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMTokenSecretManagerInNM.java
Tue Jul 29 00:49:14 2014
@@ -73,7 +73,7 @@ public class TestNMTokenSecretManagerInN
// restart and verify key is still there and token still valid
secretMgr = new NMTokenSecretManagerInNM(stateStore);
- secretMgr.recover(stateStore.loadNMTokenState());
+ secretMgr.recover();
secretMgr.setNodeId(nodeId);
assertEquals(currentKey, secretMgr.getCurrentKey());
assertTrue(secretMgr.isAppAttemptNMTokenKeyPresent(attempt1));
@@ -88,7 +88,7 @@ public class TestNMTokenSecretManagerInN
// restart and verify attempt1 key is still valid due to prev key persist
secretMgr = new NMTokenSecretManagerInNM(stateStore);
- secretMgr.recover(stateStore.loadNMTokenState());
+ secretMgr.recover();
secretMgr.setNodeId(nodeId);
assertEquals(currentKey, secretMgr.getCurrentKey());
assertFalse(secretMgr.isAppAttemptNMTokenKeyPresent(attempt1));
@@ -101,7 +101,7 @@ public class TestNMTokenSecretManagerInN
currentKey = keygen.generateKey();
secretMgr.setMasterKey(currentKey);
secretMgr = new NMTokenSecretManagerInNM(stateStore);
- secretMgr.recover(stateStore.loadNMTokenState());
+ secretMgr.recover();
secretMgr.setNodeId(nodeId);
assertEquals(currentKey, secretMgr.getCurrentKey());
assertFalse(secretMgr.isAppAttemptNMTokenKeyPresent(attempt1));
@@ -117,7 +117,7 @@ public class TestNMTokenSecretManagerInN
// remove last attempt, restart, verify both tokens are now bad
secretMgr.appFinished(attempt2.getApplicationId());
secretMgr = new NMTokenSecretManagerInNM(stateStore);
- secretMgr.recover(stateStore.loadNMTokenState());
+ secretMgr.recover();
secretMgr.setNodeId(nodeId);
assertEquals(currentKey, secretMgr.getCurrentKey());
assertFalse(secretMgr.isAppAttemptNMTokenKeyPresent(attempt1));
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml?rev=1614234&r1=1614233&r2=1614234&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
Tue Jul 29 00:49:14 2014
@@ -244,6 +244,37 @@
</execution>
</executions>
</plugin>
+
+ <plugin>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-maven-plugins</artifactId>
+ <executions>
+ <execution>
+ <id>compile-protoc</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>protoc</goal>
+ </goals>
+ <configuration>
+ <protocVersion>${protobuf.version}</protocVersion>
+ <protocCommand>${protoc.path}</protocCommand>
+ <imports>
+
<param>${basedir}/../../../../hadoop-common-project/hadoop-common/src/main/proto</param>
+ <param>${basedir}/../../hadoop-yarn-api/src/main/proto</param>
+
<param>${basedir}/../hadoop-yarn-server-common/src/main/proto</param>
+ <param>${basedir}/src/main/proto</param>
+ </imports>
+ <source>
+ <directory>${basedir}/src/main/proto</directory>
+ <includes>
+
<include>yarn_server_resourcemanager_recovery.proto</include>
+ </includes>
+ </source>
+
<output>${project.build.directory}/generated-sources/java</output>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java?rev=1614234&r1=1614233&r2=1614234&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java
Tue Jul 29 00:49:14 2014
@@ -60,7 +60,7 @@ public class RMSecretManagerService exte
clientToAMSecretManager = createClientToAMTokenSecretManager();
rmContext.setClientToAMTokenSecretManager(clientToAMSecretManager);
- amRmTokenSecretManager = createAMRMTokenSecretManager(conf);
+ amRmTokenSecretManager = createAMRMTokenSecretManager(conf,
this.rmContext);
rmContext.setAMRMTokenSecretManager(amRmTokenSecretManager);
rmDTSecretManager =
@@ -115,8 +115,8 @@ public class RMSecretManagerService exte
}
protected AMRMTokenSecretManager createAMRMTokenSecretManager(
- Configuration conf) {
- return new AMRMTokenSecretManager(conf);
+ Configuration conf, RMContext rmContext) {
+ return new AMRMTokenSecretManager(conf, rmContext);
}
protected ClientToAMTokenSecretManagerInRM
createClientToAMTokenSecretManager() {
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1614234&r1=1614233&r2=1614234&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
Tue Jul 29 00:49:14 2014
@@ -32,11 +32,13 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+import org.apache.hadoop.http.lib.StaticUserWebFilter;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.source.JvmMetrics;
import org.apache.hadoop.security.Groups;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
+import
org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.service.CompositeService;
@@ -88,8 +90,11 @@ import org.apache.hadoop.yarn.server.res
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
import
org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager;
+import
org.apache.hadoop.yarn.server.resourcemanager.security.RMAuthenticationHandler;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebApp;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
+import org.apache.hadoop.yarn.server.security.http.RMAuthenticationFilter;
+import
org.apache.hadoop.yarn.server.security.http.RMAuthenticationFilterInitializer;
import org.apache.hadoop.yarn.server.webproxy.AppReportFetcher;
import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
import org.apache.hadoop.yarn.server.webproxy.WebAppProxy;
@@ -789,6 +794,62 @@ public class ResourceManager extends Com
}
protected void startWepApp() {
+
+ // Use the customized yarn filter instead of the standard kerberos filter
to
+ // allow users to authenticate using delegation tokens
+ // 3 conditions need to be satisfied -
+ // 1. security is enabled
+ // 2. http auth type is set to kerberos
+ // 3. "yarn.resourcemanager.webapp.use-yarn-filter" override is set to true
+
+ Configuration conf = getConfig();
+ boolean useYarnAuthenticationFilter =
+ conf.getBoolean(
+ YarnConfiguration.RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER,
+ YarnConfiguration.DEFAULT_RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER);
+ String authPrefix = "hadoop.http.authentication.";
+ String authTypeKey = authPrefix + "type";
+ String initializers = conf.get("hadoop.http.filter.initializers");
+ if (UserGroupInformation.isSecurityEnabled()
+ && useYarnAuthenticationFilter
+ && conf.get(authTypeKey, "").equalsIgnoreCase(
+ KerberosAuthenticationHandler.TYPE)) {
+ LOG.info("Using RM authentication filter(kerberos/delegation-token)"
+ + " for RM webapp authentication");
+ RMAuthenticationHandler
+ .setSecretManager(getClientRMService().rmDTSecretManager);
+ String yarnAuthKey =
+ authPrefix + RMAuthenticationFilter.AUTH_HANDLER_PROPERTY;
+ conf.setStrings(yarnAuthKey, RMAuthenticationHandler.class.getName());
+
+ initializers =
+ initializers == null || initializers.isEmpty() ? "" : ","
+ + initializers;
+ if (!initializers.contains(RMAuthenticationFilterInitializer.class
+ .getName())) {
+ conf.set("hadoop.http.filter.initializers",
+ RMAuthenticationFilterInitializer.class.getName() + initializers);
+ }
+ }
+
+ // if security is not enabled and the default filter initializer has been
+ // set, set the initializer to include the
+ // RMAuthenticationFilterInitializer which in turn will set up the simple
+ // auth filter.
+
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ if (initializers == null || initializers.isEmpty()) {
+ conf.set("hadoop.http.filter.initializers",
+ RMAuthenticationFilterInitializer.class.getName());
+ conf.set(authTypeKey, "simple");
+ } else if (initializers.equals(StaticUserWebFilter.class.getName())) {
+ conf.set("hadoop.http.filter.initializers",
+ RMAuthenticationFilterInitializer.class.getName() + ","
+ + initializers);
+ conf.set(authTypeKey, "simple");
+ }
+ }
+
Builder<ApplicationMasterService> builder =
WebApps
.$for("cluster", ApplicationMasterService.class, masterService,
@@ -1026,6 +1087,9 @@ public class ResourceManager extends Com
// recover RMdelegationTokenSecretManager
rmContext.getRMDelegationTokenSecretManager().recover(state);
+ // recover AMRMTokenSecretManager
+ rmContext.getAMRMTokenSecretManager().recover(state);
+
// recover applications
rmAppManager.recover(state);
}
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java?rev=1614234&r1=1614233&r2=1614234&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
Tue Jul 29 00:49:14 2014
@@ -22,6 +22,7 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -43,16 +44,18 @@ import org.apache.hadoop.security.token.
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import
org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.AMRMTokenSecretManagerStateProto;
import
org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.EpochProto;
import
org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ApplicationAttemptStateDataProto;
import
org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ApplicationStateDataProto;
import
org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RMStateVersionProto;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
+import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenSecretManagerState;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
-
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.Epoch;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion;
+import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.AMRMTokenSecretManagerStatePBImpl;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.EpochPBImpl;
@@ -76,6 +79,8 @@ public class FileSystemRMStateStore exte
protected static final String ROOT_DIR_NAME = "FSRMStateRoot";
protected static final RMStateVersion CURRENT_VERSION_INFO = RMStateVersion
.newInstance(1, 1);
+ protected static final String AMRMTOKEN_SECRET_MANAGER_NODE =
+ "AMRMTokenSecretManagerNode";
protected FileSystem fs;
@@ -89,6 +94,7 @@ public class FileSystemRMStateStore exte
@VisibleForTesting
Path fsWorkingPath;
+ Path amrmTokenSecretManagerRoot;
@Override
public synchronized void initInternal(Configuration conf)
throws Exception{
@@ -96,6 +102,8 @@ public class FileSystemRMStateStore exte
rootDirPath = new Path(fsWorkingPath, ROOT_DIR_NAME);
rmDTSecretManagerRoot = new Path(rootDirPath, RM_DT_SECRET_MANAGER_ROOT);
rmAppRoot = new Path(rootDirPath, RM_APP_ROOT);
+ amrmTokenSecretManagerRoot =
+ new Path(rootDirPath, AMRMTOKEN_SECRET_MANAGER_ROOT);
}
@Override
@@ -113,6 +121,7 @@ public class FileSystemRMStateStore exte
fs = fsWorkingPath.getFileSystem(conf);
fs.mkdirs(rmDTSecretManagerRoot);
fs.mkdirs(rmAppRoot);
+ fs.mkdirs(amrmTokenSecretManagerRoot);
}
@Override
@@ -180,9 +189,32 @@ public class FileSystemRMStateStore exte
loadRMDTSecretManagerState(rmState);
// recover RM applications
loadRMAppState(rmState);
+ // recover AMRMTokenSecretManager
+ loadAMRMTokenSecretManagerState(rmState);
return rmState;
}
+ private void loadAMRMTokenSecretManagerState(RMState rmState)
+ throws Exception {
+ checkAndResumeUpdateOperation(amrmTokenSecretManagerRoot);
+ Path amrmTokenSecretManagerStateDataDir =
+ new Path(amrmTokenSecretManagerRoot, AMRMTOKEN_SECRET_MANAGER_NODE);
+ FileStatus status;
+ try {
+ status = fs.getFileStatus(amrmTokenSecretManagerStateDataDir);
+ assert status.isFile();
+ } catch (FileNotFoundException ex) {
+ return;
+ }
+ byte[] data = readFile(amrmTokenSecretManagerStateDataDir,
status.getLen());
+ AMRMTokenSecretManagerStatePBImpl stateData =
+ new AMRMTokenSecretManagerStatePBImpl(
+ AMRMTokenSecretManagerStateProto.parseFrom(data));
+ rmState.amrmTokenSecretManagerState =
+ AMRMTokenSecretManagerState.newInstance(
+ stateData.getCurrentMasterKey(), stateData.getNextMasterKey());
+ }
+
private void loadRMAppState(RMState rmState) throws Exception {
try {
List<ApplicationAttemptState> attempts =
@@ -597,4 +629,25 @@ public class FileSystemRMStateStore exte
return new Path(root, nodeName);
}
+ @Override
+ public synchronized void storeOrUpdateAMRMTokenSecretManagerState(
+ AMRMTokenSecretManagerState amrmTokenSecretManagerState,
+ boolean isUpdate){
+ Path nodeCreatePath =
+ getNodePath(amrmTokenSecretManagerRoot, AMRMTOKEN_SECRET_MANAGER_NODE);
+ AMRMTokenSecretManagerState data =
+ AMRMTokenSecretManagerState.newInstance(amrmTokenSecretManagerState);
+ byte[] stateData = data.getProto().toByteArray();
+ try {
+ if (isUpdate) {
+ updateFile(nodeCreatePath, stateData);
+ } else {
+ writeFile(nodeCreatePath, stateData);
+ }
+ } catch (Exception ex) {
+ LOG.info("Error storing info for AMRMTokenSecretManager", ex);
+ notifyStoreOperationFailed(ex);
+ }
+ }
+
}
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java?rev=1614234&r1=1614233&r2=1614234&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
Tue Jul 29 00:49:14 2014
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
+import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenSecretManagerState;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion;
@@ -72,6 +73,10 @@ public class MemoryRMStateStore extends
state.rmSecretManagerState.getTokenState());
returnState.rmSecretManagerState.dtSequenceNumber =
state.rmSecretManagerState.dtSequenceNumber;
+ returnState.amrmTokenSecretManagerState =
+ state.amrmTokenSecretManagerState == null ? null
+ : AMRMTokenSecretManagerState
+ .newInstance(state.amrmTokenSecretManagerState);
return returnState;
}
@@ -268,6 +273,16 @@ public class MemoryRMStateStore extends
}
@Override
+ public void storeOrUpdateAMRMTokenSecretManagerState(
+ AMRMTokenSecretManagerState amrmTokenSecretManagerState,
+ boolean isUpdate) {
+ if (amrmTokenSecretManagerState != null) {
+ state.amrmTokenSecretManagerState = AMRMTokenSecretManagerState
+ .newInstance(amrmTokenSecretManagerState);
+ }
+ }
+
+ @Override
public void deleteStore() throws Exception {
}
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java?rev=1614234&r1=1614233&r2=1614234&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java
Tue Jul 29 00:49:14 2014
@@ -25,6 +25,7 @@ import org.apache.hadoop.security.token.
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
+import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenSecretManagerState;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion;
@@ -139,6 +140,12 @@ public class NullRMStateStore extends RM
}
@Override
+ public void storeOrUpdateAMRMTokenSecretManagerState(
+ AMRMTokenSecretManagerState state, boolean isUpdate) {
+ //DO Nothing
+ }
+
+ @Override
public void deleteStore() throws Exception {
// Do nothing
}
