Author: clamb
Date: Sun Jul 27 13:32:51 2014
New Revision: 1613788
URL: http://svn.apache.org/r1613788
Log:
merge from trunk r1613787
Added:
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/AMRMTokenSecretManagerState.java
- copied unchanged from r1613787,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/AMRMTokenSecretManagerState.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/AMRMTokenSecretManagerStatePBImpl.java
- copied unchanged from r1613787,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/AMRMTokenSecretManagerStatePBImpl.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/
- copied from r1613787,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
- copied unchanged from r1613787,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
Modified:
hadoop/common/branches/fs-encryption/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceOnHA.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/CHANGES.txt?rev=1613788&r1=1613787&r2=1613788&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/CHANGES.txt
(original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/CHANGES.txt Sun
Jul 27 13:32:51 2014
@@ -65,6 +65,12 @@ Release 2.6.0 - UNRELEASED
YARN-1342. Recover container tokens upon nodemanager restart. (Jason Lowe
via
devaraj)
+ YARN-2214. FairScheduler: preemptContainerPreCheck() in FSParentQueue
delays
+ convergence towards fairness. (Ashwin Shankar via kasha)
+
+ YARN-2211. Persist AMRMToken master key in RMStateStore for RM recovery.
+ (Xuan Gong via jianhe)
+
OPTIMIZATIONS
BUG FIXES
@@ -100,6 +106,8 @@ Release 2.6.0 - UNRELEASED
YARN-2147. client lacks delegation token exception details when
application submit fails (Chen He via jlowe)
+ YARN-1796. container-executor shouldn't require o-r permissions (atm)
+
Release 2.5.0 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -436,6 +444,11 @@ Release 2.5.0 - UNRELEASED
YARN-2319. Made the MiniKdc instance start/close before/after the class of
TestRMWebServicesDelegationTokens. (Wenwu Peng via zjshen)
+ YARN-2335. Annotate all hadoop-sls APIs as @Private. (Wei Yan via kasha)
+
+ YARN-1726. ResourceSchedulerWrapper broken due to AbstractYarnScheduler.
+ (Wei Yan via kasha)
+
Release 2.4.1 - 2014-06-23
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java?rev=1613788&r1=1613787&r2=1613788&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
Sun Jul 27 13:32:51 2014
@@ -267,6 +267,7 @@ public abstract class ProtocolHATestBase
protected void startHACluster(int numOfNMs, boolean overrideClientRMService,
boolean overrideRTS, boolean overrideApplicationMasterService)
throws Exception {
+ conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
cluster =
new MiniYARNClusterForHATesting(TestRMFailover.class.getName(), 2,
Modified:
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceOnHA.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceOnHA.java?rev=1613788&r1=1613787&r2=1613788&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceOnHA.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceOnHA.java
Sun Jul 27 13:32:51 2014
@@ -54,11 +54,9 @@ public class TestApplicationMasterServic
amClient = ClientRMProxy
.createRMProxy(this.conf, ApplicationMasterProtocol.class);
- AMRMTokenIdentifier id =
- new AMRMTokenIdentifier(attemptId);
Token<AMRMTokenIdentifier> appToken =
- new Token<AMRMTokenIdentifier>(id, this.cluster.getResourceManager()
- .getRMContext().getAMRMTokenSecretManager());
+ this.cluster.getResourceManager().getRMContext()
+ .getAMRMTokenSecretManager().createAndGetAMRMToken(attemptId);
appToken.setService(new Text("appToken service"));
UserGroupInformation.setLoginUser(UserGroupInformation
.createRemoteUser(UserGroupInformation.getCurrentUser()
Modified:
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c?rev=1613788&r1=1613787&r2=1613788&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
Sun Jul 27 13:32:51 2014
@@ -111,16 +111,16 @@ int check_executor_permissions(char *exe
return -1;
}
- // check others do not have read/write/execute permissions
- if ((filestat.st_mode & S_IROTH) == S_IROTH || (filestat.st_mode & S_IWOTH)
- == S_IWOTH || (filestat.st_mode & S_IXOTH) == S_IXOTH) {
+ // check others do not have write/execute permissions
+ if ((filestat.st_mode & S_IWOTH) == S_IWOTH ||
+ (filestat.st_mode & S_IXOTH) == S_IXOTH) {
fprintf(LOGFILE,
- "The container-executor binary should not have read or write or"
- " execute for others.\n");
+ "The container-executor binary should not have write or execute "
+ "for others.\n");
return -1;
}
- // Binary should be setuid/setgid executable
+ // Binary should be setuid executable
if ((filestat.st_mode & S_ISUID) == 0) {
fprintf(LOGFILE, "The container-executor binary should be set setuid.\n");
return -1;
Modified:
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml?rev=1613788&r1=1613787&r2=1613788&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
Sun Jul 27 13:32:51 2014
@@ -244,6 +244,37 @@
</execution>
</executions>
</plugin>
+
+ <plugin>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-maven-plugins</artifactId>
+ <executions>
+ <execution>
+ <id>compile-protoc</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>protoc</goal>
+ </goals>
+ <configuration>
+ <protocVersion>${protobuf.version}</protocVersion>
+ <protocCommand>${protoc.path}</protocCommand>
+ <imports>
+
<param>${basedir}/../../../../hadoop-common-project/hadoop-common/src/main/proto</param>
+ <param>${basedir}/../../hadoop-yarn-api/src/main/proto</param>
+
<param>${basedir}/../hadoop-yarn-server-common/src/main/proto</param>
+ <param>${basedir}/src/main/proto</param>
+ </imports>
+ <source>
+ <directory>${basedir}/src/main/proto</directory>
+ <includes>
+
<include>yarn_server_resourcemanager_recovery.proto</include>
+ </includes>
+ </source>
+
<output>${project.build.directory}/generated-sources/java</output>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
Modified:
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java?rev=1613788&r1=1613787&r2=1613788&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java
Sun Jul 27 13:32:51 2014
@@ -60,7 +60,7 @@ public class RMSecretManagerService exte
clientToAMSecretManager = createClientToAMTokenSecretManager();
rmContext.setClientToAMTokenSecretManager(clientToAMSecretManager);
- amRmTokenSecretManager = createAMRMTokenSecretManager(conf);
+ amRmTokenSecretManager = createAMRMTokenSecretManager(conf,
this.rmContext);
rmContext.setAMRMTokenSecretManager(amRmTokenSecretManager);
rmDTSecretManager =
@@ -115,8 +115,8 @@ public class RMSecretManagerService exte
}
protected AMRMTokenSecretManager createAMRMTokenSecretManager(
- Configuration conf) {
- return new AMRMTokenSecretManager(conf);
+ Configuration conf, RMContext rmContext) {
+ return new AMRMTokenSecretManager(conf, rmContext);
}
protected ClientToAMTokenSecretManagerInRM
createClientToAMTokenSecretManager() {
Modified:
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1613788&r1=1613787&r2=1613788&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
Sun Jul 27 13:32:51 2014
@@ -1026,6 +1026,9 @@ public class ResourceManager extends Com
// recover RMdelegationTokenSecretManager
rmContext.getRMDelegationTokenSecretManager().recover(state);
+ // recover AMRMTokenSecretManager
+ rmContext.getAMRMTokenSecretManager().recover(state);
+
// recover applications
rmAppManager.recover(state);
}
Modified:
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java?rev=1613788&r1=1613787&r2=1613788&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
Sun Jul 27 13:32:51 2014
@@ -22,6 +22,7 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -43,16 +44,18 @@ import org.apache.hadoop.security.token.
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import
org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.AMRMTokenSecretManagerStateProto;
import
org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.EpochProto;
import
org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ApplicationAttemptStateDataProto;
import
org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ApplicationStateDataProto;
import
org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RMStateVersionProto;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
+import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenSecretManagerState;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
-
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.Epoch;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion;
+import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.AMRMTokenSecretManagerStatePBImpl;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.EpochPBImpl;
@@ -76,6 +79,8 @@ public class FileSystemRMStateStore exte
protected static final String ROOT_DIR_NAME = "FSRMStateRoot";
protected static final RMStateVersion CURRENT_VERSION_INFO = RMStateVersion
.newInstance(1, 1);
+ protected static final String AMRMTOKEN_SECRET_MANAGER_NODE =
+ "AMRMTokenSecretManagerNode";
protected FileSystem fs;
@@ -89,6 +94,7 @@ public class FileSystemRMStateStore exte
@VisibleForTesting
Path fsWorkingPath;
+ Path amrmTokenSecretManagerRoot;
@Override
public synchronized void initInternal(Configuration conf)
throws Exception{
@@ -96,6 +102,8 @@ public class FileSystemRMStateStore exte
rootDirPath = new Path(fsWorkingPath, ROOT_DIR_NAME);
rmDTSecretManagerRoot = new Path(rootDirPath, RM_DT_SECRET_MANAGER_ROOT);
rmAppRoot = new Path(rootDirPath, RM_APP_ROOT);
+ amrmTokenSecretManagerRoot =
+ new Path(rootDirPath, AMRMTOKEN_SECRET_MANAGER_ROOT);
}
@Override
@@ -113,6 +121,7 @@ public class FileSystemRMStateStore exte
fs = fsWorkingPath.getFileSystem(conf);
fs.mkdirs(rmDTSecretManagerRoot);
fs.mkdirs(rmAppRoot);
+ fs.mkdirs(amrmTokenSecretManagerRoot);
}
@Override
@@ -180,9 +189,32 @@ public class FileSystemRMStateStore exte
loadRMDTSecretManagerState(rmState);
// recover RM applications
loadRMAppState(rmState);
+ // recover AMRMTokenSecretManager
+ loadAMRMTokenSecretManagerState(rmState);
return rmState;
}
+ private void loadAMRMTokenSecretManagerState(RMState rmState)
+ throws Exception {
+ checkAndResumeUpdateOperation(amrmTokenSecretManagerRoot);
+ Path amrmTokenSecretManagerStateDataDir =
+ new Path(amrmTokenSecretManagerRoot, AMRMTOKEN_SECRET_MANAGER_NODE);
+ FileStatus status;
+ try {
+ status = fs.getFileStatus(amrmTokenSecretManagerStateDataDir);
+ assert status.isFile();
+ } catch (FileNotFoundException ex) {
+ return;
+ }
+ byte[] data = readFile(amrmTokenSecretManagerStateDataDir,
status.getLen());
+ AMRMTokenSecretManagerStatePBImpl stateData =
+ new AMRMTokenSecretManagerStatePBImpl(
+ AMRMTokenSecretManagerStateProto.parseFrom(data));
+ rmState.amrmTokenSecretManagerState =
+ AMRMTokenSecretManagerState.newInstance(
+ stateData.getCurrentMasterKey(), stateData.getNextMasterKey());
+ }
+
private void loadRMAppState(RMState rmState) throws Exception {
try {
List<ApplicationAttemptState> attempts =
@@ -597,4 +629,25 @@ public class FileSystemRMStateStore exte
return new Path(root, nodeName);
}
+ @Override
+ public synchronized void storeOrUpdateAMRMTokenSecretManagerState(
+ AMRMTokenSecretManagerState amrmTokenSecretManagerState,
+ boolean isUpdate){
+ Path nodeCreatePath =
+ getNodePath(amrmTokenSecretManagerRoot, AMRMTOKEN_SECRET_MANAGER_NODE);
+ AMRMTokenSecretManagerState data =
+ AMRMTokenSecretManagerState.newInstance(amrmTokenSecretManagerState);
+ byte[] stateData = data.getProto().toByteArray();
+ try {
+ if (isUpdate) {
+ updateFile(nodeCreatePath, stateData);
+ } else {
+ writeFile(nodeCreatePath, stateData);
+ }
+ } catch (Exception ex) {
+ LOG.info("Error storing info for AMRMTokenSecretManager", ex);
+ notifyStoreOperationFailed(ex);
+ }
+ }
+
}
Modified:
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java?rev=1613788&r1=1613787&r2=1613788&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
Sun Jul 27 13:32:51 2014
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
+import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenSecretManagerState;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion;
@@ -72,6 +73,10 @@ public class MemoryRMStateStore extends
state.rmSecretManagerState.getTokenState());
returnState.rmSecretManagerState.dtSequenceNumber =
state.rmSecretManagerState.dtSequenceNumber;
+ returnState.amrmTokenSecretManagerState =
+ state.amrmTokenSecretManagerState == null ? null
+ : AMRMTokenSecretManagerState
+ .newInstance(state.amrmTokenSecretManagerState);
return returnState;
}
@@ -268,6 +273,16 @@ public class MemoryRMStateStore extends
}
@Override
+ public void storeOrUpdateAMRMTokenSecretManagerState(
+ AMRMTokenSecretManagerState amrmTokenSecretManagerState,
+ boolean isUpdate) {
+ if (amrmTokenSecretManagerState != null) {
+ state.amrmTokenSecretManagerState = AMRMTokenSecretManagerState
+ .newInstance(amrmTokenSecretManagerState);
+ }
+ }
+
+ @Override
public void deleteStore() throws Exception {
}
Modified:
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java?rev=1613788&r1=1613787&r2=1613788&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java
Sun Jul 27 13:32:51 2014
@@ -25,6 +25,7 @@ import org.apache.hadoop.security.token.
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
+import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenSecretManagerState;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion;
@@ -139,6 +140,12 @@ public class NullRMStateStore extends RM
}
@Override
+ public void storeOrUpdateAMRMTokenSecretManagerState(
+ AMRMTokenSecretManagerState state, boolean isUpdate) {
+ //DO Nothing
+ }
+
+ @Override
public void deleteStore() throws Exception {
// Do nothing
}
Modified:
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java?rev=1613788&r1=1613787&r2=1613788&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
Sun Jul 27 13:32:51 2014
@@ -45,16 +45,14 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.RMFatalEvent;
import org.apache.hadoop.yarn.server.resourcemanager.RMFatalEventType;
+import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenSecretManagerState;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion;
-import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl;
-import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppNewSavedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
@@ -85,6 +83,8 @@ public abstract class RMStateStore exten
protected static final String DELEGATION_TOKEN_PREFIX = "RMDelegationToken_";
protected static final String DELEGATION_TOKEN_SEQUENCE_NUMBER_PREFIX =
"RMDTSequenceNumber_";
+ protected static final String AMRMTOKEN_SECRET_MANAGER_ROOT =
+ "AMRMTokenSecretManagerRoot";
protected static final String VERSION_NODE = "RMVersionNode";
protected static final String EPOCH_NODE = "EpochNode";
@@ -412,6 +412,8 @@ public abstract class RMStateStore exten
RMDTSecretManagerState rmSecretManagerState = new RMDTSecretManagerState();
+ AMRMTokenSecretManagerState amrmTokenSecretManagerState = null;
+
public Map<ApplicationId, ApplicationState> getApplicationState() {
return appState;
}
@@ -419,6 +421,10 @@ public abstract class RMStateStore exten
public RMDTSecretManagerState getRMDTSecretManagerState() {
return rmSecretManagerState;
}
+
+ public AMRMTokenSecretManagerState getAMRMTokenSecretManagerState() {
+ return amrmTokenSecretManagerState;
+ }
}
private Dispatcher rmDispatcher;
@@ -714,6 +720,14 @@ public abstract class RMStateStore exten
throws Exception;
/**
+ * Blocking API Derived classes must implement this method to store or update
+ * the state of AMRMToken Master Key
+ */
+ public abstract void storeOrUpdateAMRMTokenSecretManagerState(
+ AMRMTokenSecretManagerState amrmTokenSecretManagerState,
+ boolean isUpdate);
+
+ /**
* Non-blocking API
* ResourceManager services call this to remove an application from the state
* store
Modified:
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java?rev=1613788&r1=1613787&r2=1613788&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
Sun Jul 27 13:32:51 2014
@@ -44,18 +44,19 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos;
+import
org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.AMRMTokenSecretManagerStateProto;
import
org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ApplicationAttemptStateDataProto;
import
org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ApplicationStateDataProto;
import
org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RMStateVersionProto;
import
org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.EpochProto;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.RMZKUtils;
+import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenSecretManagerState;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
-
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.Epoch;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion;
+import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.AMRMTokenSecretManagerStatePBImpl;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.EpochPBImpl;
@@ -128,6 +129,9 @@ public class ZKRMStateStore extends RMSt
* | |----- Key_1
* | |----- Key_2
* ....
+ * |--- AMRMTOKEN_SECRET_MANAGER_ROOT
+ * |----- currentMasterKey
+ * |----- nextMasterKey
*
*/
private String zkRootNodePath;
@@ -136,6 +140,7 @@ public class ZKRMStateStore extends RMSt
private String dtMasterKeysRootPath;
private String delegationTokensRootPath;
private String dtSequenceNumberPath;
+ private String amrmTokenSecretManagerRoot;
@VisibleForTesting
protected String znodeWorkingPath;
@@ -255,6 +260,8 @@ public class ZKRMStateStore extends RMSt
RM_DELEGATION_TOKENS_ROOT_ZNODE_NAME);
dtSequenceNumberPath = getNodePath(rmDTSecretManagerRoot,
RM_DT_SEQUENTIAL_NUMBER_ZNODE_NAME);
+ amrmTokenSecretManagerRoot =
+ getNodePath(zkRootNodePath, AMRMTOKEN_SECRET_MANAGER_ROOT);
}
@Override
@@ -275,6 +282,7 @@ public class ZKRMStateStore extends RMSt
createRootDir(dtMasterKeysRootPath);
createRootDir(delegationTokensRootPath);
createRootDir(dtSequenceNumberPath);
+ createRootDir(amrmTokenSecretManagerRoot);
}
private void createRootDir(final String rootPath) throws Exception {
@@ -427,9 +435,27 @@ public class ZKRMStateStore extends RMSt
loadRMDTSecretManagerState(rmState);
// recover RM applications
loadRMAppState(rmState);
+ // recover AMRMTokenSecretManager
+ loadAMRMTokenSecretManagerState(rmState);
return rmState;
}
+ private void loadAMRMTokenSecretManagerState(RMState rmState)
+ throws Exception {
+ byte[] data = getDataWithRetries(amrmTokenSecretManagerRoot, true);
+ if (data == null) {
+ LOG.warn("There is no data saved");
+ return;
+ }
+ AMRMTokenSecretManagerStatePBImpl stateData =
+ new AMRMTokenSecretManagerStatePBImpl(
+ AMRMTokenSecretManagerStateProto.parseFrom(data));
+ rmState.amrmTokenSecretManagerState =
+ AMRMTokenSecretManagerState.newInstance(
+ stateData.getCurrentMasterKey(), stateData.getNextMasterKey());
+
+ }
+
private synchronized void loadRMDTSecretManagerState(RMState rmState)
throws Exception {
loadRMDelegationKeyState(rmState);
@@ -1112,4 +1138,19 @@ public class ZKRMStateStore extends RMSt
return zk;
}
+ @Override
+ public synchronized void storeOrUpdateAMRMTokenSecretManagerState(
+ AMRMTokenSecretManagerState amrmTokenSecretManagerState,
+ boolean isUpdate) {
+ AMRMTokenSecretManagerState data =
+ AMRMTokenSecretManagerState.newInstance(amrmTokenSecretManagerState);
+ byte[] stateData = data.getProto().toByteArray();
+ try {
+ setDataWithRetries(amrmTokenSecretManagerRoot, stateData, -1);
+ } catch (Exception ex) {
+ LOG.info("Error storing info for AMRMTokenSecretManager", ex);
+ notifyStoreOperationFailed(ex);
+ }
+ }
+
}
Modified:
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java?rev=1613788&r1=1613787&r2=1613788&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
Sun Jul 27 13:32:51 2014
@@ -224,16 +224,17 @@ public class FSLeafQueue extends FSQueue
@Override
public RMContainer preemptContainer() {
RMContainer toBePreempted = null;
- if (LOG.isDebugEnabled()) {
- LOG.debug("Queue " + getName() + " is going to preempt a container " +
- "from its applications.");
- }
// If this queue is not over its fair share, reject
if (!preemptContainerPreCheck()) {
return toBePreempted;
}
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Queue " + getName() + " is going to preempt a container " +
+ "from its applications.");
+ }
+
// Choose the app that is most over fair share
Comparator<Schedulable> comparator = policy.getComparator();
AppSchedulable candidateSched = null;
@@ -328,4 +329,14 @@ public class FSLeafQueue extends FSQueue
SchedulerApplicationAttempt schedulerAttempt, RMContainer rmContainer) {
// TODO Auto-generated method stub
}
+
+ /**
+ * Helper method to check if the queue should preempt containers
+ *
+ * @return true if check passes (can preempt) or false otherwise
+ */
+ private boolean preemptContainerPreCheck() {
+ return parent.getPolicy().checkIfUsageOverFairShare(getResourceUsage(),
+ getFairShare());
+ }
}
Modified:
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java?rev=1613788&r1=1613787&r2=1613788&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
Sun Jul 27 13:32:51 2014
@@ -164,11 +164,6 @@ public class FSParentQueue extends FSQue
public RMContainer preemptContainer() {
RMContainer toBePreempted = null;
- // If this queue is not over its fair share, reject
- if (!preemptContainerPreCheck()) {
- return toBePreempted;
- }
-
// Find the childQueue which is most over fair share
FSQueue candidateQueue = null;
Comparator<Schedulable> comparator = policy.getComparator();
Modified:
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java?rev=1613788&r1=1613787&r2=1613788&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
Sun Jul 27 13:32:51 2014
@@ -187,17 +187,4 @@ public abstract class FSQueue extends Sc
}
return true;
}
-
- /**
- * Helper method to check if the queue should preempt containers
- *
- * @return true if check passes (can preempt) or false otherwise
- */
- protected boolean preemptContainerPreCheck() {
- if (this == scheduler.getQueueManager().getRootQueue()) {
- return true;
- }
- return parent.getPolicy()
- .checkIfUsageOverFairShare(getResourceUsage(), getFairShare());
- }
}
Modified:
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java?rev=1613788&r1=1613787&r2=1613788&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java
Sun Jul 27 13:32:51 2014
@@ -38,6 +38,10 @@ import org.apache.hadoop.security.token.
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
+import org.apache.hadoop.yarn.server.api.records.MasterKey;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import
org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
+import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenSecretManagerState;
import org.apache.hadoop.yarn.server.security.MasterKeyData;
import com.google.common.annotations.VisibleForTesting;
@@ -66,6 +70,7 @@ public class AMRMTokenSecretManager exte
private final Timer timer;
private final long rollingInterval;
private final long activationDelay;
+ private RMContext rmContext;
private final Set<ApplicationAttemptId> appAttemptSet =
new HashSet<ApplicationAttemptId>();
@@ -73,7 +78,8 @@ public class AMRMTokenSecretManager exte
/**
* Create an {@link AMRMTokenSecretManager}
*/
- public AMRMTokenSecretManager(Configuration conf) {
+ public AMRMTokenSecretManager(Configuration conf, RMContext rmContext) {
+ this.rmContext = rmContext;
this.timer = new Timer();
this.rollingInterval =
conf
@@ -98,6 +104,11 @@ public class AMRMTokenSecretManager exte
public void start() {
if (this.currentMasterKey == null) {
this.currentMasterKey = createNewMasterKey();
+ AMRMTokenSecretManagerState state =
+ AMRMTokenSecretManagerState.newInstance(
+ this.currentMasterKey.getMasterKey(), null);
+ rmContext.getStateStore().storeOrUpdateAMRMTokenSecretManagerState(state,
+ false);
}
this.timer.scheduleAtFixedRate(new MasterKeyRoller(), rollingInterval,
rollingInterval);
@@ -130,6 +141,12 @@ public class AMRMTokenSecretManager exte
try {
LOG.info("Rolling master-key for amrm-tokens");
this.nextMasterKey = createNewMasterKey();
+ AMRMTokenSecretManagerState state =
+ AMRMTokenSecretManagerState.newInstance(
+ this.currentMasterKey.getMasterKey(),
+ this.nextMasterKey.getMasterKey());
+ rmContext.getStateStore().storeOrUpdateAMRMTokenSecretManagerState(state,
+ true);
this.timer.schedule(new NextKeyActivator(), this.activationDelay);
} finally {
this.writeLock.unlock();
@@ -225,8 +242,8 @@ public class AMRMTokenSecretManager exte
LOG.debug("Trying to retrieve password for " + applicationAttemptId);
}
if (!appAttemptSet.contains(applicationAttemptId)) {
- throw new InvalidToken("Password not found for ApplicationAttempt "
- + applicationAttemptId);
+ throw new InvalidToken(applicationAttemptId
+ + " not found in AMRMTokenSecretManager.");
}
if (identifier.getKeyId() == this.currentMasterKey.getMasterKey()
.getKeyId()) {
@@ -238,9 +255,7 @@ public class AMRMTokenSecretManager exte
return createPassword(identifier.getBytes(),
this.nextMasterKey.getSecretKey());
}
- throw new InvalidToken("Given AMRMToken for application : "
- + applicationAttemptId.toString()
- + " seems to have been generated illegally.");
+ throw new InvalidToken("Invalid AMRMToken from " + applicationAttemptId);
} finally {
this.readLock.unlock();
}
@@ -291,4 +306,25 @@ public class AMRMTokenSecretManager exte
this.readLock.unlock();
}
}
+
+ public void recover(RMState state) {
+ if (state.getAMRMTokenSecretManagerState() != null) {
+ // recover the current master key
+ MasterKey currentKey =
+ state.getAMRMTokenSecretManagerState().getCurrentMasterKey();
+ this.currentMasterKey =
+ new MasterKeyData(currentKey, createSecretKey(currentKey.getBytes()
+ .array()));
+
+ // recover the next master key if not null
+ MasterKey nextKey =
+ state.getAMRMTokenSecretManagerState().getNextMasterKey();
+ if (nextKey != null) {
+ this.nextMasterKey =
+ new MasterKeyData(nextKey, createSecretKey(nextKey.getBytes()
+ .array()));
+ this.timer.schedule(new NextKeyActivator(), this.activationDelay);
+ }
+ }
+ }
}
Modified:
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java?rev=1613788&r1=1613787&r2=1613788&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
Sun Jul 27 13:32:51 2014
@@ -1250,11 +1250,10 @@ public class TestRMRestart {
.getEncoded());
// assert AMRMTokenSecretManager also knows about the AMRMToken password
- // TODO: fix this on YARN-2211
-// Token<AMRMTokenIdentifier> amrmToken = loadedAttempt1.getAMRMToken();
-// Assert.assertArrayEquals(amrmToken.getPassword(),
-// rm2.getRMContext().getAMRMTokenSecretManager().retrievePassword(
-// amrmToken.decodeIdentifier()));
+ Token<AMRMTokenIdentifier> amrmToken = loadedAttempt1.getAMRMToken();
+ Assert.assertArrayEquals(amrmToken.getPassword(),
+ rm2.getRMContext().getAMRMTokenSecretManager().retrievePassword(
+ amrmToken.decodeIdentifier()));
rm1.stop();
rm2.stop();
}
Modified:
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java?rev=1613788&r1=1613787&r2=1613788&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
Sun Jul 27 13:32:51 2014
@@ -55,10 +55,12 @@ import org.apache.hadoop.yarn.event.Disp
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationAttemptState;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMDTSecretManagerState;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
+import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenSecretManagerState;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
@@ -176,8 +178,12 @@ public class RMStateStoreTestBase extend
TestDispatcher dispatcher = new TestDispatcher();
store.setRMDispatcher(dispatcher);
- AMRMTokenSecretManager appTokenMgr = spy(
- new AMRMTokenSecretManager(conf));
+ RMContext rmContext = mock(RMContext.class);
+ when(rmContext.getStateStore()).thenReturn(store);
+
+ AMRMTokenSecretManager appTokenMgr =
+ spy(new AMRMTokenSecretManager(conf, rmContext));
+
MasterKeyData masterKeyData = appTokenMgr.createNewMasterKey();
when(appTokenMgr.getMasterKey()).thenReturn(masterKeyData);
@@ -576,4 +582,65 @@ public class RMStateStoreTestBase extend
}
+ public void testAMRMTokenSecretManagerStateStore(
+ RMStateStoreHelper stateStoreHelper) throws Exception {
+ System.out.println("Start testing");
+ RMStateStore store = stateStoreHelper.getRMStateStore();
+ TestDispatcher dispatcher = new TestDispatcher();
+ store.setRMDispatcher(dispatcher);
+
+ RMContext rmContext = mock(RMContext.class);
+ when(rmContext.getStateStore()).thenReturn(store);
+ Configuration conf = new YarnConfiguration();
+ AMRMTokenSecretManager appTokenMgr =
+ new AMRMTokenSecretManager(conf, rmContext);
+
+ //create and save the first masterkey
+ MasterKeyData firstMasterKeyData = appTokenMgr.createNewMasterKey();
+
+ AMRMTokenSecretManagerState state1 =
+ AMRMTokenSecretManagerState.newInstance(
+ firstMasterKeyData.getMasterKey(), null);
+ rmContext.getStateStore().storeOrUpdateAMRMTokenSecretManagerState(state1,
+ false);
+
+ // load state
+ store = stateStoreHelper.getRMStateStore();
+ store.setRMDispatcher(dispatcher);
+ RMState state = store.loadState();
+ Assert.assertNotNull(state.getAMRMTokenSecretManagerState());
+ Assert.assertEquals(firstMasterKeyData.getMasterKey(), state
+ .getAMRMTokenSecretManagerState().getCurrentMasterKey());
+ Assert.assertNull(state
+ .getAMRMTokenSecretManagerState().getNextMasterKey());
+
+ //create and save the second masterkey
+ MasterKeyData secondMasterKeyData = appTokenMgr.createNewMasterKey();
+ AMRMTokenSecretManagerState state2 =
+ AMRMTokenSecretManagerState
+ .newInstance(firstMasterKeyData.getMasterKey(),
+ secondMasterKeyData.getMasterKey());
+ rmContext.getStateStore().storeOrUpdateAMRMTokenSecretManagerState(state2,
+ true);
+
+ // load state
+ store = stateStoreHelper.getRMStateStore();
+ store.setRMDispatcher(dispatcher);
+ RMState state_2 = store.loadState();
+ Assert.assertNotNull(state_2.getAMRMTokenSecretManagerState());
+ Assert.assertEquals(firstMasterKeyData.getMasterKey(), state_2
+ .getAMRMTokenSecretManagerState().getCurrentMasterKey());
+ Assert.assertEquals(secondMasterKeyData.getMasterKey(), state_2
+ .getAMRMTokenSecretManagerState().getNextMasterKey());
+
+ // re-create the masterKeyData based on the recovered masterkey
+ // should have the same secretKey
+ appTokenMgr.recover(state_2);
+ Assert.assertEquals(appTokenMgr.getCurrnetMasterKeyData().getSecretKey(),
+ firstMasterKeyData.getSecretKey());
+ Assert.assertEquals(appTokenMgr.getNextMasterKeyData().getSecretKey(),
+ secondMasterKeyData.getSecretKey());
+
+ store.close();
+ }
}
Modified:
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java?rev=1613788&r1=1613787&r2=1613788&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
Sun Jul 27 13:32:51 2014
@@ -38,7 +38,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion;
-import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.RMStateVersionPBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
@@ -161,6 +160,7 @@ public class TestFSRMStateStore extends
testEpoch(fsTester);
testAppDeletion(fsTester);
testDeleteStore(fsTester);
+ testAMRMTokenSecretManagerStateStore(fsTester);
} finally {
cluster.shutdown();
}
Modified:
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java?rev=1613788&r1=1613787&r2=1613788&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
Sun Jul 27 13:32:51 2014
@@ -123,6 +123,7 @@ public class TestZKRMStateStore extends
testEpoch(zkTester);
testAppDeletion(zkTester);
testDeleteStore(zkTester);
+ testAMRMTokenSecretManagerStateStore(zkTester);
}
private Configuration createHARMConf(
Modified:
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java?rev=1613788&r1=1613787&r2=1613788&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
Sun Jul 27 13:32:51 2014
@@ -193,7 +193,7 @@ public class TestRMAppTransitions {
this.rmContext =
new RMContextImpl(rmDispatcher,
containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
- null, new AMRMTokenSecretManager(conf),
+ null, new AMRMTokenSecretManager(conf, this.rmContext),
new RMContainerTokenSecretManager(conf),
new NMTokenSecretManagerInRM(conf),
new ClientToAMTokenSecretManagerInRM(),
Modified:
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1613788&r1=1613787&r2=1613788&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
Sun Jul 27 13:32:51 2014
@@ -134,7 +134,8 @@ public class TestRMAppAttemptTransitions
private RMAppAttempt applicationAttempt;
private Configuration conf = new Configuration();
- private AMRMTokenSecretManager amRMTokenManager = spy(new
AMRMTokenSecretManager(conf));
+ private AMRMTokenSecretManager amRMTokenManager =
+ spy(new AMRMTokenSecretManager(conf, rmContext));
private ClientToAMTokenSecretManagerInRM clientToAMTokenManager =
spy(new ClientToAMTokenSecretManagerInRM());
private NMTokenSecretManagerInRM nmTokenManager =
Modified:
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java?rev=1613788&r1=1613787&r2=1613788&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
Sun Jul 27 13:32:51 2014
@@ -86,13 +86,12 @@ public class TestUtils {
Configuration conf = new Configuration();
RMApplicationHistoryWriter writer =
mock(RMApplicationHistoryWriter.class);
- RMContext rmContext =
+ RMContextImpl rmContext =
new RMContextImpl(nullDispatcher, cae, null, null, null,
- new AMRMTokenSecretManager(conf),
+ new AMRMTokenSecretManager(conf, null),
new RMContainerTokenSecretManager(conf),
new NMTokenSecretManagerInRM(conf),
new ClientToAMTokenSecretManagerInRM(), writer);
-
return rmContext;
}
Modified:
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1613788&r1=1613787&r2=1613788&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
Sun Jul 27 13:32:51 2014
@@ -1221,6 +1221,79 @@ public class TestFairScheduler extends F
scheduler.getSchedulerApp(app4).getPreemptionContainers().isEmpty());
}
+ @Test
+ public void testPreemptionIsNotDelayedToNextRound() throws Exception {
+ conf.setLong(FairSchedulerConfiguration.PREEMPTION_INTERVAL, 5000);
+ conf.setLong(FairSchedulerConfiguration.WAIT_TIME_BEFORE_KILL, 10000);
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+ conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "false");
+
+ MockClock clock = new MockClock();
+ scheduler.setClock(clock);
+
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<queue name=\"queueA\">");
+ out.println("<weight>8</weight>");
+ out.println("<queue name=\"queueA1\" />");
+ out.println("<queue name=\"queueA2\" />");
+ out.println("</queue>");
+ out.println("<queue name=\"queueB\">");
+ out.println("<weight>2</weight>");
+ out.println("</queue>");
+ out.print("<fairSharePreemptionTimeout>10</fairSharePreemptionTimeout>");
+ out.println("</allocations>");
+ out.close();
+
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node of 8G
+ RMNode node1 = MockNodes.newNodeInfo(1,
+ Resources.createResource(8 * 1024, 8), 1, "127.0.0.1");
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+
+ // Run apps in queueA.A1 and queueB
+ ApplicationAttemptId app1 = createSchedulingRequest(1 * 1024, 1,
+ "queueA.queueA1", "user1", 7, 1);
+ // createSchedulingRequestExistingApplication(1 * 1024, 1, 2, app1);
+ ApplicationAttemptId app2 = createSchedulingRequest(1 * 1024, 1, "queueB",
+ "user2", 1, 1);
+
+ scheduler.update();
+
+ NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1);
+ for (int i = 0; i < 8; i++) {
+ scheduler.handle(nodeUpdate1);
+ }
+
+ // verify if the apps got the containers they requested
+ assertEquals(7,
scheduler.getSchedulerApp(app1).getLiveContainers().size());
+ assertEquals(1,
scheduler.getSchedulerApp(app2).getLiveContainers().size());
+
+ // Now submit an app in queueA.queueA2
+ ApplicationAttemptId app3 = createSchedulingRequest(1 * 1024, 1,
+ "queueA.queueA2", "user3", 7, 1);
+ scheduler.update();
+
+ // Let 11 sec pass
+ clock.tick(11);
+
+ scheduler.update();
+ Resource toPreempt = scheduler.resToPreempt(scheduler.getQueueManager()
+ .getLeafQueue("queueA.queueA2", false), clock.getTime());
+ assertEquals(2980, toPreempt.getMemory());
+
+ // verify if the 3 containers required by queueA2 are preempted in the same
+ // round
+ scheduler.preemptResources(toPreempt);
+ assertEquals(3, scheduler.getSchedulerApp(app1).getPreemptionContainers()
+ .size());
+ }
+
@Test (timeout = 5000)
/**
* Tests the timing of decision to preempt tasks.
Modified:
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java?rev=1613788&r1=1613787&r2=1613788&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java
Sun Jul 27 13:32:51 2014
@@ -184,8 +184,8 @@ public class TestAMRMTokens {
// The exception will still have the earlier appAttemptId as it picks
it
// up from the token.
Assert.assertTrue(t.getCause().getMessage().contains(
- "Password not found for ApplicationAttempt " +
- applicationAttemptId.toString()));
+ applicationAttemptId.toString()
+ + " not found in AMRMTokenSecretManager."));
}
} finally {
