Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientTokens.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientTokens.java?rev=1437843&r1=1437842&r2=1437843&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientTokens.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientTokens.java Thu Jan 24 02:45:45 2013 @@ -52,8 +52,10 @@ import org.apache.hadoop.yarn.api.protoc import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ClientToken; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; @@ -67,6 +69,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.service.AbstractService; import org.apache.hadoop.yarn.util.BuilderUtils; +import org.apache.hadoop.yarn.util.ProtoUtils; import org.apache.hadoop.yarn.util.Records; import org.junit.Test; @@ -106,14 +109,14 @@ public class TestClientTokens { private static class CustomAM extends AbstractService implements CustomProtocol { - private final ApplicationId appId; + private final ApplicationAttemptId appAttemptId; private final String secretKey; private InetSocketAddress address; private boolean pinged = false; - public CustomAM(ApplicationId appId, String secretKeyStr) { + public CustomAM(ApplicationAttemptId appId, String secretKeyStr) { super("CustomAM"); - this.appId = appId; + this.appAttemptId = appId; this.secretKey = secretKeyStr; } @@ -128,7 +131,7 @@ public class TestClientTokens { ClientToAMTokenSecretManager secretManager = null; byte[] bytes = Base64.decodeBase64(this.secretKey); - secretManager = new ClientToAMTokenSecretManager(this.appId, bytes); + secretManager = new ClientToAMTokenSecretManager(this.appAttemptId, bytes); Server server; try { server = @@ -216,7 +219,7 @@ public class TestClientTokens { GetApplicationReportResponse reportResponse = rm.getClientRMService().getApplicationReport(request); ApplicationReport appReport = reportResponse.getApplicationReport(); - String clientTokenEncoded = appReport.getClientToken(); + ClientToken clientToken = appReport.getClientToken(); // Wait till AM is 'launched' int waitTime = 0; @@ -226,9 +229,11 @@ public class TestClientTokens { Assert.assertNotNull(containerManager.clientTokensSecret); // Start the AM with the correct shared-secret. + ApplicationAttemptId appAttemptId = + app.getAppAttempts().keySet().iterator().next(); + Assert.assertNotNull(appAttemptId); final CustomAM am = - new CustomAM(app.getApplicationId(), - containerManager.clientTokensSecret); + new CustomAM(appAttemptId, containerManager.clientTokensSecret); am.init(conf); am.start(); @@ -249,21 +254,19 @@ public class TestClientTokens { // Verify denial for a malicious user UserGroupInformation ugi = UserGroupInformation.createRemoteUser("me"); - Token<ClientTokenIdentifier> clientToken = - new Token<ClientTokenIdentifier>(); - clientToken.decodeFromUrlString(clientTokenEncoded); - // RPC layer client expects ip:port as service for tokens - SecurityUtil.setTokenService(clientToken, am.address); + Token<ClientTokenIdentifier> token = + ProtoUtils.convertFromProtoFormat(clientToken, am.address); // Malicious user, messes with appId ClientTokenIdentifier maliciousID = - new ClientTokenIdentifier(BuilderUtils.newApplicationId(app - .getApplicationId().getClusterTimestamp(), 42)); + new ClientTokenIdentifier(BuilderUtils.newApplicationAttemptId( + BuilderUtils.newApplicationId(app.getApplicationId() + .getClusterTimestamp(), 42), 43)); Token<ClientTokenIdentifier> maliciousToken = new Token<ClientTokenIdentifier>(maliciousID.getBytes(), - clientToken.getPassword(), clientToken.getKind(), - clientToken.getService()); + token.getPassword(), token.getKind(), + token.getService()); ugi.addToken(maliciousToken); try { @@ -297,7 +300,7 @@ public class TestClientTokens { // Now for an authenticated user ugi = UserGroupInformation.createRemoteUser("me"); - ugi.addToken(clientToken); + ugi.addToken(token); ugi.doAs(new PrivilegedExceptionAction<Void>() { @Override
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer?rev=1437843&r1=1437842&r2=1437843&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer Thu Jan 24 02:45:45 2013 @@ -1 +1,14 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# org.apache.hadoop.yarn.server.resourcemanager.security.TestDelegationTokenRenewer$Renewer Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java?rev=1437843&r1=1437842&r2=1437843&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java Thu Jan 24 02:45:45 2013 @@ -114,4 +114,13 @@ public class WebAppProxy extends Abstrac } super.stop(); } + + public void join() { + if(proxyServer != null) { + try { + proxyServer.join(); + } catch (InterruptedException e) { + } + } + } } Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java?rev=1437843&r1=1437842&r2=1437843&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java Thu Jan 24 02:45:45 2013 @@ -73,6 +73,14 @@ public class WebAppProxyServer extends C YarnConfiguration.PROXY_PRINCIPAL); } + /** + * Wait for service to finish. + * (Normally, it runs forever.) + */ + private void join() { + proxy.join(); + } + public static void main(String[] args) { Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); StringUtils.startupShutdownMessage(WebAppProxyServer.class, args, LOG); @@ -84,6 +92,7 @@ public class WebAppProxyServer extends C YarnConfiguration conf = new YarnConfiguration(); proxy.init(conf); proxy.start(); + proxy.join(); } catch (Throwable t) { LOG.fatal("Error starting Proxy server", t); System.exit(-1); Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1437843&r1=1437842&r2=1437843&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm Thu Jan 24 02:45:45 2013 @@ -132,20 +132,45 @@ Hadoop MapReduce Next Generation - Fair * Whether to use the username associated with the allocation as the default queue name, in the event that a queue name is not specified. If this is set to "false" or unset, all jobs have a shared default queue, called "default". + Defaults to true. * <<<yarn.scheduler.fair.preemption>>> * Whether to use preemption. Note that preemption is experimental in the current - version. + version. Defaults to false. * <<<yarn.scheduler.fair.sizebasedweight>>> * Whether to assign shares to individual apps based on their size, rather than - providing an equal share to all apps regardless of size. + providing an equal share to all apps regardless of size. Defaults to false. * <<<yarn.scheduler.fair.assignmultiple>>> - * Whether to allow multiple container assignments in one heartbeat. + * Whether to allow multiple container assignments in one heartbeat. Defaults + to false. + + * <<<yarn.scheduler.fair.max.assign>>> + + * If assignmultiple is true, the maximum amount of containers that can be + assigned in one heartbeat. Defaults to -1, which sets no limit. + + * <<<locality.threshold.node>>> + + * For applications that request containers on particular nodes, the number of + scheduling opportunities since the last container assignment to wait before + accepting a placement on another node. Expressed as a float between 0 and 1, + which, as a fraction of the cluster size, is the number of scheduling + opportunities to pass up. The default value of -1.0 means don't pass up any + scheduling opportunities. + + * <<<locality.threshold.rack>>> + + * For applications that request containers on particular racks, the number of + scheduling opportunities since the last container assignment to wait before + accepting a placement on another rack. Expressed as a float between 0 and 1, + which, as a fraction of the cluster size, is the number of scheduling + opportunities to pass up. The default value of -1.0 means don't pass up any + scheduling opportunities. Allocation file format @@ -166,6 +191,14 @@ Allocation file format * schedulingMode: either "fifo" or "fair" depending on the in-queue scheduling policy desired + * aclSubmitApps: a list of users that can submit apps to the queue. A (default) + value of "*" means that any users can submit apps. A queue inherits the ACL of + its parent, so if a queue2 descends from queue1, and user1 is in queue1's ACL, + and user2 is in queue2's ACL, then both users may submit to queue2. + + * minSharePreemptionTimeout: number of seconds the queue is under its minimum share + before it will try to preempt containers to take resources from other queues. + * <<User elements>>, which represent settings governing the behavior of individual users. They can contain a single property: maxRunningApps, a limit on the number of running apps for a particular user. @@ -173,6 +206,10 @@ Allocation file format * <<A userMaxAppsDefault element>>, which sets the default running app limit for any users whose limit is not otherwise specified. + * <<A fairSharePreemptionTimeout element>>, number of seconds a queue is under + its fair share before it will try to preempt containers to take resources from + other queues. + An example allocation file is given here: --- Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/pom.xml?rev=1437843&r1=1437842&r2=1437843&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/pom.xml (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/pom.xml Thu Jan 24 02:45:45 2013 @@ -149,9 +149,10 @@ <groupId>org.apache.rat</groupId> <artifactId>apache-rat-plugin</artifactId> <configuration> - <includes> - <include>pom.xml</include> - </includes> + <excludes> + <exclude>conf/slaves</exclude> + <exclude>conf/container-executor.cfg</exclude> + </excludes> </configuration> </plugin> <plugin> Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/pom.xml?rev=1437843&r1=1437842&r2=1437843&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/pom.xml (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/pom.xml Thu Jan 24 02:45:45 2013 @@ -213,9 +213,9 @@ <groupId>org.apache.rat</groupId> <artifactId>apache-rat-plugin</artifactId> <configuration> - <includes> - <include>pom.xml</include> - </includes> + <excludes> + <exclude>CHANGES.txt</exclude> + </excludes> </configuration> </plugin> </plugins>
