Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1611531&r1=1611530&r2=1611531&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Fri Jul 18 02:21:21 2014 @@ -53,10 +53,13 @@ import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl; @@ -77,11 +80,13 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; @@ -788,14 +793,14 @@ public class TestFairScheduler extends F scheduler.reinitialize(conf, resourceManager.getRMContext()); ApplicationAttemptId id11 = createAppAttemptId(1, 1); - scheduler.addApplication(id11.getApplicationId(), "root.queue1", "user1"); - scheduler.addApplicationAttempt(id11, false, true); + scheduler.addApplication(id11.getApplicationId(), "root.queue1", "user1", false); + scheduler.addApplicationAttempt(id11, false, false); ApplicationAttemptId id21 = createAppAttemptId(2, 1); - scheduler.addApplication(id21.getApplicationId(), "root.queue2", "user1"); - scheduler.addApplicationAttempt(id21, false, true); + scheduler.addApplication(id21.getApplicationId(), "root.queue2", "user1", false); + scheduler.addApplicationAttempt(id21, false, false); ApplicationAttemptId id22 = createAppAttemptId(2, 2); - scheduler.addApplication(id22.getApplicationId(), "root.queue2", "user1"); - scheduler.addApplicationAttempt(id22, false, true); + scheduler.addApplication(id22.getApplicationId(), "root.queue2", "user1", false); + scheduler.addApplicationAttempt(id22, false, false); int minReqSize = FairSchedulerConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB; @@ -1556,8 +1561,8 @@ public class TestFairScheduler extends F scheduler.handle(nodeEvent2); ApplicationAttemptId appId = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); - scheduler.addApplication(appId.getApplicationId(), "queue1", "user1"); - scheduler.addApplicationAttempt(appId, false, true); + scheduler.addApplication(appId.getApplicationId(), "queue1", "user1", false); + scheduler.addApplicationAttempt(appId, false, false); // 1 request with 2 nodes on the same rack. another request with 1 node on // a different rack @@ -1838,7 +1843,7 @@ public class TestFairScheduler extends F ApplicationAttemptId attId = ApplicationAttemptId.newInstance(applicationId, this.ATTEMPT_ID++); - scheduler.addApplication(attId.getApplicationId(), queue, user); + scheduler.addApplication(attId.getApplicationId(), queue, user, false); numTries = 0; while (application.getFinishTime() == 0 && numTries < MAX_TRIES) { @@ -2715,8 +2720,8 @@ public class TestFairScheduler extends F // send application request ApplicationAttemptId appAttemptId = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); - fs.addApplication(appAttemptId.getApplicationId(), "queue11", "user11"); - fs.addApplicationAttempt(appAttemptId, false, true); + fs.addApplication(appAttemptId.getApplicationId(), "queue11", "user11", false); + fs.addApplicationAttempt(appAttemptId, false, false); List<ResourceRequest> ask = new ArrayList<ResourceRequest>(); ResourceRequest request = createResourceRequest(1024, 1, ResourceRequest.ANY, 1, 1, true); @@ -2831,6 +2836,87 @@ public class TestFairScheduler extends F } } } + + @Test(timeout=5000) + public void testRecoverRequestAfterPreemption() throws Exception { + conf.setLong(FairSchedulerConfiguration.WAIT_TIME_BEFORE_KILL, 10); + + MockClock clock = new MockClock(); + scheduler.setClock(clock); + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + Priority priority = Priority.newInstance(20); + String host = "127.0.0.1"; + int GB = 1024; + + // Create Node and raised Node Added event + RMNode node = MockNodes.newNodeInfo(1, + Resources.createResource(16 * 1024, 4), 0, host); + NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node); + scheduler.handle(nodeEvent); + + // Create 3 container requests and place it in ask + List<ResourceRequest> ask = new ArrayList<ResourceRequest>(); + ResourceRequest nodeLocalRequest = createResourceRequest(GB, 1, host, + priority.getPriority(), 1, true); + ResourceRequest rackLocalRequest = createResourceRequest(GB, 1, + node.getRackName(), priority.getPriority(), 1, true); + ResourceRequest offRackRequest = createResourceRequest(GB, 1, + ResourceRequest.ANY, priority.getPriority(), 1, true); + ask.add(nodeLocalRequest); + ask.add(rackLocalRequest); + ask.add(offRackRequest); + + // Create Request and update + ApplicationAttemptId appAttemptId = createSchedulingRequest("queueA", + "user1", ask); + scheduler.update(); + + // Sufficient node check-ins to fully schedule containers + NodeUpdateSchedulerEvent nodeUpdate = new NodeUpdateSchedulerEvent(node); + scheduler.handle(nodeUpdate); + + assertEquals(1, scheduler.getSchedulerApp(appAttemptId).getLiveContainers() + .size()); + FSSchedulerApp app = scheduler.getSchedulerApp(appAttemptId); + + // ResourceRequest will be empty once NodeUpdate is completed + Assert.assertNull(app.getResourceRequest(priority, host)); + + ContainerId containerId1 = ContainerId.newInstance(appAttemptId, 1); + RMContainer rmContainer = app.getRMContainer(containerId1); + + // Create a preempt event and register for preemption + scheduler.warnOrKillContainer(rmContainer); + + // Wait for few clock ticks + clock.tick(5); + + // preempt now + scheduler.warnOrKillContainer(rmContainer); + + List<ResourceRequest> requests = rmContainer.getResourceRequests(); + // Once recovered, resource request will be present again in app + Assert.assertEquals(3, requests.size()); + for (ResourceRequest request : requests) { + Assert.assertEquals(1, + app.getResourceRequest(priority, request.getResourceName()) + .getNumContainers()); + } + + // Send node heartbeat + scheduler.update(); + scheduler.handle(nodeUpdate); + + List<Container> containers = scheduler.allocate(appAttemptId, + Collections.<ResourceRequest> emptyList(), + Collections.<ContainerId> emptyList(), null, null).getContainers(); + + // Now with updated ResourceRequest, a container is allocated for AM. + Assert.assertTrue(containers.size() == 1); + } @SuppressWarnings("resource") @Test
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm?rev=1611531&r1=1611530&r2=1611531&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm (original) +++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm Fri Jul 18 02:21:21 2014 @@ -2707,3 +2707,223 @@ Server: Jetty(6.1.26) +---+ +* Cluster {Delegation Tokens API} + + The Delegation Tokens API can be used to create, renew and cancel YARN ResourceManager delegation tokens. All delegation token requests must be carried out on a Kerberos authenticated connection(using SPNEGO). Carrying out operations on a non-kerberos connection will result in a FORBIDDEN response. In case of renewing a token, only the renewer specified when creating the token can renew the token. Other users(including the owner) are forbidden from renewing tokens. It should be noted that when cancelling or renewing a token, the token to be cancelled or renewed is specified by setting a header. + + This feature is currently in the alpha stage and may change in the future. + +** URI + + Use the following URI to create and cancel delegation tokens. + +------ + * http://<rm http address:port>/ws/v1/cluster/delegation-token +------ + + Use the following URI to renew delegation tokens. + +------ + * http://<rm http address:port>/ws/v1/cluster/delegation-token/expiration +------ + +** HTTP Operations Supported + +------ + * POST + * DELETE +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the <delegation-token> object + + The response from the delegation tokens API contains one of the fields listed below. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| token | string | The delegation token | +*---------------+--------------+-------------------------------+ +| renewer | string | The user who is allowed to renew the delegation token | +*---------------+--------------+-------------------------------+ +| owner | string | The owner of the delegation token | +*---------------+--------------+-------------------------------+ +| kind | string | The kind of delegation token | +*---------------+--------------+-------------------------------+ +| expiration-time | long | The expiration time of the token | +*---------------+--------------+-------------------------------+ +| max-validity | long | The maximum validity of the token | +*---------------+--------------+-------------------------------+ + +** Response Examples + +*** Creating a token + + <<JSON response>> + + HTTP Request: + +------ + POST http://<rm http address:port>/ws/v1/cluster/delegation-token + Accept: application/json + Content-Type: application/json + { + "renewer" : "test-renewer" + } +------ + + Response Header + ++---+ + HTTP/1.1 200 OK + WWW-Authenticate: Negotiate ... + Date: Sat, 28 Jun 2014 18:08:11 GMT + Server: Jetty(6.1.26) + Set-Cookie: ... + Content-Type: application/json ++---+ + + Response body + ++---+ + { + "token":"MgASY2xpZW50QEVYQU1QTEUuQ09NDHRlc3QtcmVuZXdlcgCKAUckiEZpigFHSJTKaQECFN9EMM9BzfPoDxu572EVUpzqhnSGE1JNX0RFTEVHQVRJT05fVE9LRU4A", + "renewer":"test-renewer", + "owner":"[email protected]", + "kind":"RM_DELEGATION_TOKEN", + "expiration-time":"1405153616489", + "max-validity":"1405672016489" + } ++---+ + + <<XML response>> + + HTTP Request + +------ + POST http://<rm http address:port>/ws/v1/cluster/delegation-token + Accept: application/xml + Content-Type: application/xml + <delegation-token> + <renewer>test-renewer</renewer> + </delegation-token> +------ + + Response Header + ++---+ + HTTP/1.1 200 OK + WWW-Authenticate: Negotiate ... + Date: Sat, 28 Jun 2014 18:08:11 GMT + Content-Length: 423 + Server: Jetty(6.1.26) + Set-Cookie: ... + Content-Type: application/xml ++---+ + + Response Body + ++---+ + <?xml version="1.0" encoding="UTF-8" standalone="yes"?> + <delegation-token> + <token>MgASY2xpZW50QEVYQU1QTEUuQ09NDHRlc3QtcmVuZXdlcgCKAUckgZ8yigFHSI4jMgcCFDTG8X6XFFn2udQngzSXQL8vWaKIE1JNX0RFTEVHQVRJT05fVE9LRU4A</token> + <renewer>test-renewer</renewer> + <owner>[email protected]</owner> + <kind>RM_DELEGATION_TOKEN</kind> + <expiration-time>1405153180466</expiration-time> + <max-validity>1405671580466</max-validity> + </delegation-token> ++---+ + +*** Renewing a token + + <<JSON response>> + + HTTP Request: + +------ + POST http://<rm http address:port>/ws/v1/cluster/delegation-token/expiration + Accept: application/json + Hadoop-YARN-RM-Delegation-Token: MgASY2xpZW50QEVYQU1QTEUuQ09NDHRlc3QtcmVuZXdlcgCKAUbjqcHHigFHB7ZFxwQCFKWD3znCkDSy6SQIjRCLDydxbxvgE1JNX0RFTEVHQVRJT05fVE9LRU4A + Content-Type: application/json +------ + + Response Header + ++---+ + HTTP/1.1 200 OK + WWW-Authenticate: Negotiate ... + Date: Sat, 28 Jun 2014 18:08:11 GMT + Server: Jetty(6.1.26) + Set-Cookie: ... + Content-Type: application/json ++---+ + + Response body + ++---+ + { + "expiration-time":"1404112520402" + } ++---+ + + <<XML response>> + + HTTP Request + +------ + POST http://<rm http address:port>/ws/v1/cluster/delegation-token/expiration + Accept: application/xml + Content-Type: application/xml + Hadoop-YARN-RM-Delegation-Token: MgASY2xpZW50QEVYQU1QTEUuQ09NDHRlc3QtcmVuZXdlcgCKAUbjqcHHigFHB7ZFxwQCFKWD3znCkDSy6SQIjRCLDydxbxvgE1JNX0RFTEVHQVRJT05fVE9LRU4A +------ + + Response Header + ++---+ + HTTP/1.1 200 OK + WWW-Authenticate: Negotiate ... + Date: Sat, 28 Jun 2014 18:08:11 GMT + Content-Length: 423 + Server: Jetty(6.1.26) + Set-Cookie: ... + Content-Type: application/xml ++---+ + + Response Body + ++---+ + <?xml version="1.0" encoding="UTF-8" standalone="yes"?> + <delegation-token> + <expiration-time>1404112520402</expiration-time> + </delegation-token> ++---+ + +*** Cancelling a token + + HTTP Request + +----- +DELETE http://<rm http address:port>/ws/v1/cluster/delegation-token +Hadoop-YARN-RM-Delegation-Token: MgASY2xpZW50QEVYQU1QTEUuQ09NDHRlc3QtcmVuZXdlcgCKAUbjqcHHigFHB7ZFxwQCFKWD3znCkDSy6SQIjRCLDydxbxvgE1JNX0RFTEVHQVRJT05fVE9LRU4A +Accept: application/xml +----- + + Response Header + ++---+ + HTTP/1.1 200 OK + WWW-Authenticate: Negotiate ... + Date: Sun, 29 Jun 2014 07:25:18 GMT + Transfer-Encoding: chunked + Server: Jetty(6.1.26) + Set-Cookie: ... + Content-Type: application/xml ++---+ + + No response body.
