Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java?rev=1467511&r1=1467510&r2=1467511&view=diff ============================================================================== --- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java (original) +++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java Fri Apr 12 23:05:28 2013 @@ -19,100 +19,225 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.impl.pb.ResourceRequestPBImpl; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.server.resourcemanager.resource.DominantResourceCalculator; import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; +import org.apache.hadoop.yarn.util.BuilderUtils; import org.junit.Test; public class TestSchedulerUtils { - @Test + @Test (timeout = 30000) public void testNormalizeRequest() { ResourceCalculator resourceCalculator = new DefaultResourceCalculator(); final int minMemory = 1024; + final int maxMemory = 8192; Resource minResource = Resources.createResource(minMemory, 0); + Resource maxResource = Resources.createResource(maxMemory, 0); ResourceRequest ask = new ResourceRequestPBImpl(); // case negative memory ask.setCapability(Resources.createResource(-1024)); - Resource before = ask.getCapability(); - SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource); - Resource after = ask.getCapability(); + SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, + maxResource); assertEquals(minMemory, ask.getCapability().getMemory()); - assertTrue(before == after); // case zero memory ask.setCapability(Resources.createResource(0)); - before = ask.getCapability(); - SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource); - after = ask.getCapability(); + SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, + maxResource); assertEquals(minMemory, ask.getCapability().getMemory()); - assertTrue(before == after); // case memory is a multiple of minMemory ask.setCapability(Resources.createResource(2 * minMemory)); - before = ask.getCapability(); - SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource); - after = ask.getCapability(); + SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, + maxResource); assertEquals(2 * minMemory, ask.getCapability().getMemory()); - assertTrue(before == after); // case memory is not a multiple of minMemory ask.setCapability(Resources.createResource(minMemory + 10)); - before = ask.getCapability(); - SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource); - after = ask.getCapability(); + SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, + maxResource); assertEquals(2 * minMemory, ask.getCapability().getMemory()); - assertTrue(before == after); + // case memory is equal to max allowed + ask.setCapability(Resources.createResource(maxMemory)); + SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, + maxResource); + assertEquals(maxMemory, ask.getCapability().getMemory()); + + // case memory is just less than max + ask.setCapability(Resources.createResource(maxMemory - 10)); + SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, + maxResource); + assertEquals(maxMemory, ask.getCapability().getMemory()); + + // max is not a multiple of min + maxResource = Resources.createResource(maxMemory - 10, 0); + ask.setCapability(Resources.createResource(maxMemory - 100)); + // multiple of minMemory > maxMemory, then reduce to maxMemory + SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, + maxResource); + assertEquals(maxResource.getMemory(), ask.getCapability().getMemory()); + + // ask is more than max + maxResource = Resources.createResource(maxMemory, 0); + ask.setCapability(Resources.createResource(maxMemory + 100)); + SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, + maxResource); + assertEquals(maxResource.getMemory(), ask.getCapability().getMemory()); } - @Test + @Test (timeout = 30000) public void testNormalizeRequestWithDominantResourceCalculator() { ResourceCalculator resourceCalculator = new DominantResourceCalculator(); Resource minResource = Resources.createResource(1024, 1); + Resource maxResource = Resources.createResource(10240, 10); Resource clusterResource = Resources.createResource(10 * 1024, 10); ResourceRequest ask = new ResourceRequestPBImpl(); // case negative memory/vcores ask.setCapability(Resources.createResource(-1024, -1)); - Resource before = ask.getCapability(); SchedulerUtils.normalizeRequest( - ask, resourceCalculator, clusterResource, minResource); - Resource after = ask.getCapability(); + ask, resourceCalculator, clusterResource, minResource, maxResource); assertEquals(minResource, ask.getCapability()); - assertTrue(before == after); // case zero memory/vcores ask.setCapability(Resources.createResource(0, 0)); - before = ask.getCapability(); SchedulerUtils.normalizeRequest( - ask, resourceCalculator, clusterResource, minResource); - after = ask.getCapability(); + ask, resourceCalculator, clusterResource, minResource, maxResource); assertEquals(minResource, ask.getCapability()); assertEquals(1, ask.getCapability().getVirtualCores()); assertEquals(1024, ask.getCapability().getMemory()); - assertTrue(before == after); // case non-zero memory & zero cores ask.setCapability(Resources.createResource(1536, 0)); - before = ask.getCapability(); SchedulerUtils.normalizeRequest( - ask, resourceCalculator, clusterResource, minResource); - after = ask.getCapability(); + ask, resourceCalculator, clusterResource, minResource, maxResource); assertEquals(Resources.createResource(2048, 1), ask.getCapability()); assertEquals(1, ask.getCapability().getVirtualCores()); assertEquals(2048, ask.getCapability().getMemory()); - assertTrue(before == after); } + + @Test (timeout = 30000) + public void testValidateResourceRequest() { + Resource maxResource = Resources.createResource( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); + + // zero memory + try { + Resource resource = Resources.createResource( + 0, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); + ResourceRequest resReq = BuilderUtils.newResourceRequest( + mock(Priority.class), ResourceRequest.ANY, resource, 1); + SchedulerUtils.validateResourceRequest(resReq, maxResource); + } catch (InvalidResourceRequestException e) { + fail("Zero memory should be accepted"); + } + + // zero vcores + try { + Resource resource = Resources.createResource( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + 0); + ResourceRequest resReq = BuilderUtils.newResourceRequest( + mock(Priority.class), ResourceRequest.ANY, resource, 1); + SchedulerUtils.validateResourceRequest(resReq, maxResource); + } catch (InvalidResourceRequestException e) { + fail("Zero vcores should be accepted"); + } + + // max memory + try { + Resource resource = Resources.createResource( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); + ResourceRequest resReq = BuilderUtils.newResourceRequest( + mock(Priority.class), ResourceRequest.ANY, resource, 1); + SchedulerUtils.validateResourceRequest(resReq, maxResource); + } catch (InvalidResourceRequestException e) { + fail("Max memory should be accepted"); + } + + // max vcores + try { + Resource resource = Resources.createResource( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); + ResourceRequest resReq = BuilderUtils.newResourceRequest( + mock(Priority.class), ResourceRequest.ANY, resource, 1); + SchedulerUtils.validateResourceRequest(resReq, maxResource); + } catch (InvalidResourceRequestException e) { + fail("Max vcores should not be accepted"); + } + + // negative memory + try { + Resource resource = Resources.createResource( + -1, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); + ResourceRequest resReq = BuilderUtils.newResourceRequest( + mock(Priority.class), ResourceRequest.ANY, resource, 1); + SchedulerUtils.validateResourceRequest(resReq, maxResource); + fail("Negative memory should not be accepted"); + } catch (InvalidResourceRequestException e) { + // expected + } + + // negative vcores + try { + Resource resource = Resources.createResource( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + -1); + ResourceRequest resReq = BuilderUtils.newResourceRequest( + mock(Priority.class), ResourceRequest.ANY, resource, 1); + SchedulerUtils.validateResourceRequest(resReq, maxResource); + fail("Negative vcores should not be accepted"); + } catch (InvalidResourceRequestException e) { + // expected + } + + // more than max memory + try { + Resource resource = Resources.createResource( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB + 1, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); + ResourceRequest resReq = BuilderUtils.newResourceRequest( + mock(Priority.class), ResourceRequest.ANY, resource, 1); + SchedulerUtils.validateResourceRequest(resReq, maxResource); + fail("More than max memory should not be accepted"); + } catch (InvalidResourceRequestException e) { + // expected + } + + // more than max vcores + try { + Resource resource = Resources.createResource( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES + + 1); + ResourceRequest resReq = BuilderUtils.newResourceRequest( + mock(Priority.class), ResourceRequest.ANY, resource, 1); + SchedulerUtils.validateResourceRequest(resReq, maxResource); + fail("More than max vcores should not be accepted"); + } catch (InvalidResourceRequestException e) { + // expected + } + } + }
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java?rev=1467511&r1=1467510&r2=1467511&view=diff ============================================================================== --- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java (original) +++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java Fri Apr 12 23:05:28 2013 @@ -19,8 +19,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import java.io.IOException; +import java.util.Comparator; import java.util.List; import junit.framework.Assert; @@ -43,13 +45,19 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; +import static org.mockito.Mockito.*; + public class TestCapacityScheduler { private static final Log LOG = LogFactory.getLog(TestCapacityScheduler.class); @@ -452,5 +460,33 @@ public class TestCapacityScheduler { } return result; } + + + @Test (timeout = 5000) + public void testApplicationComparator() + { + CapacityScheduler cs = new CapacityScheduler(); + Comparator<FiCaSchedulerApp> appComparator= cs.getApplicationComparator(); + ApplicationId id1 = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(ApplicationId.class); + id1.setClusterTimestamp(1); + id1.setId(1); + ApplicationId id2 = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(ApplicationId.class); + id2.setClusterTimestamp(1); + id2.setId(2); + ApplicationId id3 = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(ApplicationId.class); + id3.setClusterTimestamp(2); + id3.setId(1); + //same clusterId + FiCaSchedulerApp app1 = Mockito.mock(FiCaSchedulerApp.class); + when(app1.getApplicationId()).thenReturn(id1); + FiCaSchedulerApp app2 = Mockito.mock(FiCaSchedulerApp.class); + when(app2.getApplicationId()).thenReturn(id2); + FiCaSchedulerApp app3 = Mockito.mock(FiCaSchedulerApp.class); + when(app3.getApplicationId()).thenReturn(id3); + assertTrue(appComparator.compare(app1, app2) < 0); + //different clusterId + assertTrue(appComparator.compare(app1, app3) < 0); + assertTrue(appComparator.compare(app2, app3) < 0); + } } Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1467511&r1=1467510&r2=1467511&view=diff ============================================================================== --- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java (original) +++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java Fri Apr 12 23:05:28 2013 @@ -1623,6 +1623,30 @@ public class TestLeafQueue { assertEquals(3, e.activeApplications.size()); assertEquals(0, e.pendingApplications.size()); } + + @Test (timeout = 30000) + public void testNodeLocalityAfterQueueRefresh() throws Exception { + + // Manipulate queue 'e' + LeafQueue e = stubLeafQueue((LeafQueue)queues.get(E)); + + // before reinitialization + assertEquals(0, e.getNodeLocalityDelay()); + + csConf.setInt(CapacitySchedulerConfiguration + .NODE_LOCALITY_DELAY, 60); + Map<String, CSQueue> newQueues = new HashMap<String, CSQueue>(); + CSQueue newRoot = + CapacityScheduler.parseQueue(csContext, csConf, null, + CapacitySchedulerConfiguration.ROOT, + newQueues, queues, + TestUtils.spyHook); + queues = newQueues; + root.reinitialize(newRoot, cs.getClusterResources()); + + // after reinitialization + assertEquals(60, e.getNodeLocalityDelay()); + } @Test (timeout = 30000) public void testActivateApplicationByUpdatingClusterResource() Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java?rev=1467511&r1=1467510&r2=1467511&view=diff ============================================================================== --- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java (original) +++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java Fri Apr 12 23:05:28 2013 @@ -49,6 +49,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; +import org.apache.hadoop.yarn.util.BuilderUtils; public class TestUtils { private static final Log LOG = LogFactory.getLog(TestUtils.class); @@ -136,9 +137,7 @@ public class TestUtils { public static ApplicationAttemptId getMockApplicationAttemptId(int appId, int attemptId) { - ApplicationId applicationId = mock(ApplicationId.class); - when(applicationId.getClusterTimestamp()).thenReturn(0L); - when(applicationId.getId()).thenReturn(appId); + ApplicationId applicationId = BuilderUtils.newApplicationId(0l, appId); ApplicationAttemptId applicationAttemptId = mock(ApplicationAttemptId.class); when(applicationAttemptId.getApplicationId()).thenReturn(applicationId); when(applicationAttemptId.getAttemptId()).thenReturn(attemptId); Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1467511&r1=1467510&r2=1467511&view=diff ============================================================================== --- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original) +++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Fri Apr 12 23:05:28 2013 @@ -45,6 +45,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; @@ -72,6 +73,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.modes.FifoSchedulingMode; +import org.apache.hadoop.yarn.util.BuilderUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -110,6 +112,8 @@ public class TestFairScheduler { public void setUp() throws IOException { scheduler = new FairScheduler(); Configuration conf = createConfiguration(); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1024); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 10240); // All tests assume only one assignment per node update conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false"); resourceManager = new ResourceManager(); @@ -1404,6 +1408,10 @@ public class TestFairScheduler { ApplicationMasterService masterService = new ApplicationMasterService(resourceManager.getRMContext(), scheduler); ApplicationSubmissionContext submissionContext = new ApplicationSubmissionContextPBImpl(); + ContainerLaunchContext clc = + BuilderUtils.newContainerLaunchContext(user, null, null, null, null, + null, null); + submissionContext.setAMContainerSpec(clc); RMApp application = new RMAppImpl(applicationId, resourceManager.getRMContext(), conf, name, user, queue, submissionContext, scheduler, masterService, Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java?rev=1467511&r1=1467510&r2=1467511&view=diff ============================================================================== --- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java (original) +++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java Fri Apr 12 23:05:28 2013 @@ -19,6 +19,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import junit.framework.Assert; @@ -28,6 +30,7 @@ import org.apache.hadoop.conf.Configurat import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.Resource; @@ -35,15 +38,22 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.InlineDispatcher; +import org.apache.hadoop.yarn.factories.RecordFactory; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.Application; +import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.Task; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; import org.apache.hadoop.yarn.util.BuilderUtils; import org.junit.After; @@ -55,6 +65,9 @@ public class TestFifoScheduler { private ResourceManager resourceManager = null; + private static final RecordFactory recordFactory = + RecordFactoryProvider.getRecordFactory(null); + @Before public void setUp() throws Exception { resourceManager = new ResourceManager(); @@ -78,14 +91,38 @@ public class TestFifoScheduler { .getRMContext()); } - @Test + private ApplicationAttemptId createAppAttemptId(int appId, int attemptId) { + ApplicationAttemptId attId = recordFactory + .newRecordInstance(ApplicationAttemptId.class); + ApplicationId appIdImpl = recordFactory + .newRecordInstance(ApplicationId.class); + appIdImpl.setId(appId); + attId.setAttemptId(attemptId); + attId.setApplicationId(appIdImpl); + return attId; + } + + private ResourceRequest createResourceRequest(int memory, String host, + int priority, int numContainers) { + ResourceRequest request = recordFactory + .newRecordInstance(ResourceRequest.class); + request.setCapability(Resources.createResource(memory)); + request.setHostName(host); + request.setNumContainers(numContainers); + Priority prio = recordFactory.newRecordInstance(Priority.class); + prio.setPriority(priority); + request.setPriority(prio); + return request; + } + + @Test(timeout=5000) public void testFifoSchedulerCapacityWhenNoNMs() { FifoScheduler scheduler = new FifoScheduler(); QueueInfo queueInfo = scheduler.getQueueInfo(null, false, false); Assert.assertEquals(0.0f, queueInfo.getCurrentCapacity()); } - @Test + @Test(timeout=5000) public void testAppAttemptMetrics() throws Exception { AsyncDispatcher dispatcher = new InlineDispatcher(); RMContext rmContext = new RMContextImpl(dispatcher, null, @@ -111,6 +148,59 @@ public class TestFifoScheduler { Assert.assertEquals(1, metrics.getAppsSubmitted()); } + @Test(timeout=2000) + public void testNodeLocalAssignment() throws Exception { + AsyncDispatcher dispatcher = new InlineDispatcher(); + RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, + null, null, null); + + FifoScheduler scheduler = new FifoScheduler(); + scheduler.reinitialize(new Configuration(), rmContext); + + RMNode node0 = MockNodes.newNodeInfo(1, + Resources.createResource(1024 * 64), 1234); + NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0); + scheduler.handle(nodeEvent1); + + int _appId = 1; + int _appAttemptId = 1; + ApplicationAttemptId appAttemptId = createAppAttemptId(_appId, + _appAttemptId); + AppAddedSchedulerEvent appEvent1 = new AppAddedSchedulerEvent(appAttemptId, + "queue1", "user1"); + scheduler.handle(appEvent1); + + int memory = 64; + int nConts = 3; + int priority = 20; + + List<ResourceRequest> ask = new ArrayList<ResourceRequest>(); + ResourceRequest nodeLocal = createResourceRequest(memory, + node0.getHostName(), priority, nConts); + ResourceRequest rackLocal = createResourceRequest(memory, + node0.getRackName(), priority, nConts); + ResourceRequest any = createResourceRequest(memory, ResourceRequest.ANY, priority, + nConts); + ask.add(nodeLocal); + ask.add(rackLocal); + ask.add(any); + scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>()); + + NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0); + + // Before the node update event, there are 3 local requests outstanding + Assert.assertEquals(3, nodeLocal.getNumContainers()); + + scheduler.handle(node0Update); + + // After the node update event, check that there are no more local requests + // outstanding + Assert.assertEquals(0, nodeLocal.getNumContainers()); + //Also check that the containers were scheduled + SchedulerAppReport info = scheduler.getSchedulerAppInfo(appAttemptId); + Assert.assertEquals(3, info.getLiveContainers().size()); + } + // @Test public void testFifoScheduler() throws Exception { Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java?rev=1467511&r1=1467510&r2=1467511&view=diff ============================================================================== --- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java (original) +++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java Fri Apr 12 23:05:28 2013 @@ -1079,8 +1079,9 @@ public class TestRMWebServicesApps exten .getMasterContainer().getNodeId().toString(), nodeId); assertTrue("logsLink doesn't match", logsLink.startsWith("http://";)); - assertTrue("logsLink doesn't contain user info", - logsLink.endsWith("/" + appAttempt.getSubmissionContext().getUser())); + assertTrue( + "logsLink doesn't contain user info", logsLink.endsWith("/" + + appAttempt.getSubmissionContext().getAMContainerSpec().getUser())); } } Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java?rev=1467511&r1=1467510&r2=1467511&view=diff ============================================================================== --- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java (original) +++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java Fri Apr 12 23:05:28 2013 @@ -357,8 +357,13 @@ public class TestContainerManagerSecurit LOG.info("Going to contact NM with expired token"); ContainerLaunchContext context = createContainerLaunchContextForTest(newTokenId); + Container container = + BuilderUtils.newContainer(newTokenId.getContainerID(), null, null, + BuilderUtils.newResource(newTokenId.getResource().getMemory(), + newTokenId.getResource().getVirtualCores()), null, null); StartContainerRequest request = Records.newRecord(StartContainerRequest.class); request.setContainerLaunchContext(context); + request.setContainer(container); //Calling startContainer with an expired token. try { @@ -402,18 +407,19 @@ public class TestContainerManagerSecurit Arrays.asList("ping", "-n", "100", "127.0.0.1", ">nul") : Arrays.asList("sleep", "100"); - ContainerLaunchContext amContainer = BuilderUtils - .newContainerLaunchContext(null, "testUser", BuilderUtils - .newResource(1024, 1), Collections.<String, LocalResource>emptyMap(), - new HashMap<String, String>(), cmd, - new HashMap<String, ByteBuffer>(), null, - new HashMap<ApplicationAccessType, String>()); + ContainerLaunchContext amContainer = + BuilderUtils.newContainerLaunchContext("testUser", + Collections.<String, LocalResource> emptyMap(), + new HashMap<String, String>(), cmd, + new HashMap<String, ByteBuffer>(), null, + new HashMap<ApplicationAccessType, String>()); ApplicationSubmissionContext appSubmissionContext = recordFactory .newRecordInstance(ApplicationSubmissionContext.class); appSubmissionContext.setApplicationId(appID); - appSubmissionContext.setUser("testUser"); appSubmissionContext.setAMContainerSpec(amContainer); + appSubmissionContext.getAMContainerSpec().setUser("testUser"); + appSubmissionContext.setResource(BuilderUtils.newResource(1024, 1)); SubmitApplicationRequest submitRequest = recordFactory .newRecordInstance(SubmitApplicationRequest.class); @@ -539,8 +545,11 @@ public class TestContainerManagerSecurit // Authenticated but unauthorized, due to wrong resource ContainerLaunchContext context = createContainerLaunchContextForTest(tokenId); - context.getResource().setMemory(2048); // Set a different resource size. + Container container = + BuilderUtils.newContainer(tokenId.getContainerID(), null, null, + BuilderUtils.newResource(2048, 1), null, null); request.setContainerLaunchContext(context); + request.setContainer(container); try { client.startContainer(request); fail("Connection initiation with unauthorized " @@ -551,7 +560,7 @@ public class TestContainerManagerSecurit "Unauthorized request to start container. ")); Assert.assertTrue(e.getMessage().contains( "\nExpected resource " + tokenId.getResource().toString() - + " but found " + context.getResource().toString())); + + " but found " + container.getResource().toString())); } } @@ -563,7 +572,12 @@ public class TestContainerManagerSecurit ContainerLaunchContext context = createContainerLaunchContextForTest(tokenId); context.setUser("Saruman"); // Set a different user-name. + Container container = + BuilderUtils.newContainer(tokenId.getContainerID(), null, null, + BuilderUtils.newResource(tokenId.getResource().getMemory(), tokenId + .getResource().getVirtualCores()), null, null); request.setContainerLaunchContext(context); + request.setContainer(container); try { client.startContainer(request); fail("Connection initiation with unauthorized " @@ -581,12 +595,8 @@ public class TestContainerManagerSecurit private ContainerLaunchContext createContainerLaunchContextForTest( ContainerTokenIdentifier tokenId) { ContainerLaunchContext context = - BuilderUtils.newContainerLaunchContext(tokenId.getContainerID(), - "testUser", - BuilderUtils.newResource( - tokenId.getResource().getMemory(), - tokenId.getResource().getVirtualCores()), - new HashMap<String, LocalResource>(), + BuilderUtils.newContainerLaunchContext( + "testUser", new HashMap<String, LocalResource>(), new HashMap<String, String>(), new ArrayList<String>(), new HashMap<String, ByteBuffer>(), null, new HashMap<ApplicationAccessType, String>()); Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestDiskFailures.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestDiskFailures.java?rev=1467511&r1=1467510&r2=1467511&view=diff ============================================================================== --- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestDiskFailures.java (original) +++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestDiskFailures.java Fri Apr 12 23:05:28 2013 @@ -133,10 +133,10 @@ public class TestDiskFailures { dirSvc.init(conf); List<String> localDirs = dirSvc.getLocalDirs(); Assert.assertEquals(1, localDirs.size()); - Assert.assertEquals(localDir2, localDirs.get(0)); + Assert.assertEquals(new Path(localDir2).toString(), localDirs.get(0)); List<String> logDirs = dirSvc.getLogDirs(); Assert.assertEquals(1, logDirs.size()); - Assert.assertEquals(logDir1, logDirs.get(0)); + Assert.assertEquals(new Path(logDir1).toString(), logDirs.get(0)); } private void testDirsFailures(boolean localORLogDirs) throws IOException { Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1467511&r1=1467510&r2=1467511&view=diff ============================================================================== --- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm (original) +++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm Fri Apr 12 23:05:28 2013 @@ -85,8 +85,9 @@ Hadoop MapReduce Next Generation - Fair cause too much intermediate data to be created or too much context-switching. Limiting the apps does not cause any subsequently submitted apps to fail, only to wait in the scheduler's queue until some of the user's earlier apps - finish. apps to run from each user/queue are chosen in order of priority and - then submit time, as in the default FIFO scheduler in Hadoop. + finish. Apps to run from each user/queue are chosen in the same fair sharing + manner, but can alternatively be configured to be chosen in order of submit + time, as in the default FIFO scheduler in Hadoop. Certain add-ons are not yet supported which existed in the original (MR1) Fair Scheduler. Among them, is the use of a custom policies governing @@ -142,7 +143,9 @@ Hadoop MapReduce Next Generation - Fair * <<<yarn.scheduler.fair.sizebasedweight>>> * Whether to assign shares to individual apps based on their size, rather than - providing an equal share to all apps regardless of size. Defaults to false. + providing an equal share to all apps regardless of size. When set to true, + apps are weighted by the natural logarithm of one plus the app's total + requested memory, divided by the natural logarithm of 2. Defaults to false. * <<<yarn.scheduler.fair.assignmultiple>>> @@ -180,16 +183,29 @@ Allocation file format * <<Queue elements>>, which represent queues. Each may contain the following properties: - * minResources: minimum amount of aggregate memory + * minResources: minimum MB of aggregate memory the queue expects. If a queue + demands resources, and its current allocation is below its configured minimum, + it will be assigned available resources before any queue that is not in this + situation. If multiple queues are in this situation, resources go to the + queue with the smallest ratio between allocation and minimum. Note that it is + possible that a queue that is below its minimum may not immediately get up to + its minimum when it submits an application, because already-running jobs may + be using those resources. - * maxResources: maximum amount of aggregate memory + * maxResources: maximum MB of aggregate memory a queue is allowed. A queue + will never be assigned a container that would put it over this limit. * maxRunningApps: limit the number of apps from the queue to run at once - * weight: to share the cluster non-proportionally with other queues + * weight: to share the cluster non-proportionally with other queues. Weights + default to 1, and a queue with weight 2 should receive approximately twice + as many resources as a queue with the default weight. * schedulingMode: either "fifo" or "fair" depending on the in-queue scheduling - policy desired + policy desired. Defaults to "fair". If "fifo", apps with earlier submit + times are given preference for containers, but apps submitted later may + run concurrently if there is leftover space on the cluster after satisfying + the earlier app's requests. * aclSubmitApps: a list of users that can submit apps to the queue. A (default) value of "*" means that any users can submit apps. A queue inherits the ACL of
