Repository: hadoop Updated Branches: refs/heads/trunk 1268cf5fb -> ecce3b7d5
MAPREDUCE-6677. LocalContainerAllocator doesn't specify resource of the containers allocated (haibochen via rkanter) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ecce3b7d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ecce3b7d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ecce3b7d Branch: refs/heads/trunk Commit: ecce3b7d53ab624c9931273716326c76641ea11d Parents: 1268cf5 Author: Robert Kanter <rkan...@apache.org> Authored: Wed May 4 22:19:20 2016 -0700 Committer: Robert Kanter <rkan...@apache.org> Committed: Wed May 4 22:19:20 2016 -0700 ---------------------------------------------------------------------- .../v2/app/local/LocalContainerAllocator.java | 2 + .../app/local/TestLocalContainerAllocator.java | 43 ++++++++++++++++++++ .../apache/hadoop/mapreduce/v2/TestMRJobs.java | 9 ---- 3 files changed, 45 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecce3b7d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java index 7437357..269e5f4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java @@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.client.ClientRMProxy; @@ -177,6 +178,7 @@ public class LocalContainerAllocator extends RMCommunicator Container container = recordFactory.newRecordInstance(Container.class); container.setId(cID); NodeId nodeId = NodeId.newInstance(this.nmHost, this.nmPort); + container.setResource(Resource.newInstance(0, 0)); container.setNodeId(nodeId); container.setContainerToken(null); container.setNodeHttpAddress(this.nmHost + ":" + this.nmHttpPort); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecce3b7d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java index 38df8f0..0f7dc87 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java @@ -19,6 +19,8 @@ package org.apache.hadoop.mapreduce.v2.app.local; import static org.mockito.Matchers.isA; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.io.IOException; @@ -29,10 +31,15 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.v2.api.records.JobId; +import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId; +import org.apache.hadoop.mapreduce.v2.api.records.TaskId; import org.apache.hadoop.mapreduce.v2.app.AppContext; import org.apache.hadoop.mapreduce.v2.app.ClusterInfo; import org.apache.hadoop.mapreduce.v2.app.client.ClientService; import org.apache.hadoop.mapreduce.v2.app.job.Job; +import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerAssignedEvent; +import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator; +import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocatorEvent; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; @@ -46,12 +53,14 @@ import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRespo import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.NMToken; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.client.ClientRMProxy; +import org.apache.hadoop.yarn.event.Event; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; @@ -60,6 +69,7 @@ import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.Assert; import org.junit.Test; +import org.mockito.ArgumentCaptor; public class TestLocalContainerAllocator { @@ -170,6 +180,39 @@ public class TestLocalContainerAllocator { ugiToken.getService()); } + @Test + public void testAllocatedContainerResourceIsNotNull() { + ArgumentCaptor<TaskAttemptContainerAssignedEvent> containerAssignedCaptor + = ArgumentCaptor.forClass(TaskAttemptContainerAssignedEvent.class); + @SuppressWarnings("unchecked") + EventHandler<Event> eventHandler = mock(EventHandler.class); + AppContext context = mock(AppContext.class) ; + when(context.getEventHandler()).thenReturn(eventHandler); + ContainerId containerId = ContainerId.fromString( + "container_1427562107907_0002_01_000001"); + LocalContainerAllocator containerAllocator = new LocalContainerAllocator( + mock(ClientService.class), context, "localhost", -1, -1, containerId); + + ContainerAllocatorEvent containerAllocatorEvent = + createContainerRequestEvent(); + containerAllocator.handle(containerAllocatorEvent); + + verify(eventHandler, times(1)).handle(containerAssignedCaptor.capture()); + Container container = containerAssignedCaptor.getValue().getContainer(); + Resource containerResource = container.getResource(); + Assert.assertNotNull(containerResource); + Assert.assertEquals(containerResource.getMemory(), 0); + Assert.assertEquals(containerResource.getVirtualCores(), 0); + } + + private static ContainerAllocatorEvent createContainerRequestEvent() { + TaskAttemptId taskAttemptId = mock(TaskAttemptId.class); + TaskId taskId = mock(TaskId.class); + when(taskAttemptId.getTaskId()).thenReturn(taskId); + return new ContainerAllocatorEvent(taskAttemptId, + ContainerAllocator.EventType.CONTAINER_REQ); + } + private static class StubbedLocalContainerAllocator extends LocalContainerAllocator { private ApplicationMasterProtocol scheduler; http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecce3b7d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java index e685c65..ba05d9d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java @@ -471,12 +471,6 @@ public class TestMRJobs { .getValue()); Assert.assertEquals(numSleepReducers, counters.findCounter(JobCounter.TOTAL_LAUNCHED_REDUCES).getValue()); - Assert - .assertTrue(counters.findCounter(JobCounter.SLOTS_MILLIS_MAPS) != null - && counters.findCounter(JobCounter.SLOTS_MILLIS_MAPS).getValue() != 0); - Assert - .assertTrue(counters.findCounter(JobCounter.SLOTS_MILLIS_MAPS) != null - && counters.findCounter(JobCounter.SLOTS_MILLIS_MAPS).getValue() != 0); } protected void verifyTaskProgress(Job job) throws InterruptedException, @@ -548,9 +542,6 @@ public class TestMRJobs { .getValue()); Assert.assertEquals(3, counters.findCounter(JobCounter.TOTAL_LAUNCHED_MAPS) .getValue()); - Assert - .assertTrue(counters.findCounter(JobCounter.SLOTS_MILLIS_MAPS) != null - && counters.findCounter(JobCounter.SLOTS_MILLIS_MAPS).getValue() != 0); } @Test (timeout = 60000) --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
