Repository: hadoop Updated Branches: refs/heads/HDFS-7240 38dddad74 -> adc651044
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java new file mode 100644 index 0000000..078b8fd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java @@ -0,0 +1,297 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager; + +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class TestRMServerUtils { + @Test + public void testGetApplicableNodeCountForAMLocality() throws Exception { + List<NodeId> rack1Nodes = new ArrayList<>(); + for (int i = 0; i < 29; i++) { + rack1Nodes.add(NodeId.newInstance("host" + i, 1234)); + } + NodeId node1 = NodeId.newInstance("node1", 1234); + NodeId node2 = NodeId.newInstance("node2", 1234); + rack1Nodes.add(node2); + + YarnConfiguration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, false); + ResourceScheduler scheduler = Mockito.mock(ResourceScheduler.class); + Mockito.when(scheduler.getNumClusterNodes()).thenReturn(100); + Mockito.when(scheduler.getNodeIds("/rack1")).thenReturn(rack1Nodes); + Mockito.when(scheduler.getNodeIds("node1")) + .thenReturn(Collections.singletonList(node1)); + Mockito.when(scheduler.getNodeIds("node2")) + .thenReturn(Collections.singletonList(node2)); + RMContext rmContext = Mockito.mock(RMContext.class); + Mockito.when(rmContext.getScheduler()).thenReturn(scheduler); + + ResourceRequest anyReq = createResourceRequest(ResourceRequest.ANY, + true, null); + List<ResourceRequest> reqs = new ArrayList<>(); + reqs.add(anyReq); + Assert.assertEquals(100, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + + ResourceRequest rackReq = createResourceRequest("/rack1", true, null); + reqs.add(rackReq); + Assert.assertEquals(30, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + anyReq.setRelaxLocality(false); + Assert.assertEquals(30, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + rackReq.setRelaxLocality(false); + Assert.assertEquals(100, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + + ResourceRequest node1Req = createResourceRequest("node1", false, null); + reqs.add(node1Req); + Assert.assertEquals(100, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + node1Req.setRelaxLocality(true); + Assert.assertEquals(1, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + rackReq.setRelaxLocality(true); + Assert.assertEquals(31, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + + ResourceRequest node2Req = createResourceRequest("node2", false, null); + reqs.add(node2Req); + Assert.assertEquals(31, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + node2Req.setRelaxLocality(true); + Assert.assertEquals(31, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + rackReq.setRelaxLocality(false); + Assert.assertEquals(2, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + node1Req.setRelaxLocality(false); + Assert.assertEquals(1, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + node2Req.setRelaxLocality(false); + Assert.assertEquals(100, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + } + + @Test + public void testGetApplicableNodeCountForAMLabels() throws Exception { + Set<NodeId> noLabelNodes = new HashSet<>(); + for (int i = 0; i < 80; i++) { + noLabelNodes.add(NodeId.newInstance("host" + i, 1234)); + } + Set<NodeId> label1Nodes = new HashSet<>(); + for (int i = 80; i < 90; i++) { + label1Nodes.add(NodeId.newInstance("host" + i, 1234)); + } + label1Nodes.add(NodeId.newInstance("host101", 0)); + label1Nodes.add(NodeId.newInstance("host102", 0)); + Map<String, Set<NodeId>> label1NodesMap = new HashMap<>(); + label1NodesMap.put("label1", label1Nodes); + + YarnConfiguration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true); + ResourceScheduler scheduler = Mockito.mock(ResourceScheduler.class); + Mockito.when(scheduler.getNumClusterNodes()).thenReturn(100); + RMContext rmContext = Mockito.mock(RMContext.class); + Mockito.when(rmContext.getScheduler()).thenReturn(scheduler); + RMNodeLabelsManager labMan = Mockito.mock(RMNodeLabelsManager.class); + Mockito.when(labMan.getNodesWithoutALabel()).thenReturn(noLabelNodes); + Mockito.when(labMan.getLabelsToNodes(Collections.singleton("label1"))) + .thenReturn(label1NodesMap); + Mockito.when(rmContext.getNodeLabelManager()).thenReturn(labMan); + + ResourceRequest anyReq = createResourceRequest(ResourceRequest.ANY, + true, null); + List<ResourceRequest> reqs = new ArrayList<>(); + reqs.add(anyReq); + Assert.assertEquals(80, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + anyReq.setNodeLabelExpression("label1"); + Assert.assertEquals(10, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + } + + @Test + public void testGetApplicableNodeCountForAMLocalityAndLabels() + throws Exception { + List<NodeId> rack1Nodes = new ArrayList<>(); + for (int i = 0; i < 29; i++) { + rack1Nodes.add(NodeId.newInstance("host" + i, 1234)); + } + NodeId node1 = NodeId.newInstance("node1", 1234); + NodeId node2 = NodeId.newInstance("node2", 1234); + rack1Nodes.add(node2); + Set<NodeId> noLabelNodes = new HashSet<>(); + for (int i = 0; i < 19; i++) { + noLabelNodes.add(rack1Nodes.get(i)); + } + noLabelNodes.add(node2); + for (int i = 29; i < 89; i++) { + noLabelNodes.add(NodeId.newInstance("host" + i, 1234)); + } + Set<NodeId> label1Nodes = new HashSet<>(); + label1Nodes.add(node1); + for (int i = 89; i < 93; i++) { + label1Nodes.add(NodeId.newInstance("host" + i, 1234)); + } + for (int i = 19; i < 29; i++) { + label1Nodes.add(rack1Nodes.get(i)); + } + label1Nodes.add(NodeId.newInstance("host101", 0)); + label1Nodes.add(NodeId.newInstance("host102", 0)); + Map<String, Set<NodeId>> label1NodesMap = new HashMap<>(); + label1NodesMap.put("label1", label1Nodes); + + YarnConfiguration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true); + ResourceScheduler scheduler = Mockito.mock(ResourceScheduler.class); + Mockito.when(scheduler.getNumClusterNodes()).thenReturn(100); + Mockito.when(scheduler.getNodeIds("/rack1")).thenReturn(rack1Nodes); + Mockito.when(scheduler.getNodeIds("node1")) + .thenReturn(Collections.singletonList(node1)); + Mockito.when(scheduler.getNodeIds("node2")) + .thenReturn(Collections.singletonList(node2)); + RMContext rmContext = Mockito.mock(RMContext.class); + Mockito.when(rmContext.getScheduler()).thenReturn(scheduler); + RMNodeLabelsManager labMan = Mockito.mock(RMNodeLabelsManager.class); + Mockito.when(labMan.getNodesWithoutALabel()).thenReturn(noLabelNodes); + Mockito.when(labMan.getLabelsToNodes(Collections.singleton("label1"))) + .thenReturn(label1NodesMap); + Mockito.when(rmContext.getNodeLabelManager()).thenReturn(labMan); + + ResourceRequest anyReq = createResourceRequest(ResourceRequest.ANY, + true, null); + List<ResourceRequest> reqs = new ArrayList<>(); + reqs.add(anyReq); + Assert.assertEquals(80, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + + ResourceRequest rackReq = createResourceRequest("/rack1", true, null); + reqs.add(rackReq); + Assert.assertEquals(20, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + anyReq.setRelaxLocality(false); + Assert.assertEquals(20, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + rackReq.setRelaxLocality(false); + Assert.assertEquals(80, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + + ResourceRequest node1Req = createResourceRequest("node1", false, null); + reqs.add(node1Req); + Assert.assertEquals(80, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + node1Req.setRelaxLocality(true); + Assert.assertEquals(0, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + rackReq.setRelaxLocality(true); + Assert.assertEquals(20, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + + ResourceRequest node2Req = createResourceRequest("node2", false, null); + reqs.add(node2Req); + Assert.assertEquals(20, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + node2Req.setRelaxLocality(true); + Assert.assertEquals(20, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + rackReq.setRelaxLocality(false); + Assert.assertEquals(1, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + node1Req.setRelaxLocality(false); + Assert.assertEquals(1, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + node2Req.setRelaxLocality(false); + Assert.assertEquals(80, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + + anyReq.setNodeLabelExpression("label1"); + rackReq.setNodeLabelExpression("label1"); + node1Req.setNodeLabelExpression("label1"); + node2Req.setNodeLabelExpression("label1"); + anyReq.setRelaxLocality(true); + reqs = new ArrayList<>(); + reqs.add(anyReq); + Assert.assertEquals(15, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + + rackReq.setRelaxLocality(true); + reqs.add(rackReq); + Assert.assertEquals(10, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + anyReq.setRelaxLocality(false); + Assert.assertEquals(10, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + rackReq.setRelaxLocality(false); + Assert.assertEquals(15, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + + node1Req.setRelaxLocality(false); + reqs.add(node1Req); + Assert.assertEquals(15, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + node1Req.setRelaxLocality(true); + Assert.assertEquals(1, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + rackReq.setRelaxLocality(true); + Assert.assertEquals(11, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + + node2Req.setRelaxLocality(false); + reqs.add(node2Req); + Assert.assertEquals(11, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + node2Req.setRelaxLocality(true); + Assert.assertEquals(11, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + rackReq.setRelaxLocality(false); + Assert.assertEquals(1, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + node1Req.setRelaxLocality(false); + Assert.assertEquals(0, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + node2Req.setRelaxLocality(false); + Assert.assertEquals(15, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + } + + private ResourceRequest createResourceRequest(String resource, + boolean relaxLocality, String nodeLabel) { + return ResourceRequest.newInstance(Priority.newInstance(0), + resource, Resource.newInstance(1, 1), 1, relaxLocality, nodeLabel); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index 9be52c6..5246eb7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -57,7 +57,7 @@ import com.google.common.collect.Lists; public abstract class MockAsm extends MockApps { public static class ApplicationBase implements RMApp { - ResourceRequest amReq; + List<ResourceRequest> amReqs; @Override public String getUser() { throw new UnsupportedOperationException("Not supported yet."); @@ -204,8 +204,8 @@ public abstract class MockAsm extends MockApps { } @Override - public ResourceRequest getAMResourceRequest() { - return this.amReq; + public List<ResourceRequest> getAMResourceRequests() { + return this.amReqs; } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index 55e93c1..7005bca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -526,7 +526,8 @@ public class TestSystemMetricsPublisher { when(app.getAppNodeLabelExpression()).thenCallRealMethod(); ResourceRequest amReq = mock(ResourceRequest.class); when(amReq.getNodeLabelExpression()).thenReturn("high-mem"); - when(app.getAMResourceRequest()).thenReturn(amReq); + when(app.getAMResourceRequests()) + .thenReturn(Collections.singletonList(amReq)); when(app.getAmNodeLabelExpression()).thenCallRealMethod(); when(app.getApplicationPriority()).thenReturn(Priority.newInstance(10)); when(app.getCallerContext()) http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java index 118b6bc..9290ff8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp; import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Set; @@ -62,14 +63,14 @@ public class MockRMApp implements RMApp { StringBuilder diagnostics = new StringBuilder(); RMAppAttempt attempt; int maxAppAttempts = 1; - ResourceRequest amReq; + List<ResourceRequest> amReqs; public MockRMApp(int newid, long time, RMAppState newState) { finish = time; id = MockApps.newAppID(newid); state = newState; - amReq = ResourceRequest.newInstance(Priority.UNDEFINED, "0.0.0.0", - Resource.newInstance(0, 0), 1); + amReqs = Collections.singletonList(ResourceRequest.newInstance( + Priority.UNDEFINED, "0.0.0.0", Resource.newInstance(0, 0), 1)); } public MockRMApp(int newid, long time, RMAppState newState, String userName) { @@ -276,8 +277,8 @@ public class MockRMApp implements RMApp { } @Override - public ResourceRequest getAMResourceRequest() { - return this.amReq; + public List<ResourceRequest> getAMResourceRequests() { + return this.amReqs; } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index 4884851..5aa7af9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -30,8 +30,10 @@ import static org.mockito.Mockito.verify; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.Map; import org.apache.commons.logging.Log; @@ -271,7 +273,8 @@ public class TestRMAppTransitions { submissionContext.setAMContainerSpec(mock(ContainerLaunchContext.class)); RMApp application = new RMAppImpl(applicationId, rmContext, conf, name, user, queue, submissionContext, scheduler, masterService, - System.currentTimeMillis(), "YARN", null, mock(ResourceRequest.class)); + System.currentTimeMillis(), "YARN", null, + new ArrayList<ResourceRequest>()); testAppStartState(applicationId, user, name, queue, application); this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(), @@ -1024,9 +1027,9 @@ public class TestRMAppTransitions { submissionContext.getQueue(), submissionContext, scheduler, null, appState.getSubmitTime(), submissionContext.getApplicationType(), submissionContext.getApplicationTags(), - BuilderUtils.newResourceRequest( + Collections.singletonList(BuilderUtils.newResourceRequest( RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, - submissionContext.getResource(), 1)); + submissionContext.getResource(), 1))); Assert.assertEquals(RMAppState.NEW, application.getState()); RMAppEvent recoverEvent = http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index ced5bd9..9a4b6dc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -328,9 +328,9 @@ public class TestRMAppAttemptTransitions { applicationAttempt = new RMAppAttemptImpl(applicationAttemptId, spyRMContext, scheduler, masterService, submissionContext, new Configuration(), - BuilderUtils.newResourceRequest( + Collections.singletonList(BuilderUtils.newResourceRequest( RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, - submissionContext.getResource(), 1), application); + submissionContext.getResource(), 1)), application); when(application.getCurrentAppAttempt()).thenReturn(applicationAttempt); when(application.getApplicationId()).thenReturn(applicationId); @@ -1108,9 +1108,9 @@ public class TestRMAppAttemptTransitions { new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(), spyRMContext, scheduler,masterService, submissionContext, myConf, - BuilderUtils.newResourceRequest( + Collections.singletonList(BuilderUtils.newResourceRequest( RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, - submissionContext.getResource(), 1), application); + submissionContext.getResource(), 1)), application); //submit, schedule and allocate app attempt myApplicationAttempt.handle( @@ -1584,9 +1584,9 @@ public class TestRMAppAttemptTransitions { applicationAttempt = new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(), spyRMContext, scheduler, masterService, submissionContext, new Configuration(), - BuilderUtils.newResourceRequest( + Collections.singletonList(BuilderUtils.newResourceRequest( RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, - submissionContext.getResource(), 1), application); + submissionContext.getResource(), 1)), application); when(submissionContext.getKeepContainersAcrossApplicationAttempts()) .thenReturn(true); when(submissionContext.getMaxAppAttempts()).thenReturn(1); @@ -1645,9 +1645,10 @@ public class TestRMAppAttemptTransitions { applicationAttempt = new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(), spyRMContext, scheduler, masterService, submissionContext, - new Configuration(), ResourceRequest.newInstance( - Priority.UNDEFINED, "host1", Resource.newInstance(3333, 1), 3, - false, "label-expression"), application); + new Configuration(), Collections.singletonList( + ResourceRequest.newInstance(Priority.UNDEFINED, "host1", + Resource.newInstance(3333, 1), 3, + false, "label-expression")), application); new RMAppAttemptImpl.ScheduleTransition().transition( (RMAppAttemptImpl) applicationAttempt, null); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index bb0a123..8aca235 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -28,6 +28,7 @@ import static org.mockito.Mockito.when; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -613,7 +614,8 @@ public class TestApplicationLimits { ResourceRequest amResourceRequest = mock(ResourceRequest.class); Resource amResource = Resources.createResource(0, 0); when(amResourceRequest.getCapability()).thenReturn(amResource); - when(rmApp.getAMResourceRequest()).thenReturn(amResourceRequest); + when(rmApp.getAMResourceRequests()).thenReturn( + Collections.singletonList(amResourceRequest)); Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId)Matchers.any()); when(spyRMContext.getRMApps()).thenReturn(spyApps); RMAppAttempt rmAppAttempt = mock(RMAppAttempt.class); http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java index b70a359..0aac2ef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java @@ -25,6 +25,7 @@ import static org.mockito.Mockito.when; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -639,7 +640,8 @@ public class TestApplicationLimitsByPartition { ResourceRequest amResourceRequest = mock(ResourceRequest.class); Resource amResource = Resources.createResource(0, 0); when(amResourceRequest.getCapability()).thenReturn(amResource); - when(rmApp.getAMResourceRequest()).thenReturn(amResourceRequest); + when(rmApp.getAMResourceRequests()).thenReturn( + Collections.singletonList(amResourceRequest)); Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId) Matchers.any()); when(spyRMContext.getRMApps()).thenReturn(spyApps); RMAppAttempt rmAppAttempt = mock(RMAppAttempt.class); http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 293bac2..ff0f7cf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -3205,7 +3205,7 @@ public class TestCapacityScheduler { RMApp rmApp = rm.submitApp(amMemory, "app-1", "user_0", null, queueName); assertEquals("RMApp does not containes minimum allocation", - minAllocResource, rmApp.getAMResourceRequest().getCapability()); + minAllocResource, rmApp.getAMResourceRequests().get(0).getCapability()); ResourceScheduler scheduler = rm.getRMContext().getScheduler(); LeafQueue queueA = http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 3fbbae3..1162b9f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -151,7 +151,8 @@ public class TestLeafQueue { amResourceRequest = mock(ResourceRequest.class); when(amResourceRequest.getCapability()).thenReturn( Resources.createResource(0, 0)); - when(rmApp.getAMResourceRequest()).thenReturn(amResourceRequest); + when(rmApp.getAMResourceRequests()).thenReturn( + Collections.singletonList(amResourceRequest)); Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId)Matchers.any()); when(spyRMContext.getRMApps()).thenReturn(spyApps); http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index baf7434..9bf79f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -3204,6 +3204,84 @@ public class TestFairScheduler extends FairSchedulerTestBase { assertEquals(1, app.getLiveContainers().size()); } + @Test + public void testAMStrictLocalityRack() throws IOException { + testAMStrictLocality(false, false); + } + + @Test + public void testAMStrictLocalityNode() throws IOException { + testAMStrictLocality(true, false); + } + + @Test + public void testAMStrictLocalityRackInvalid() throws IOException { + testAMStrictLocality(false, true); + } + + @Test + public void testAMStrictLocalityNodeInvalid() throws IOException { + testAMStrictLocality(true, true); + } + + private void testAMStrictLocality(boolean node, boolean invalid) + throws IOException { + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 1, + "127.0.0.1"); + NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); + scheduler.handle(nodeEvent1); + + RMNode node2 = MockNodes.newNodeInfo(2, Resources.createResource(1024), 2, + "127.0.0.2"); + NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2); + scheduler.handle(nodeEvent2); + + List<ResourceRequest> reqs = new ArrayList<>(); + ResourceRequest nodeRequest = createResourceRequest(1024, + node2.getHostName(), 1, 1, true); + if (node && invalid) { + nodeRequest.setResourceName("invalid"); + } + ResourceRequest rackRequest = createResourceRequest(1024, + node2.getRackName(), 1, 1, !node); + if (!node && invalid) { + rackRequest.setResourceName("invalid"); + } + ResourceRequest anyRequest = createResourceRequest(1024, + ResourceRequest.ANY, 1, 1, false); + reqs.add(anyRequest); + reqs.add(rackRequest); + if (node) { + reqs.add(nodeRequest); + } + + ApplicationAttemptId attId1 = + createSchedulingRequest("queue1", "user1", reqs); + + scheduler.update(); + + NodeUpdateSchedulerEvent node2UpdateEvent = + new NodeUpdateSchedulerEvent(node2); + + FSAppAttempt app = scheduler.getSchedulerApp(attId1); + + // node2 should get the container + scheduler.handle(node2UpdateEvent); + if (invalid) { + assertEquals(0, app.getLiveContainers().size()); + assertEquals(0, scheduler.getNode(node2.getNodeID()).getNumContainers()); + assertEquals(0, scheduler.getNode(node1.getNodeID()).getNumContainers()); + } else { + assertEquals(1, app.getLiveContainers().size()); + assertEquals(1, scheduler.getNode(node2.getNodeID()).getNumContainers()); + assertEquals(0, scheduler.getNode(node1.getNodeID()).getNumContainers()); + } + } + /** * Strict locality requests shouldn't reserve resources on another node. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 30f25e9..fb9e8ed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1213,8 +1213,8 @@ public class TestRMWebServicesApps extends JerseyTestBase { assertEquals(app1.getApplicationId().toString(), appInfo.getAppId()); assertEquals(app1.getName(), appInfo.getName()); assertEquals(app1.createApplicationState(), appInfo.getState()); - assertEquals(app1.getAMResourceRequest().getCapability().getMemorySize(), - appInfo.getAllocatedMB()); + assertEquals(app1.getAMResourceRequests().get(0).getCapability() + .getMemorySize(), appInfo.getAllocatedMB()); rm.stop(); } @@ -1427,7 +1427,7 @@ public class TestRMWebServicesApps extends JerseyTestBase { expectedNumberOfElements++; appNodeLabelExpression = info.getString("appNodeLabelExpression"); } - if (app.getAMResourceRequest().getNodeLabelExpression() != null) { + if (app.getAMResourceRequests().get(0).getNodeLabelExpression() != null) { expectedNumberOfElements++; amNodeLabelExpression = info.getString("amNodeLabelExpression"); } @@ -1534,7 +1534,7 @@ public class TestRMWebServicesApps extends JerseyTestBase { app.getApplicationSubmissionContext().getNodeLabelExpression(), appNodeLabelExpression); assertEquals("unmanagedApplication doesn't match", - app.getAMResourceRequest().getNodeLabelExpression(), + app.getAMResourceRequests().get(0).getNodeLabelExpression(), amNodeLabelExpression); assertEquals("amRPCAddress", AppInfo.getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()), @@ -1561,7 +1561,7 @@ public class TestRMWebServicesApps extends JerseyTestBase { String nodeLabelExpression, int numContainers, boolean relaxLocality, int priority, String resourceName, long memory, long vCores, String executionType, boolean enforceExecutionType) { - ResourceRequest request = app.getAMResourceRequest(); + ResourceRequest request = app.getAMResourceRequests().get(0); assertEquals("nodeLabelExpression doesn't match", request.getNodeLabelExpression(), nodeLabelExpression); assertEquals("numContainers doesn't match", request.getNumContainers(), --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
