http://git-wip-us.apache.org/repos/asf/hadoop/blob/d199274b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocation.java index 27d8661..d7124bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocation.java @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; import com.google.common.collect.ImmutableSet; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagNamespace; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceSizing; @@ -224,6 +225,131 @@ public class TestSchedulingRequestContainerAllocation { rm1.close(); } + /** + * This UT covers some basic end-to-end inter-app anti-affinity + * constraint tests. For comprehensive tests over different namespace + * types, see more in TestPlacementConstraintsUtil. + * @throws Exception + */ + @Test + public void testInterAppAntiAffinity() throws Exception { + Configuration csConf = TestUtils.getConfigurationWithMultipleQueues( + new Configuration()); + csConf.set(YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_HANDLER, + YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER); + + // inject node label manager + MockRM rm1 = new MockRM(csConf) { + @Override + public RMNodeLabelsManager createNodeLabelManager() { + return mgr; + } + }; + + rm1.getRMContext().setNodeLabelManager(mgr); + rm1.start(); + + // 4 NMs. + MockNM[] nms = new MockNM[4]; + RMNode[] rmNodes = new RMNode[4]; + for (int i = 0; i < 4; i++) { + nms[i] = rm1.registerNode("192.168.0." + i + ":1234", 10 * GB); + rmNodes[i] = rm1.getRMContext().getRMNodes().get(nms[i].getNodeId()); + } + + // app1 -> c + RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nms[0]); + + // app1 asks for 3 anti-affinity containers for the same app. It should + // only get 3 containers allocated to 3 different nodes.. + am1.allocateIntraAppAntiAffinity( + ResourceSizing.newInstance(3, Resource.newInstance(1024, 1)), + Priority.newInstance(1), 1L, ImmutableSet.of("mapper"), "mapper"); + + CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler(); + + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 4; j++) { + cs.handle(new NodeUpdateSchedulerEvent(rmNodes[j])); + } + } + + System.out.println("Mappers on HOST0: " + + rmNodes[0].getAllocationTagsWithCount().get("mapper")); + System.out.println("Mappers on HOST1: " + + rmNodes[1].getAllocationTagsWithCount().get("mapper")); + System.out.println("Mappers on HOST2: " + + rmNodes[2].getAllocationTagsWithCount().get("mapper")); + + // App1 should get 4 containers allocated (1 AM + 3 mappers). + FiCaSchedulerApp schedulerApp = cs.getApplicationAttempt( + am1.getApplicationAttemptId()); + Assert.assertEquals(4, schedulerApp.getLiveContainers().size()); + + // app2 -> c + RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nms[0]); + + // App2 asks for 3 containers that anti-affinity with any mapper, + // since 3 out of 4 nodes already have mapper containers, all 3 + // containers will be allocated on the other node. + AllocationTagNamespace.All allNs = new AllocationTagNamespace.All(); + am2.allocateAppAntiAffinity( + ResourceSizing.newInstance(3, Resource.newInstance(1024, 1)), + Priority.newInstance(1), 1L, allNs.toString(), + ImmutableSet.of("foo"), "mapper"); + + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 4; j++) { + cs.handle(new NodeUpdateSchedulerEvent(rmNodes[j])); + } + } + + FiCaSchedulerApp schedulerApp2 = cs.getApplicationAttempt( + am2.getApplicationAttemptId()); + + // App2 should get 4 containers allocated (1 AM + 3 container). + Assert.assertEquals(4, schedulerApp2.getLiveContainers().size()); + + // The allocated node should not have mapper tag. + Assert.assertTrue(schedulerApp2.getLiveContainers() + .stream().allMatch(rmContainer -> { + // except the nm host + if (!rmContainer.getContainer().getNodeId().equals(rmNodes[0])) { + return !rmContainer.getAllocationTags().contains("mapper"); + } + return true; + })); + + // app3 -> c + RMApp app3 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nms[0]); + + // App3 asks for 3 containers that anti-affinity with any mapper. + // Unlike the former case, since app3 source tags are also mapper, + // it will anti-affinity with itself too. So there will be only 1 + // container be allocated. + am3.allocateAppAntiAffinity( + ResourceSizing.newInstance(3, Resource.newInstance(1024, 1)), + Priority.newInstance(1), 1L, allNs.toString(), + ImmutableSet.of("mapper"), "mapper"); + + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 4; j++) { + cs.handle(new NodeUpdateSchedulerEvent(rmNodes[j])); + } + } + + FiCaSchedulerApp schedulerApp3 = cs.getApplicationAttempt( + am3.getApplicationAttemptId()); + + // App3 should get 2 containers allocated (1 AM + 1 container). + Assert.assertEquals(2, schedulerApp3.getLiveContainers().size()); + + rm1.close(); + } + @Test public void testSchedulingRequestDisabledByDefault() throws Exception { Configuration csConf = TestUtils.getConfigurationWithMultipleQueues(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d199274b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java index 76f451e..cbf5968 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint; import com.google.common.collect.ImmutableSet; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; @@ -96,7 +97,9 @@ public class TestAllocationTagsManager { // Get Node Cardinality of app1 on node1, with tag "mapper" Assert.assertEquals(1, atm.getNodeCardinalityByOp(NodeId.fromString("host1:123"), - TestUtils.getMockApplicationId(1), ImmutableSet.of("mapper"), + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of("mapper")), Long::max)); // Get Rack Cardinality of app1 on rack0, with tag "mapper" @@ -106,20 +109,26 @@ public class TestAllocationTagsManager { // Get Node Cardinality of app1 on node2, with tag "mapper/reducer", op=min Assert.assertEquals(1, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(1), - ImmutableSet.of("mapper", "reducer"), Long::min)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of("mapper", "reducer")), + Long::min)); // Get Node Cardinality of app1 on node2, with tag "mapper/reducer", op=max Assert.assertEquals(2, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(1), - ImmutableSet.of("mapper", "reducer"), Long::max)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of("mapper", "reducer")), + Long::max)); // Get Node Cardinality of app1 on node2, with tag "mapper/reducer", op=sum Assert.assertEquals(3, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(1), - ImmutableSet.of("mapper", "reducer"), Long::sum)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of("mapper", "reducer")), + Long::sum)); // Get Node Cardinality by passing single tag. Assert.assertEquals(1, @@ -134,38 +143,52 @@ public class TestAllocationTagsManager { // op=min Assert.assertEquals(0, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(1), - ImmutableSet.of("no_existed", "reducer"), Long::min)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of("no_existed", "reducer")), + Long::min)); // Get Node Cardinality of app1 on node2, with tag "<applicationId>", op=max // (Expect this returns #containers from app1 on node2) Assert.assertEquals(2, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(1), null, Long::max)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), null), + Long::max)); // Get Node Cardinality of app1 on node2, with empty tag set, op=max Assert.assertEquals(2, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(1), null, Long::max)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), null), + Long::max)); // Get Cardinality of app1 on node2, with empty tag set, op=max Assert.assertEquals(2, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::max)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), ImmutableSet.of()), + Long::max)); // Get Node Cardinality of all apps on node2, with empty tag set, op=sum Assert.assertEquals(4, atm.getNodeCardinalityByOp( - NodeId.fromString("host2:123"), null, ImmutableSet.of(), Long::sum)); + NodeId.fromString("host2:123"), + AllocationTags.createGlobalAllocationTags(ImmutableSet.of()), + Long::sum)); // Get Node Cardinality of app_1 on node2, with empty tag set, op=sum Assert.assertEquals(3, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::sum)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), ImmutableSet.of()), + Long::sum)); // Get Node Cardinality of app_1 on node2, with empty tag set, op=sum Assert.assertEquals(1, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(2), ImmutableSet.of(), Long::sum)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(2), ImmutableSet.of()), + Long::sum)); // Finish all containers: atm.removeContainer(NodeId.fromString("host1:123"), @@ -189,33 +212,42 @@ public class TestAllocationTagsManager { // Get Cardinality of app1 on node1, with tag "mapper" Assert.assertEquals(0, atm.getNodeCardinalityByOp(NodeId.fromString("host1:123"), - TestUtils.getMockApplicationId(1), ImmutableSet.of("mapper"), + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of("mapper")), Long::max)); // Get Node Cardinality of app1 on node2, with tag "mapper/reducer", op=min Assert.assertEquals(0, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(1), - ImmutableSet.of("mapper", "reducer"), Long::min)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of("mapper", "reducer")), + Long::min)); // Get Node Cardinality of app1 on node2, with tag "mapper/reducer", op=max Assert.assertEquals(0, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(1), - ImmutableSet.of("mapper", "reducer"), Long::max)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of("mapper", "reducer")), + Long::max)); // Get Node Cardinality of app1 on node2, with tag "mapper/reducer", op=sum Assert.assertEquals(0, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(1), - ImmutableSet.of("mapper", "reducer"), Long::sum)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of("mapper", "reducer")), + Long::sum)); // Get Node Cardinality of app1 on node2, with tag "<applicationId>", op=max // (Expect this returns #containers from app1 on node2) Assert.assertEquals(0, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(1), - ImmutableSet.of(TestUtils.getMockApplicationId(1).toString()), + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of(TestUtils.getMockApplicationId(1).toString())), Long::max)); Assert.assertEquals(0, @@ -226,21 +258,32 @@ public class TestAllocationTagsManager { // Get Node Cardinality of app1 on node2, with empty tag set, op=max Assert.assertEquals(0, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::max)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of()), + Long::max)); // Get Node Cardinality of all apps on node2, with empty tag set, op=sum Assert.assertEquals(0, atm.getNodeCardinalityByOp( - NodeId.fromString("host2:123"), null, ImmutableSet.of(), Long::sum)); + NodeId.fromString("host2:123"), + AllocationTags.createGlobalAllocationTags(ImmutableSet.of()), + Long::sum)); // Get Node Cardinality of app_1 on node2, with empty tag set, op=sum Assert.assertEquals(0, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::sum)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of()), + Long::sum)); // Get Node Cardinality of app_2 on node2, with empty tag set, op=sum Assert.assertEquals(0, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(2), ImmutableSet.of(), Long::sum)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of()), + Long::sum)); } @@ -296,20 +339,26 @@ public class TestAllocationTagsManager { // Get Rack Cardinality of app_1 on rack0, with empty tag set, op=max Assert.assertEquals(1, atm.getRackCardinalityByOp("rack0", - TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::max)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of()), + Long::max)); // Get Rack Cardinality of app_1 on rack0, with empty tag set, op=min Assert.assertEquals(1, atm.getRackCardinalityByOp("rack0", - TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::min)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of()), + Long::min)); // Get Rack Cardinality of all apps on rack0, with empty tag set, op=min - Assert.assertEquals(3, atm.getRackCardinalityByOp("rack0", null, - ImmutableSet.of(), Long::max)); + Assert.assertEquals(3, atm.getRackCardinalityByOp("rack0", + AllocationTags.createGlobalAllocationTags(ImmutableSet.of()), + Long::max)); } @Test - public void testAllocationTagsManagerMemoryAfterCleanup() - throws InvalidAllocationTagsQueryException { + public void testAllocationTagsManagerMemoryAfterCleanup() { /** * Make sure YARN cleans up all memory once container/app finishes. */ @@ -362,8 +411,7 @@ public class TestAllocationTagsManager { } @Test - public void testQueryCardinalityWithIllegalParameters() - throws InvalidAllocationTagsQueryException { + public void testQueryCardinalityWithIllegalParameters() { /** * Make sure YARN cleans up all memory once container/app finishes. */ @@ -391,9 +439,12 @@ public class TestAllocationTagsManager { // No node-id boolean caughtException = false; try { - atm.getNodeCardinalityByOp(null, TestUtils.getMockApplicationId(2), - ImmutableSet.of("mapper"), Long::min); - } catch (InvalidAllocationTagsQueryException e) { + atm.getNodeCardinalityByOp(null, + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(2), + ImmutableSet.of("mapper")), + Long::min); + } catch (InvalidAllocationTagsQueryException e1) { caughtException = true; } Assert.assertTrue("should fail because of nodeId specified", @@ -403,11 +454,150 @@ public class TestAllocationTagsManager { caughtException = false; try { atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(2), ImmutableSet.of("mapper"), null); - } catch (InvalidAllocationTagsQueryException e) { + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(2), + ImmutableSet.of("mapper")), + null); + } catch (InvalidAllocationTagsQueryException e1) { caughtException = true; } Assert.assertTrue("should fail because of nodeId specified", caughtException); } + + @Test + public void testNodeAllocationTagsAggregation() + throws InvalidAllocationTagsQueryException { + + AllocationTagsManager atm = new AllocationTagsManager(rmContext); + ApplicationId app1 = TestUtils.getMockApplicationId(1); + ApplicationId app2 = TestUtils.getMockApplicationId(2); + ApplicationId app3 = TestUtils.getMockApplicationId(3); + NodeId host1 = NodeId.fromString("host1:123"); + NodeId host2 = NodeId.fromString("host2:123"); + NodeId host3 = NodeId.fromString("host3:123"); + + /** + * Node1 (rack0) + * app1/A(2) + * app1/B(1) + * app2/A(3) + * app3/A(1) + * + * Node2 (rack0) + * app2/A(1) + * app2/B(2) + * app1/C(1) + * app3/B(1) + * + * Node3 (rack1): + * app2/D(1) + * app3/D(1) + */ + atm.addContainer(host1, TestUtils.getMockContainerId(1, 1), + ImmutableSet.of("A", "B")); + atm.addContainer(host1, TestUtils.getMockContainerId(1, 2), + ImmutableSet.of("A")); + atm.addContainer(host1, TestUtils.getMockContainerId(2, 1), + ImmutableSet.of("A")); + atm.addContainer(host1, TestUtils.getMockContainerId(2, 2), + ImmutableSet.of("A")); + atm.addContainer(host1, TestUtils.getMockContainerId(2, 3), + ImmutableSet.of("A")); + atm.addContainer(host1, TestUtils.getMockContainerId(3, 1), + ImmutableSet.of("A")); + + atm.addContainer(host2, TestUtils.getMockContainerId(1, 3), + ImmutableSet.of("C")); + atm.addContainer(host2, TestUtils.getMockContainerId(2, 4), + ImmutableSet.of("A")); + atm.addContainer(host2, TestUtils.getMockContainerId(2, 5), + ImmutableSet.of("B")); + atm.addContainer(host2, TestUtils.getMockContainerId(2, 6), + ImmutableSet.of("B")); + atm.addContainer(host2, TestUtils.getMockContainerId(3, 2), + ImmutableSet.of("B")); + + atm.addContainer(host3, TestUtils.getMockContainerId(2, 7), + ImmutableSet.of("D")); + atm.addContainer(host3, TestUtils.getMockContainerId(3, 3), + ImmutableSet.of("D")); + + // Target applications, current app: app1 + // all apps: app1, app2, app3 + TargetApplications ta = new TargetApplications(app1, + ImmutableSet.of(app1, app2, app3)); + + //******************************** + // 1) self (app1) + //******************************** + AllocationTags tags = AllocationTags + .createSingleAppAllocationTags(app1, ImmutableSet.of("A", "C")); + Assert.assertEquals(2, atm.getNodeCardinalityByOp(host1, tags, Long::max)); + Assert.assertEquals(0, atm.getNodeCardinalityByOp(host1, tags, Long::min)); + Assert.assertEquals(1, atm.getNodeCardinalityByOp(host2, tags, Long::max)); + Assert.assertEquals(0, atm.getNodeCardinalityByOp(host2, tags, Long::min)); + Assert.assertEquals(0, atm.getNodeCardinalityByOp(host3, tags, Long::max)); + Assert.assertEquals(0, atm.getNodeCardinalityByOp(host3, tags, Long::min)); + + //******************************** + // 2) not-self (app2, app3) + //******************************** + /** + * Verify max/min cardinality of tag "A" on host1 from all applications + * other than app1. This returns the max/min cardinality of tag "A" of + * app2 or app3 on this node. + * + * Node1 (rack0) + * app1/A(1) + * app1/B(1) + * app2/A(3) + * app3/A(1) + * + * app2_app3/A(4) + * app2_app3/B(0) + * + * expecting to return max=3, min=1 + * + */ + tags = AllocationTags.createOtherAppAllocationTags(app1, + ImmutableSet.of(app1, app2, app3), ImmutableSet.of("A", "B")); + + Assert.assertEquals(4, atm.getNodeCardinalityByOp(host1, tags, Long::max)); + Assert.assertEquals(0, atm.getNodeCardinalityByOp(host1, tags, Long::min)); + Assert.assertEquals(4, atm.getNodeCardinalityByOp(host1, tags, Long::sum)); + + //******************************** + // 3) app-id/app2 (app2) + //******************************** + tags = AllocationTags + .createSingleAppAllocationTags(app2, ImmutableSet.of("A", "B")); + Assert.assertEquals(3, atm.getNodeCardinalityByOp(host1, tags, Long::max)); + Assert.assertEquals(0, atm.getNodeCardinalityByOp(host1, tags, Long::min)); + Assert.assertEquals(2, atm.getNodeCardinalityByOp(host2, tags, Long::max)); + Assert.assertEquals(1, atm.getNodeCardinalityByOp(host2, tags, Long::min)); + Assert.assertEquals(3, atm.getNodeCardinalityByOp(host2, tags, Long::sum)); + + + //******************************** + // 4) all (app1, app2, app3) + //******************************** + tags = AllocationTags + .createGlobalAllocationTags(ImmutableSet.of("A")); + Assert.assertEquals(6, atm.getNodeCardinalityByOp(host1, tags, Long::sum)); + Assert.assertEquals(1, atm.getNodeCardinalityByOp(host2, tags, Long::sum)); + Assert.assertEquals(0, atm.getNodeCardinalityByOp(host3, tags, Long::sum)); + + tags = AllocationTags + .createGlobalAllocationTags(ImmutableSet.of("A", "B")); + Assert.assertEquals(7, atm.getNodeCardinalityByOp(host1, tags, Long::sum)); + Assert.assertEquals(4, atm.getNodeCardinalityByOp(host2, tags, Long::sum)); + Assert.assertEquals(0, atm.getNodeCardinalityByOp(host3, tags, Long::sum)); + Assert.assertEquals(6, atm.getNodeCardinalityByOp(host1, tags, Long::max)); + Assert.assertEquals(3, atm.getNodeCardinalityByOp(host2, tags, Long::max)); + Assert.assertEquals(0, atm.getNodeCardinalityByOp(host3, tags, Long::max)); + Assert.assertEquals(1, atm.getNodeCardinalityByOp(host1, tags, Long::min)); + Assert.assertEquals(1, atm.getNodeCardinalityByOp(host2, tags, Long::min)); + Assert.assertEquals(0, atm.getNodeCardinalityByOp(host3, tags, Long::min)); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d199274b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsNamespace.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsNamespace.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsNamespace.java index 67a3901..d1ef331 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsNamespace.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsNamespace.java @@ -16,10 +16,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint; /** * limitations under the License. */ import com.google.common.collect.ImmutableSet; -import org.apache.hadoop.yarn.api.records.AllocationTagNamespace; +import org.apache.hadoop.yarn.api.records.AllocationTagNamespaceType; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.TargetApplications; -import org.apache.hadoop.yarn.exceptions.InvalidAllocationTagException; import org.junit.Assert; import org.junit.Test; @@ -29,29 +27,34 @@ import org.junit.Test; public class TestAllocationTagsNamespace { @Test - public void testNamespaceParse() throws InvalidAllocationTagException { + public void testNamespaceParse() throws InvalidAllocationTagsQueryException { AllocationTagNamespace namespace; String namespaceStr = "self"; namespace = AllocationTagNamespace.parse(namespaceStr); - Assert.assertTrue(namespace.isIntraApp()); + Assert.assertEquals(AllocationTagNamespaceType.SELF, + namespace.getNamespaceType()); namespaceStr = "not-self"; namespace = AllocationTagNamespace.parse(namespaceStr); - Assert.assertTrue(namespace.isNotSelf()); + Assert.assertEquals(AllocationTagNamespaceType.NOT_SELF, + namespace.getNamespaceType()); namespaceStr = "all"; namespace = AllocationTagNamespace.parse(namespaceStr); - Assert.assertTrue(namespace.isGlobal()); + Assert.assertEquals(AllocationTagNamespaceType.ALL, + namespace.getNamespaceType()); namespaceStr = "app-label"; namespace = AllocationTagNamespace.parse(namespaceStr); - Assert.assertTrue(namespace.isAppLabel()); + Assert.assertEquals(AllocationTagNamespaceType.APP_LABEL, + namespace.getNamespaceType()); ApplicationId applicationId = ApplicationId.newInstance(12345, 1); namespaceStr = "app-id/" + applicationId.toString(); namespace = AllocationTagNamespace.parse(namespaceStr); - Assert.assertTrue(namespace.isSingleInterApp()); + Assert.assertEquals(AllocationTagNamespaceType.APP_ID, + namespace.getNamespaceType()); // Invalid app-id namespace syntax, invalid app ID. try { @@ -59,7 +62,7 @@ public class TestAllocationTagsNamespace { AllocationTagNamespace.parse(namespaceStr); Assert.fail("Parsing should fail as the given app ID is invalid"); } catch (Exception e) { - Assert.assertTrue(e instanceof InvalidAllocationTagException); + Assert.assertTrue(e instanceof InvalidAllocationTagsQueryException); Assert.assertTrue(e.getMessage().startsWith( "Invalid application ID for app-id")); } @@ -71,7 +74,7 @@ public class TestAllocationTagsNamespace { Assert.fail("Parsing should fail as the given namespace" + " is missing application ID"); } catch (Exception e) { - Assert.assertTrue(e instanceof InvalidAllocationTagException); + Assert.assertTrue(e instanceof InvalidAllocationTagsQueryException); Assert.assertTrue(e.getMessage().startsWith( "Missing the application ID in the namespace string")); } @@ -82,14 +85,15 @@ public class TestAllocationTagsNamespace { AllocationTagNamespace.parse(namespaceStr); Assert.fail("Parsing should fail as the giving type is not supported."); } catch (Exception e) { - Assert.assertTrue(e instanceof InvalidAllocationTagException); + Assert.assertTrue(e instanceof InvalidAllocationTagsQueryException); Assert.assertTrue(e.getMessage().startsWith( "Invalid namespace prefix")); } } @Test - public void testNamespaceEvaluation() throws InvalidAllocationTagException { + public void testNamespaceEvaluation() throws + InvalidAllocationTagsQueryException { AllocationTagNamespace namespace; TargetApplications targetApplications; ApplicationId app1 = ApplicationId.newInstance(10000, 1); @@ -131,10 +135,8 @@ public class TestAllocationTagsNamespace { namespaceStr = "all"; namespace = AllocationTagNamespace.parse(namespaceStr); - targetApplications = new TargetApplications(null, - ImmutableSet.of(app1, app2)); - namespace.evaluate(targetApplications); - Assert.assertEquals(2, namespace.getNamespaceScope().size()); + Assert.assertEquals(AllocationTagNamespaceType.ALL, + namespace.getNamespaceType()); namespaceStr = "app-id/" + app2.toString(); namespace = AllocationTagNamespace.parse(namespaceStr); http://git-wip-us.apache.org/repos/asf/hadoop/blob/d199274b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementConstraintsUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementConstraintsUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementConstraintsUtil.java index 5ba8948..4814321 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementConstraintsUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementConstraintsUtil.java @@ -41,7 +41,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.concurrent.atomic.AtomicLong; -import org.apache.hadoop.yarn.api.records.AllocationTagNamespace; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -513,6 +512,252 @@ public class TestPlacementConstraintsUtil { } @Test + public void testGlobalAppConstraints() + throws InvalidAllocationTagsQueryException { + AllocationTagsManager tm = new AllocationTagsManager(rmContext); + PlacementConstraintManagerService pcm = + new MemoryPlacementConstraintManager(); + rmContext.setAllocationTagsManager(tm); + rmContext.setPlacementConstraintManager(pcm); + + long ts = System.currentTimeMillis(); + ApplicationId application1 = BuilderUtils.newApplicationId(ts, 100); + ApplicationId application2 = BuilderUtils.newApplicationId(ts, 101); + ApplicationId application3 = BuilderUtils.newApplicationId(ts, 102); + + // Register App1 with anti-affinity constraint map. + RMNode n0r1 = rmNodes.get(0); + RMNode n1r1 = rmNodes.get(1); + RMNode n2r2 = rmNodes.get(2); + RMNode n3r2 = rmNodes.get(3); + + /** + * Place container: + * n0: app1/A(1), app2/A(1) + * n1: app3/A(3) + * n2: app1/A(2) + * n3: "" + */ + tm.addContainer(n0r1.getNodeID(), + newContainerId(application1), ImmutableSet.of("A")); + tm.addContainer(n0r1.getNodeID(), + newContainerId(application2), ImmutableSet.of("A")); + tm.addContainer(n1r1.getNodeID(), + newContainerId(application3), ImmutableSet.of("A")); + tm.addContainer(n1r1.getNodeID(), + newContainerId(application3), ImmutableSet.of("A")); + tm.addContainer(n1r1.getNodeID(), + newContainerId(application3), ImmutableSet.of("A")); + tm.addContainer(n2r2.getNodeID(), + newContainerId(application1), ImmutableSet.of("A")); + tm.addContainer(n2r2.getNodeID(), + newContainerId(application1), ImmutableSet.of("A")); + + SchedulerNode schedulerNode0 = newSchedulerNode(n0r1.getHostName(), + n0r1.getRackName(), n0r1.getNodeID()); + SchedulerNode schedulerNode1 = newSchedulerNode(n1r1.getHostName(), + n1r1.getRackName(), n1r1.getNodeID()); + SchedulerNode schedulerNode2 = newSchedulerNode(n2r2.getHostName(), + n2r2.getRackName(), n2r2.getNodeID()); + SchedulerNode schedulerNode3 = newSchedulerNode(n3r2.getHostName(), + n3r2.getRackName(), n3r2.getNodeID()); + + AllocationTagNamespace namespaceAll = + new AllocationTagNamespace.All(); + + //*************************** + // 1) all, anti-affinity + //*************************** + // Anti-affinity with "A" from any application including itself. + PlacementConstraint constraint1 = PlacementConstraints.targetNotIn( + NODE, allocationTagWithNamespace(namespaceAll.toString(), "A")) + .build(); + Map<Set<String>, PlacementConstraint> constraintMap = new HashMap<>(); + Set<String> srcTags1 = ImmutableSet.of("A"); + constraintMap.put(srcTags1, constraint1); + pcm.registerApplication(application1, constraintMap); + + Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints( + application1, createSchedulingRequest(srcTags1), + schedulerNode0, pcm, tm)); + Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints( + application1, createSchedulingRequest(srcTags1), + schedulerNode1, pcm, tm)); + Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints( + application1, createSchedulingRequest(srcTags1), + schedulerNode2, pcm, tm)); + Assert.assertTrue(PlacementConstraintsUtil.canSatisfyConstraints( + application1, createSchedulingRequest(srcTags1), + schedulerNode3, pcm, tm)); + + pcm.unregisterApplication(application1); + + //*************************** + // 2) all, max cardinality + //*************************** + PlacementConstraint constraint2 = PlacementConstraints + .maxCardinality(NODE, namespaceAll.toString(), 2, "A") + .build(); + constraintMap.clear(); + Set<String> srcTags2 = ImmutableSet.of("foo"); + constraintMap.put(srcTags2, constraint2); + pcm.registerApplication(application2, constraintMap); + + Assert.assertTrue(PlacementConstraintsUtil.canSatisfyConstraints( + application2, createSchedulingRequest(srcTags2), + schedulerNode0, pcm, tm)); + Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints( + application2, createSchedulingRequest(srcTags2), + schedulerNode1, pcm, tm)); + Assert.assertTrue(PlacementConstraintsUtil.canSatisfyConstraints( + application2, createSchedulingRequest(srcTags2), + schedulerNode2, pcm, tm)); + Assert.assertTrue(PlacementConstraintsUtil.canSatisfyConstraints( + application2, createSchedulingRequest(srcTags2), + schedulerNode3, pcm, tm)); + + pcm.unregisterApplication(application2); + + //*************************** + // 3) all, min cardinality + //*************************** + PlacementConstraint constraint3 = PlacementConstraints + .minCardinality(NODE, namespaceAll.toString(), 3, "A") + .build(); + constraintMap.clear(); + Set<String> srcTags3 = ImmutableSet.of("foo"); + constraintMap.put(srcTags3, constraint3); + pcm.registerApplication(application3, constraintMap); + + Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints( + application3, createSchedulingRequest(srcTags3), + schedulerNode0, pcm, tm)); + Assert.assertTrue(PlacementConstraintsUtil.canSatisfyConstraints( + application3, createSchedulingRequest(srcTags3), + schedulerNode1, pcm, tm)); + Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints( + application3, createSchedulingRequest(srcTags3), + schedulerNode2, pcm, tm)); + Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints( + application3, createSchedulingRequest(srcTags3), + schedulerNode3, pcm, tm)); + + pcm.unregisterApplication(application3); + } + + @Test + public void testNotSelfAppConstraints() + throws InvalidAllocationTagsQueryException { + AllocationTagsManager tm = new AllocationTagsManager(rmContext); + PlacementConstraintManagerService pcm = + new MemoryPlacementConstraintManager(); + rmContext.setAllocationTagsManager(tm); + rmContext.setPlacementConstraintManager(pcm); + + long ts = System.currentTimeMillis(); + ApplicationId application1 = BuilderUtils.newApplicationId(ts, 100); + ApplicationId application2 = BuilderUtils.newApplicationId(ts, 101); + ApplicationId application3 = BuilderUtils.newApplicationId(ts, 102); + + // Register App1 with anti-affinity constraint map. + RMNode n0r1 = rmNodes.get(0); + RMNode n1r1 = rmNodes.get(1); + RMNode n2r2 = rmNodes.get(2); + RMNode n3r2 = rmNodes.get(3); + + /** + * Place container: + * n0: app1/A(1), app2/A(1) + * n1: app3/A(3) + * n2: app1/A(2) + * n3: "" + */ + tm.addContainer(n0r1.getNodeID(), + newContainerId(application1), ImmutableSet.of("A")); + tm.addContainer(n0r1.getNodeID(), + newContainerId(application2), ImmutableSet.of("A")); + tm.addContainer(n1r1.getNodeID(), + newContainerId(application3), ImmutableSet.of("A")); + tm.addContainer(n1r1.getNodeID(), + newContainerId(application3), ImmutableSet.of("A")); + tm.addContainer(n1r1.getNodeID(), + newContainerId(application3), ImmutableSet.of("A")); + tm.addContainer(n2r2.getNodeID(), + newContainerId(application1), ImmutableSet.of("A")); + tm.addContainer(n2r2.getNodeID(), + newContainerId(application1), ImmutableSet.of("A")); + + SchedulerNode schedulerNode0 = newSchedulerNode(n0r1.getHostName(), + n0r1.getRackName(), n0r1.getNodeID()); + SchedulerNode schedulerNode1 = newSchedulerNode(n1r1.getHostName(), + n1r1.getRackName(), n1r1.getNodeID()); + SchedulerNode schedulerNode2 = newSchedulerNode(n2r2.getHostName(), + n2r2.getRackName(), n2r2.getNodeID()); + SchedulerNode schedulerNode3 = newSchedulerNode(n3r2.getHostName(), + n3r2.getRackName(), n3r2.getNodeID()); + + AllocationTagNamespace notSelf = + new AllocationTagNamespace.NotSelf(); + + //*************************** + // 1) not-self, app1 + //*************************** + // Anti-affinity with "A" from app2 and app3, + // n0 and n1 both have tag "A" from either app2 or app3, so they are + // not qualified for the placement. + PlacementConstraint constraint1 = PlacementConstraints.targetNotIn( + NODE, allocationTagWithNamespace(notSelf.toString(), "A")) + .build(); + Map<Set<String>, PlacementConstraint> constraintMap = new HashMap<>(); + Set<String> srcTags1 = ImmutableSet.of("A"); + constraintMap.put(srcTags1, constraint1); + pcm.registerApplication(application1, constraintMap); + + Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints( + application1, createSchedulingRequest(srcTags1), + schedulerNode0, pcm, tm)); + Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints( + application1, createSchedulingRequest(srcTags1), + schedulerNode1, pcm, tm)); + Assert.assertTrue(PlacementConstraintsUtil.canSatisfyConstraints( + application1, createSchedulingRequest(srcTags1), + schedulerNode2, pcm, tm)); + Assert.assertTrue(PlacementConstraintsUtil.canSatisfyConstraints( + application1, createSchedulingRequest(srcTags1), + schedulerNode3, pcm, tm)); + + pcm.unregisterApplication(application1); + + //*************************** + // 2) not-self, app1 + //*************************** + // Affinity with "A" from app2 and app3, + // N0 and n1 are qualified for the placement. + PlacementConstraint constraint2 = PlacementConstraints.targetIn( + NODE, allocationTagWithNamespace(notSelf.toString(), "A")) + .build(); + Map<Set<String>, PlacementConstraint> cm2 = new HashMap<>(); + Set<String> srcTags2 = ImmutableSet.of("A"); + cm2.put(srcTags2, constraint2); + pcm.registerApplication(application1, cm2); + + Assert.assertTrue(PlacementConstraintsUtil.canSatisfyConstraints( + application1, createSchedulingRequest(srcTags2), + schedulerNode0, pcm, tm)); + Assert.assertTrue(PlacementConstraintsUtil.canSatisfyConstraints( + application1, createSchedulingRequest(srcTags2), + schedulerNode1, pcm, tm)); + Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints( + application1, createSchedulingRequest(srcTags2), + schedulerNode2, pcm, tm)); + Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints( + application1, createSchedulingRequest(srcTags2), + schedulerNode3, pcm, tm)); + + pcm.unregisterApplication(application1); + } + + @Test public void testInterAppConstraintsByAppID() throws InvalidAllocationTagsQueryException { AllocationTagsManager tm = new AllocationTagsManager(rmContext); http://git-wip-us.apache.org/repos/asf/hadoop/blob/d199274b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/TestLocalAllocationTagsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/TestLocalAllocationTagsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/TestLocalAllocationTagsManager.java index 0b9657f..2ac5c3d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/TestLocalAllocationTagsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/TestLocalAllocationTagsManager.java @@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTags; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.InvalidAllocationTagsQueryException; import org.junit.Assert; @@ -85,46 +86,62 @@ public class TestLocalAllocationTagsManager { // Expect tag mappings to be present including temp Tags Assert.assertEquals(1, atm.getNodeCardinalityByOp(NodeId.fromString("host1:123"), - TestUtils.getMockApplicationId(1), ImmutableSet.of("mapper"), + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of("mapper")), Long::sum)); Assert.assertEquals(1, atm.getNodeCardinalityByOp(NodeId.fromString("host1:123"), - TestUtils.getMockApplicationId(1), ImmutableSet.of("service"), + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of("service")), Long::sum)); Assert.assertEquals(1, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(2), ImmutableSet.of("service"), + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(2), + ImmutableSet.of("service")), Long::sum)); // Do a temp Tag cleanup on app2 ephAtm.cleanTempContainers(TestUtils.getMockApplicationId(2)); Assert.assertEquals(0, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(2), ImmutableSet.of("service"), + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(2), + ImmutableSet.of("service")), Long::sum)); // Expect app1 to be unaffected Assert.assertEquals(1, atm.getNodeCardinalityByOp(NodeId.fromString("host1:123"), - TestUtils.getMockApplicationId(1), ImmutableSet.of("mapper"), + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of("mapper")), Long::sum)); // Do a cleanup on app1 as well ephAtm.cleanTempContainers(TestUtils.getMockApplicationId(1)); Assert.assertEquals(0, atm.getNodeCardinalityByOp(NodeId.fromString("host1:123"), - TestUtils.getMockApplicationId(1), ImmutableSet.of("mapper"), + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of("mapper")), Long::sum)); // Non temp-tags should be unaffected Assert.assertEquals(1, atm.getNodeCardinalityByOp(NodeId.fromString("host1:123"), - TestUtils.getMockApplicationId(1), ImmutableSet.of("service"), + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of("service")), Long::sum)); Assert.assertEquals(0, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(2), ImmutableSet.of("service"), + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(2), + ImmutableSet.of("service")), Long::sum)); // Expect app2 with no containers, and app1 with 2 containers across 2 nodes http://git-wip-us.apache.org/repos/asf/hadoop/blob/d199274b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/TestSingleConstraintAppPlacementAllocator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/TestSingleConstraintAppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/TestSingleConstraintAppPlacementAllocator.java index 9be56ff..4c6afd4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/TestSingleConstraintAppPlacementAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/TestSingleConstraintAppPlacementAllocator.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement; import com.google.common.collect.ImmutableSet; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTags; import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest; import org.apache.hadoop.yarn.api.records.NodeId; @@ -366,8 +367,7 @@ public class TestSingleConstraintAppPlacementAllocator { allocator.canAllocate(NodeType.NODE_LOCAL, TestUtils.getMockNode("host1", "/rack1", 123, 1024)); verify(spyAllocationTagsManager, Mockito.times(1)).getNodeCardinalityByOp( - eq(NodeId.fromString("host1:123")), eq(TestUtils.getMockApplicationId(1)), - eq(ImmutableSet.of("mapper", "reducer")), + eq(NodeId.fromString("host1:123")), any(AllocationTags.class), any(LongBinaryOperator.class)); allocator = new SingleConstraintAppPlacementAllocator(); @@ -388,9 +388,8 @@ public class TestSingleConstraintAppPlacementAllocator { allocator.canAllocate(NodeType.NODE_LOCAL, TestUtils.getMockNode("host1", "/rack1", 123, 1024)); verify(spyAllocationTagsManager, Mockito.atLeast(1)).getNodeCardinalityByOp( - eq(NodeId.fromString("host1:123")), - eq(TestUtils.getMockApplicationId(1)), eq(ImmutableSet - .of("mapper", "reducer")), any(LongBinaryOperator.class)); + eq(NodeId.fromString("host1:123")), any(AllocationTags.class), + any(LongBinaryOperator.class)); SchedulerNode node1 = mock(SchedulerNode.class); when(node1.getPartition()).thenReturn("x"); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
