http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java new file mode 100644 index 0000000..9571f0e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api; + +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.SchedulingRequest; + +import java.util.ArrayList; +import java.util.List; + +/** + * Encapsulates the output of the ConstraintPlacementAlgorithm. The Algorithm + * is free to produce multiple of output objects at the end of each run and it + * must use the provided ConstraintPlacementAlgorithmOutputCollector to + * aggregate/collect this output. Similar to the MapReduce Mapper/Reducer + * which is provided a collector to collect output. + */ +public class ConstraintPlacementAlgorithmOutput { + + private final ApplicationId applicationId; + + public ConstraintPlacementAlgorithmOutput(ApplicationId applicationId) { + this.applicationId = applicationId; + } + + private final List<PlacedSchedulingRequest> placedRequests = + new ArrayList<>(); + + private final List<SchedulingRequest> rejectedRequests = + new ArrayList<>(); + + public List<PlacedSchedulingRequest> getPlacedRequests() { + return placedRequests; + } + + public List<SchedulingRequest> getRejectedRequests() { + return rejectedRequests; + } + + public ApplicationId getApplicationId() { + return applicationId; + } +}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java new file mode 100644 index 0000000..131fd42 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api; + +/** + * The ConstraintPlacementAlgorithm uses the + * ConstraintPlacementAlgorithmOutputCollector to collect any output it + * spits out. + */ +public interface ConstraintPlacementAlgorithmOutputCollector { + + /** + * Collect an ConstraintPlacementAlgorithm output. + * @param algorithmOutput ConstraintPlacementAlgorithm Output. + */ + void collect(ConstraintPlacementAlgorithmOutput algorithmOutput); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/PlacedSchedulingRequest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/PlacedSchedulingRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/PlacedSchedulingRequest.java new file mode 100644 index 0000000..2cd90d6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/PlacedSchedulingRequest.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api; + +import org.apache.hadoop.yarn.api.records.SchedulingRequest; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; + +import java.util.ArrayList; +import java.util.List; + +/** + * Class to encapsulate a Placed scheduling Request. + * It has the original Scheduling Request and a list of SchedulerNodes (one + * for each 'numAllocation' field in the corresponding ResourceSizing object. + * + * NOTE: Clients of this class SHOULD NOT rely on the value of + * resourceSizing.numAllocations and instead should use the + * size of collection returned by getNodes() instead. + */ +public class PlacedSchedulingRequest { + + // The number of times the Algorithm tried to place the SchedulingRequest + // after it was rejected by the commit phase of the Scheduler (due to some + // transient state of the cluster. For eg: no space on Node / user limit etc.) + // The Algorithm can then try to probably place on a different node. + private int placementAttempt = 0; + private final SchedulingRequest request; + // One Node per numContainers in the SchedulingRequest; + private final List<SchedulerNode> nodes = new ArrayList<>(); + + public PlacedSchedulingRequest(SchedulingRequest request) { + this.request = request; + } + + public SchedulingRequest getSchedulingRequest() { + return request; + } + + /** + * List of Node locations on which this Scheduling Request can be placed. + * The size of this list = schedulingRequest.resourceSizing.numAllocations. + * @return List of Scheduler nodes. + */ + public List<SchedulerNode> getNodes() { + return nodes; + } + + public int getPlacementAttempt() { + return placementAttempt; + } + + public void setPlacementAttempt(int attempt) { + this.placementAttempt = attempt; + } + + @Override + public String toString() { + return "PlacedSchedulingRequest{" + + "placementAttempt=" + placementAttempt + + ", request=" + request + + ", nodes=" + nodes + + '}'; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/SchedulingResponse.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/SchedulingResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/SchedulingResponse.java new file mode 100644 index 0000000..6c65d84 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/SchedulingResponse.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api; + +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.SchedulingRequest; + +/** + * This class encapsulates the response received from the ResourceScheduler's + * attemptAllocateOnNode method. + */ +public class SchedulingResponse { + + private final boolean isSuccess; + private final ApplicationId applicationId; + private final SchedulingRequest schedulingRequest; + + /** + * Create a SchedulingResponse. + * @param isSuccess did scheduler accept. + * @param applicationId Application Id. + * @param schedulingRequest Scheduling Request. + */ + public SchedulingResponse(boolean isSuccess, ApplicationId applicationId, + SchedulingRequest schedulingRequest) { + this.isSuccess = isSuccess; + this.applicationId = applicationId; + this.schedulingRequest = schedulingRequest; + } + + /** + * Returns true if Scheduler was able to accept and commit this request. + * @return isSuccessful. + */ + public boolean isSuccess() { + return this.isSuccess; + } + + /** + * Get Application Id. + * @return Application Id. + */ + public ApplicationId getApplicationId() { + return this.applicationId; + } + + /** + * Get Scheduling Request. + * @return Scheduling Request. + */ + public SchedulingRequest getSchedulingRequest() { + return this.schedulingRequest; + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/package-info.java new file mode 100644 index 0000000..01ed713 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/package-info.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This Package contains classes related to constrained placement of + * Requests. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/constraint/TestAllocationTagsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/constraint/TestAllocationTagsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/constraint/TestAllocationTagsManager.java deleted file mode 100644 index 0358792..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/constraint/TestAllocationTagsManager.java +++ /dev/null @@ -1,328 +0,0 @@ -/* - * * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * / - */ - -package org.apache.hadoop.yarn.server.resourcemanager.constraint; - -import com.google.common.collect.ImmutableSet; -import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils; -import org.junit.Assert; -import org.junit.Test; - -/** - * Test functionality of AllocationTagsManager. - */ -public class TestAllocationTagsManager { - @Test - public void testAllocationTagsManagerSimpleCases() - throws InvalidAllocationTagsQueryException { - AllocationTagsManager atm = new AllocationTagsManager(); - - /** - * Construct test case: - * Node1: - * container_1_1 (mapper/reducer/app_1) - * container_1_3 (service/app_1) - * - * Node2: - * container_1_2 (mapper/reducer/app_1) - * container_1_4 (reducer/app_1) - * container_2_1 (service/app_2) - */ - - // 3 Containers from app1 - atm.addContainer(NodeId.fromString("node1:1234"), - TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1), - ImmutableSet.of("mapper", "reducer")); - - atm.addContainer(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2), - ImmutableSet.of("mapper", "reducer")); - - atm.addContainer(NodeId.fromString("node1:1234"), - TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3), - ImmutableSet.of("service")); - - atm.addContainer(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4), - ImmutableSet.of("reducer")); - - // 1 Container from app2 - atm.addContainer(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3), - ImmutableSet.of("service")); - - // Get Cardinality of app1 on node1, with tag "mapper" - Assert.assertEquals(1, - atm.getNodeCardinalityByOp(NodeId.fromString("node1:1234"), - TestUtils.getMockApplicationId(1), ImmutableSet.of("mapper"), - Long::max)); - - // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=min - Assert.assertEquals(1, - atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(1), - ImmutableSet.of("mapper", "reducer"), Long::min)); - - // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=max - Assert.assertEquals(2, - atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(1), - ImmutableSet.of("mapper", "reducer"), Long::max)); - - // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=sum - Assert.assertEquals(3, - atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(1), - ImmutableSet.of("mapper", "reducer"), Long::sum)); - - // Get Cardinality by passing single tag. - Assert.assertEquals(1, - atm.getNodeCardinality(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(1), "mapper")); - - Assert.assertEquals(2, - atm.getNodeCardinality(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(1), "reducer")); - - // Get Cardinality of app1 on node2, with tag "no_existed/reducer", op=min - Assert.assertEquals(0, - atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(1), - ImmutableSet.of("no_existed", "reducer"), Long::min)); - - // Get Cardinality of app1 on node2, with tag "<applicationId>", op=max - // (Expect this returns #containers from app1 on node2) - Assert.assertEquals(2, - atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(1), ImmutableSet - .of(AllocationTagsNamespaces.APP_ID + TestUtils - .getMockApplicationId(1).toString()), Long::max)); - - // Get Cardinality of app1 on node2, with empty tag set, op=max - Assert.assertEquals(2, - atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::max)); - - // Get Cardinality of all apps on node2, with empty tag set, op=sum - Assert.assertEquals(7, - atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), null, - ImmutableSet.of(), Long::sum)); - - // Get Cardinality of app_1 on node2, with empty tag set, op=sum - Assert.assertEquals(5, - atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::sum)); - - // Get Cardinality of app_1 on node2, with empty tag set, op=sum - Assert.assertEquals(2, - atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(2), ImmutableSet.of(), Long::sum)); - - // Finish all containers: - atm.removeContainer(NodeId.fromString("node1:1234"), - TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1), - ImmutableSet.of("mapper", "reducer")); - - atm.removeContainer(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2), - ImmutableSet.of("mapper", "reducer")); - - atm.removeContainer(NodeId.fromString("node1:1234"), - TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3), - ImmutableSet.of("service")); - - atm.removeContainer(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4), - ImmutableSet.of("reducer")); - - atm.removeContainer(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3), - ImmutableSet.of("service")); - - // Expect all cardinality to be 0 - // Get Cardinality of app1 on node1, with tag "mapper" - Assert.assertEquals(0, - atm.getNodeCardinalityByOp(NodeId.fromString("node1:1234"), - TestUtils.getMockApplicationId(1), ImmutableSet.of("mapper"), - Long::max)); - - // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=min - Assert.assertEquals(0, - atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(1), - ImmutableSet.of("mapper", "reducer"), Long::min)); - - // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=max - Assert.assertEquals(0, - atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(1), - ImmutableSet.of("mapper", "reducer"), Long::max)); - - // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=sum - Assert.assertEquals(0, - atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(1), - ImmutableSet.of("mapper", "reducer"), Long::sum)); - - // Get Cardinality of app1 on node2, with tag "<applicationId>", op=max - // (Expect this returns #containers from app1 on node2) - Assert.assertEquals(0, - atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(1), - ImmutableSet.of(TestUtils.getMockApplicationId(1).toString()), - Long::max)); - - Assert.assertEquals(0, - atm.getNodeCardinality(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(1), - TestUtils.getMockApplicationId(1).toString())); - - // Get Cardinality of app1 on node2, with empty tag set, op=max - Assert.assertEquals(0, - atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::max)); - - // Get Cardinality of all apps on node2, with empty tag set, op=sum - Assert.assertEquals(0, - atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), null, - ImmutableSet.of(), Long::sum)); - - // Get Cardinality of app_1 on node2, with empty tag set, op=sum - Assert.assertEquals(0, - atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::sum)); - - // Get Cardinality of app_1 on node2, with empty tag set, op=sum - Assert.assertEquals(0, - atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(2), ImmutableSet.of(), Long::sum)); - } - - @Test - public void testAllocationTagsManagerMemoryAfterCleanup() - throws InvalidAllocationTagsQueryException { - /** - * Make sure YARN cleans up all memory once container/app finishes. - */ - - AllocationTagsManager atm = new AllocationTagsManager(); - - // Add a bunch of containers - atm.addContainer(NodeId.fromString("node1:1234"), - TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1), - ImmutableSet.of("mapper", "reducer")); - - atm.addContainer(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2), - ImmutableSet.of("mapper", "reducer")); - - atm.addContainer(NodeId.fromString("node1:1234"), - TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3), - ImmutableSet.of("service")); - - atm.addContainer(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4), - ImmutableSet.of("reducer")); - - atm.addContainer(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3), - ImmutableSet.of("service")); - - // Remove all these containers - atm.removeContainer(NodeId.fromString("node1:1234"), - TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1), - ImmutableSet.of("mapper", "reducer")); - - atm.removeContainer(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2), - ImmutableSet.of("mapper", "reducer")); - - atm.removeContainer(NodeId.fromString("node1:1234"), - TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3), - ImmutableSet.of("service")); - - atm.removeContainer(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4), - ImmutableSet.of("reducer")); - - atm.removeContainer(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3), - ImmutableSet.of("service")); - - // Check internal data structure - Assert.assertEquals(0, - atm.getGlobalMapping().getNodeToTagsWithCount().size()); - Assert.assertEquals(0, atm.getPerAppMappings().size()); - } - - @Test - public void testQueryCardinalityWithIllegalParameters() - throws InvalidAllocationTagsQueryException { - /** - * Make sure YARN cleans up all memory once container/app finishes. - */ - - AllocationTagsManager atm = new AllocationTagsManager(); - - // Add a bunch of containers - atm.addContainer(NodeId.fromString("node1:1234"), - TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1), - ImmutableSet.of("mapper", "reducer")); - - atm.addContainer(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2), - ImmutableSet.of("mapper", "reducer")); - - atm.addContainer(NodeId.fromString("node1:1234"), - TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3), - ImmutableSet.of("service")); - - atm.addContainer(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4), - ImmutableSet.of("reducer")); - - atm.addContainer(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3), - ImmutableSet.of("service")); - - // No node-id - boolean caughtException = false; - try { - atm.getNodeCardinalityByOp(null, TestUtils.getMockApplicationId(2), - ImmutableSet.of("mapper"), Long::min); - } catch (InvalidAllocationTagsQueryException e) { - caughtException = true; - } - Assert.assertTrue("should fail because of nodeId specified", - caughtException); - - // No op - caughtException = false; - try { - atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), - TestUtils.getMockApplicationId(2), ImmutableSet.of("mapper"), null); - } catch (InvalidAllocationTagsQueryException e) { - caughtException = true; - } - Assert.assertTrue("should fail because of nodeId specified", - caughtException); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java index 27ff311..538d128 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java @@ -54,7 +54,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; -import org.apache.hadoop.yarn.server.resourcemanager.constraint.AllocationTagsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; @@ -62,6 +61,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAt import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager; import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.junit.Assert; http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java index 61a5555..e8734cc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java @@ -42,12 +42,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; -import org.apache.hadoop.yarn.server.resourcemanager.constraint.AllocationTagsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager; import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java new file mode 100644 index 0000000..4bb2a18 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java @@ -0,0 +1,328 @@ +/* + * * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint; + +import com.google.common.collect.ImmutableSet; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +/** + * Test functionality of AllocationTagsManager. + */ +public class TestAllocationTagsManager { + @Test + public void testAllocationTagsManagerSimpleCases() + throws InvalidAllocationTagsQueryException { + AllocationTagsManager atm = new AllocationTagsManager(); + + /** + * Construct test case: + * Node1: + * container_1_1 (mapper/reducer/app_1) + * container_1_3 (service/app_1) + * + * Node2: + * container_1_2 (mapper/reducer/app_1) + * container_1_4 (reducer/app_1) + * container_2_1 (service/app_2) + */ + + // 3 Containers from app1 + atm.addContainer(NodeId.fromString("node1:1234"), + TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1), + ImmutableSet.of("mapper", "reducer")); + + atm.addContainer(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2), + ImmutableSet.of("mapper", "reducer")); + + atm.addContainer(NodeId.fromString("node1:1234"), + TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3), + ImmutableSet.of("service")); + + atm.addContainer(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4), + ImmutableSet.of("reducer")); + + // 1 Container from app2 + atm.addContainer(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3), + ImmutableSet.of("service")); + + // Get Cardinality of app1 on node1, with tag "mapper" + Assert.assertEquals(1, + atm.getNodeCardinalityByOp(NodeId.fromString("node1:1234"), + TestUtils.getMockApplicationId(1), ImmutableSet.of("mapper"), + Long::max)); + + // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=min + Assert.assertEquals(1, + atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(1), + ImmutableSet.of("mapper", "reducer"), Long::min)); + + // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=max + Assert.assertEquals(2, + atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(1), + ImmutableSet.of("mapper", "reducer"), Long::max)); + + // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=sum + Assert.assertEquals(3, + atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(1), + ImmutableSet.of("mapper", "reducer"), Long::sum)); + + // Get Cardinality by passing single tag. + Assert.assertEquals(1, + atm.getNodeCardinality(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(1), "mapper")); + + Assert.assertEquals(2, + atm.getNodeCardinality(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(1), "reducer")); + + // Get Cardinality of app1 on node2, with tag "no_existed/reducer", op=min + Assert.assertEquals(0, + atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(1), + ImmutableSet.of("no_existed", "reducer"), Long::min)); + + // Get Cardinality of app1 on node2, with tag "<applicationId>", op=max + // (Expect this returns #containers from app1 on node2) + Assert.assertEquals(2, + atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(1), ImmutableSet + .of(AllocationTagsNamespaces.APP_ID + TestUtils + .getMockApplicationId(1).toString()), Long::max)); + + // Get Cardinality of app1 on node2, with empty tag set, op=max + Assert.assertEquals(2, + atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::max)); + + // Get Cardinality of all apps on node2, with empty tag set, op=sum + Assert.assertEquals(7, + atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), null, + ImmutableSet.of(), Long::sum)); + + // Get Cardinality of app_1 on node2, with empty tag set, op=sum + Assert.assertEquals(5, + atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::sum)); + + // Get Cardinality of app_1 on node2, with empty tag set, op=sum + Assert.assertEquals(2, + atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(2), ImmutableSet.of(), Long::sum)); + + // Finish all containers: + atm.removeContainer(NodeId.fromString("node1:1234"), + TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1), + ImmutableSet.of("mapper", "reducer")); + + atm.removeContainer(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2), + ImmutableSet.of("mapper", "reducer")); + + atm.removeContainer(NodeId.fromString("node1:1234"), + TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3), + ImmutableSet.of("service")); + + atm.removeContainer(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4), + ImmutableSet.of("reducer")); + + atm.removeContainer(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3), + ImmutableSet.of("service")); + + // Expect all cardinality to be 0 + // Get Cardinality of app1 on node1, with tag "mapper" + Assert.assertEquals(0, + atm.getNodeCardinalityByOp(NodeId.fromString("node1:1234"), + TestUtils.getMockApplicationId(1), ImmutableSet.of("mapper"), + Long::max)); + + // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=min + Assert.assertEquals(0, + atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(1), + ImmutableSet.of("mapper", "reducer"), Long::min)); + + // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=max + Assert.assertEquals(0, + atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(1), + ImmutableSet.of("mapper", "reducer"), Long::max)); + + // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=sum + Assert.assertEquals(0, + atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(1), + ImmutableSet.of("mapper", "reducer"), Long::sum)); + + // Get Cardinality of app1 on node2, with tag "<applicationId>", op=max + // (Expect this returns #containers from app1 on node2) + Assert.assertEquals(0, + atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(1), + ImmutableSet.of(TestUtils.getMockApplicationId(1).toString()), + Long::max)); + + Assert.assertEquals(0, + atm.getNodeCardinality(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(1), + TestUtils.getMockApplicationId(1).toString())); + + // Get Cardinality of app1 on node2, with empty tag set, op=max + Assert.assertEquals(0, + atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::max)); + + // Get Cardinality of all apps on node2, with empty tag set, op=sum + Assert.assertEquals(0, + atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), null, + ImmutableSet.of(), Long::sum)); + + // Get Cardinality of app_1 on node2, with empty tag set, op=sum + Assert.assertEquals(0, + atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::sum)); + + // Get Cardinality of app_1 on node2, with empty tag set, op=sum + Assert.assertEquals(0, + atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(2), ImmutableSet.of(), Long::sum)); + } + + @Test + public void testAllocationTagsManagerMemoryAfterCleanup() + throws InvalidAllocationTagsQueryException { + /** + * Make sure YARN cleans up all memory once container/app finishes. + */ + + AllocationTagsManager atm = new AllocationTagsManager(); + + // Add a bunch of containers + atm.addContainer(NodeId.fromString("node1:1234"), + TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1), + ImmutableSet.of("mapper", "reducer")); + + atm.addContainer(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2), + ImmutableSet.of("mapper", "reducer")); + + atm.addContainer(NodeId.fromString("node1:1234"), + TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3), + ImmutableSet.of("service")); + + atm.addContainer(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4), + ImmutableSet.of("reducer")); + + atm.addContainer(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3), + ImmutableSet.of("service")); + + // Remove all these containers + atm.removeContainer(NodeId.fromString("node1:1234"), + TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1), + ImmutableSet.of("mapper", "reducer")); + + atm.removeContainer(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2), + ImmutableSet.of("mapper", "reducer")); + + atm.removeContainer(NodeId.fromString("node1:1234"), + TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3), + ImmutableSet.of("service")); + + atm.removeContainer(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4), + ImmutableSet.of("reducer")); + + atm.removeContainer(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3), + ImmutableSet.of("service")); + + // Check internal data structure + Assert.assertEquals(0, + atm.getGlobalMapping().getNodeToTagsWithCount().size()); + Assert.assertEquals(0, atm.getPerAppMappings().size()); + } + + @Test + public void testQueryCardinalityWithIllegalParameters() + throws InvalidAllocationTagsQueryException { + /** + * Make sure YARN cleans up all memory once container/app finishes. + */ + + AllocationTagsManager atm = new AllocationTagsManager(); + + // Add a bunch of containers + atm.addContainer(NodeId.fromString("node1:1234"), + TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1), + ImmutableSet.of("mapper", "reducer")); + + atm.addContainer(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2), + ImmutableSet.of("mapper", "reducer")); + + atm.addContainer(NodeId.fromString("node1:1234"), + TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3), + ImmutableSet.of("service")); + + atm.addContainer(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4), + ImmutableSet.of("reducer")); + + atm.addContainer(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3), + ImmutableSet.of("service")); + + // No node-id + boolean caughtException = false; + try { + atm.getNodeCardinalityByOp(null, TestUtils.getMockApplicationId(2), + ImmutableSet.of("mapper"), Long::min); + } catch (InvalidAllocationTagsQueryException e) { + caughtException = true; + } + Assert.assertTrue("should fail because of nodeId specified", + caughtException); + + // No op + caughtException = false; + try { + atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), + TestUtils.getMockApplicationId(2), ImmutableSet.of("mapper"), null); + } catch (InvalidAllocationTagsQueryException e) { + caughtException = true; + } + Assert.assertTrue("should fail because of nodeId specified", + caughtException); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index 4b902a7..db749ac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -74,7 +74,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWri import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; -import org.apache.hadoop.yarn.server.resourcemanager.constraint.AllocationTagsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; @@ -93,6 +92,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org