SLIDER-799 SLIDER-817 track unplaced outstanding requests
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/43c61fbb Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/43c61fbb Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/43c61fbb Branch: refs/heads/feature/SLIDER-799-AM-managed-relax Commit: 43c61fbba2ee448f2797790aaaf52ddaaf9ac6f5 Parents: 1938323 Author: Steve Loughran <[email protected]> Authored: Tue Mar 24 16:31:02 2015 +0000 Committer: Steve Loughran <[email protected]> Committed: Tue Mar 24 16:31:02 2015 +0000 ---------------------------------------------------------------------- .../apache/slider/common/tools/SliderUtils.java | 5 +- .../slider/server/appmaster/state/AppState.java | 1 + .../appmaster/state/ContainerAllocation.java | 46 ++++++++++ .../appmaster/state/OutstandingRequest.java | 28 +++++- .../state/OutstandingRequestTracker.java | 96 +++++++++++++++++--- .../server/appmaster/state/RoleHistory.java | 24 +++-- ...tRoleHistoryOutstandingRequestTracker.groovy | 95 +++++++++++++++---- .../TestRoleHistoryRequestTracking.groovy | 32 ++++--- .../appmaster/model/mock/MockPriority.groovy | 43 +++++++++ 9 files changed, 314 insertions(+), 56 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43c61fbb/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index ce52b89..b6cb42b 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -995,11 +995,12 @@ public final class SliderUtils { return "null container"; } return String.format(Locale.ENGLISH, - "ContainerID=%s nodeID=%s http=%s priority=%s", + "ContainerID=%s nodeID=%s http=%s priority=%s resource=%s", container.getId(), container.getNodeId(), container.getNodeHttpAddress(), - container.getPriority()); + container.getPriority(), + container.getResource()); } /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43c61fbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index 34b0492..20c1792 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -1945,6 +1945,7 @@ public class AppState { ContainerPriority.createPriority(role.getPriority(), true); Priority p2 = ContainerPriority.createPriority(role.getPriority(), false); + // TODO Delegate to Role History operations.add(new CancelRequestOperation(p1, p2, toCancel)); role.cancel(toCancel); excess -= toCancel; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43c61fbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocation.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocation.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocation.java new file mode 100644 index 0000000..306ffb2 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocation.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.state; + +/** + * This is just a tuple of the outcome of a container allocation + */ +public class ContainerAllocation { + + /** + * What was the outcome of this allocation: placed, escalated, ... + */ + public ContainerAllocationOutcome outcome; + + /** + * The outstanding request which originated this. + * This will be null if the outcome is {@link ContainerAllocationOutcome#Unallocated} + * as it wasn't expected. + */ + public OutstandingRequest origin; + + public ContainerAllocation(ContainerAllocationOutcome outcome, + OutstandingRequest origin) { + this.outcome = outcome; + this.origin = origin; + } + + public ContainerAllocation() { + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43c61fbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java index 8c320f0..24946af 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java @@ -91,6 +91,11 @@ public final class OutstandingRequest { private boolean mayEscalate; /** + * Priority of request; only valid after the request is built up + */ + private int priority = -1; + + /** * Create a request * @param roleId role * @param node node -can be null @@ -144,6 +149,10 @@ public final class OutstandingRequest { return issuedRequest; } + public int getPriority() { + return priority; + } + /** * Build a container request. * <p> @@ -215,6 +224,7 @@ public final class OutstandingRequest { mayEscalate = false; } Priority pri = ContainerPriority.createPriority(roleId, !relaxLocality); + priority = pri.getPriority(); issuedRequest = new AMRMClient.ContainerRequest(resource, hosts, null, @@ -236,6 +246,8 @@ public final class OutstandingRequest { public synchronized AMRMClient.ContainerRequest escalate() { Preconditions.checkNotNull(issuedRequest, "cannot escalate if request not issued "+ this); escalated = true; + + // this is now the priority Priority pri = ContainerPriority.createPriority(roleId, true); String[] nodes; List<String> issuedRequestNodes = issuedRequest.getNodes(); @@ -258,10 +270,13 @@ public final class OutstandingRequest { /** * Mark the request as completed (or canceled). + * <p> + * Current action: if a node is defined, its request count is decremented */ public void completed() { - assert node != null : "null node"; - node.getOrCreate(roleId).requestCompleted(); + if (node != null) { + node.getOrCreate(roleId).requestCompleted(); + } } /** @@ -277,6 +292,15 @@ public final class OutstandingRequest { } /** + * Query for the resource requirements matching; always false before a request is issued + * @param resource + * @return + */ + public synchronized boolean resourceRequirementsMatch(Resource resource) { + return issuedRequest != null && issuedRequest.getCapability().equals(resource); + } + + /** * Equality is on hostname and role * @param o other * @return true on a match http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43c61fbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java index e226a22..05a8052 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java @@ -18,8 +18,12 @@ package org.apache.slider.server.appmaster.state; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.client.api.AMRMClient; +import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.server.appmaster.operations.AbstractRMOperation; import org.apache.slider.server.appmaster.operations.CancelSingleRequest; import org.apache.slider.server.appmaster.operations.ContainerRequestOperation; @@ -33,6 +37,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.ListIterator; import java.util.Map; /** @@ -61,9 +66,15 @@ public class OutstandingRequestTracker { new HashMap<>(); /** - * Create a new request for the specific role. If a - * location is set, the request is added to the list of requests to track. - * if it isn't, it is not tracked. + * List of open requests; no specific details on them. + */ + private List<OutstandingRequest> openRequests = new ArrayList<>(); + + /** + * Create a new request for the specific role. + * <p> + * If a location is set, the request is added to {@link #placedRequests}. + * If not, it is added to {@link #openRequests} * <p> * This does not update the node instance's role's request count * @param instance node instance to manager @@ -75,6 +86,8 @@ public class OutstandingRequestTracker { new OutstandingRequest(role, instance); if (request.isLocated()) { placedRequests.put(request, request); + } else { + openRequests.add(request); } return request; } @@ -85,7 +98,9 @@ public class OutstandingRequestTracker { * @param hostname hostname * @return the request or null if there was no outstanding one in the {@link #placedRequests} */ - public synchronized OutstandingRequest lookup(int role, String hostname) { + @VisibleForTesting + public synchronized OutstandingRequest lookupPlacedRequest(int role, String hostname) { + Preconditions.checkArgument(hostname != null, "null hostname"); return placedRequests.get(new OutstandingRequest(role, hostname)); } @@ -94,7 +109,8 @@ public class OutstandingRequestTracker { * @param request matching request to find * @return the request or null for no match in the {@link #placedRequests} */ - public synchronized OutstandingRequest remove(OutstandingRequest request) { + @VisibleForTesting + public synchronized OutstandingRequest removePlacedRequest(OutstandingRequest request) { return placedRequests.remove(request); } @@ -103,22 +119,62 @@ public class OutstandingRequestTracker { * from the {@link #placedRequests} structure. * @param role role index * @param hostname hostname + * @param resource * @return the allocation outcome */ - public synchronized ContainerAllocationOutcome onContainerAllocated(int role, String hostname) { + public synchronized ContainerAllocationOutcome onContainerAllocated(int role, + String hostname, + Container container) { + ContainerAllocationOutcome outcome; OutstandingRequest request = placedRequests.remove(new OutstandingRequest(role, hostname)); - if (request == null) { - // not in the list; this is an open placement - return ContainerAllocationOutcome.Open; - } else { + if (request != null) { //satisfied request + log.info("Found placed request for container: {}", request); request.completed(); // derive outcome from status of tracked request - return request.isEscalated() + outcome = request.isEscalated() ? ContainerAllocationOutcome.Escalated : ContainerAllocationOutcome.Placed; + } else { + // not in the list; this is an open placement + // scan through all containers in the open request list + request = removeOpenRequest(container); + if (request != null) { + log.info("Found open request for container: {}", request); + request.completed(); + outcome = ContainerAllocationOutcome.Open; + } else { + log.warn("Container allocation was not expected :" + + SliderUtils.containerToString(container)); + outcome = ContainerAllocationOutcome.Unallocated; + } + } + return outcome; + } + + /** + * Find and remove an open request. Determine it by scanning open requests + * for one whose priority & resource requirements match that of the container + * allocated. + * @param container container allocated + * @return a request which matches the allocation, or null for "no match" + */ + private OutstandingRequest removeOpenRequest(Container container) { + int pri = container.getPriority().getPriority(); + Resource resource = container.getResource(); + OutstandingRequest request = null; + ListIterator<OutstandingRequest> openlist = openRequests.listIterator(); + while (openlist.hasNext() && request == null) { + OutstandingRequest r = openlist.next(); + if (r.getPriority() == pri + && r.resourceRequirementsMatch(resource)) { + // match of priority and resources + request = r; + openlist.remove(); + } } + return request; } /** @@ -226,6 +282,13 @@ public class OutstandingRequestTracker { hosts.add(request.node); } } + ListIterator<OutstandingRequest> openlist = openRequests.listIterator(); + while (openlist.hasNext()) { + OutstandingRequest next = openlist.next(); + if (next.roleId == role) { + openlist.remove(); + } + } return hosts; } @@ -234,11 +297,20 @@ public class OutstandingRequestTracker { * are shared * @return a list of the current outstanding requests */ - public synchronized List<OutstandingRequest> listOutstandingRequests() { + public synchronized List<OutstandingRequest> listPlacedRequests() { return new ArrayList<>(placedRequests.values()); } /** + * Get a list of outstanding requests. The list is cloned, but the contents + * are shared + * @return a list of the current outstanding requests + */ + public synchronized List<OutstandingRequest> listOpenRequests() { + return new ArrayList<>(openRequests); + } + + /** * Escalate operation as triggered by external timer. * @return a (usually empty) list of cancel/request operations. */ http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43c61fbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java index 64f9184..0b981b8 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java @@ -643,12 +643,12 @@ public class RoleHistory { int actualCount) { int role = ContainerPriority.extractRole(container); String hostname = RoleHistoryUtils.hostnameOf(container); - LinkedList<NodeInstance> nodeInstances = - getOrCreateNodesForRoleId(role); - ContainerAllocationOutcome outcome = outstandingRequests.onContainerAllocated(role, hostname); + List<NodeInstance> nodeInstances = getOrCreateNodesForRoleId(role); + ContainerAllocationOutcome outcome = + outstandingRequests.onContainerAllocated(role, hostname, container); if (desiredCount <= actualCount) { // all outstanding requests have been satisfied - // tag nodes as available + // clear all the lists, so returning nodes to the available set List<NodeInstance> hosts = outstandingRequests.resetOutstandingRequests(role); if (!hosts.isEmpty()) { @@ -841,8 +841,18 @@ public class RoleHistory { * @return a list of the requests outstanding at the time of requesting */ @VisibleForTesting - public synchronized List<OutstandingRequest> listOutstandingPlacedRequests() { - return outstandingRequests.listOutstandingRequests(); + public List<OutstandingRequest> listPlacedRequests() { + return outstandingRequests.listPlacedRequests(); + } + + + /** + * Get a snapshot of the outstanding placed request list + * @return a list of the requests outstanding at the time of requesting + */ + @VisibleForTesting + public List<OutstandingRequest> listOpenRequests() { + return outstandingRequests.listOpenRequests(); } /** @@ -856,7 +866,6 @@ public class RoleHistory { return lst; } - /** * Escalate operation as triggered by external timer. * @return a (usually empty) list of cancel/request operations. @@ -864,6 +873,5 @@ public class RoleHistory { public List<AbstractRMOperation> escalateOutstandingRequests() { long now = now(); return outstandingRequests.escalateOutstandingRequests(now); - } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43c61fbb/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy index c30537a..3f41dfd 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy @@ -23,11 +23,15 @@ import org.apache.hadoop.yarn.client.api.AMRMClient import org.apache.slider.providers.PlacementPolicy import org.apache.slider.providers.ProviderRole import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest +import org.apache.slider.server.appmaster.model.mock.MockContainer +import org.apache.slider.server.appmaster.model.mock.MockNodeId +import org.apache.slider.server.appmaster.model.mock.MockPriority import org.apache.slider.server.appmaster.model.mock.MockResource import org.apache.slider.server.appmaster.operations.AbstractRMOperation import org.apache.slider.server.appmaster.operations.CancelSingleRequest import org.apache.slider.server.appmaster.operations.ContainerRequestOperation import org.apache.slider.server.appmaster.state.ContainerAllocationOutcome +import org.apache.slider.server.appmaster.state.ContainerPriority import org.apache.slider.server.appmaster.state.NodeInstance import org.apache.slider.server.appmaster.state.OutstandingRequest import org.apache.slider.server.appmaster.state.OutstandingRequestTracker @@ -49,9 +53,9 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest { @Test public void testAddRetrieveEntry() throws Throwable { OutstandingRequest request = tracker.newRequest(host1, 0) - assert tracker.lookup(0, "host1").equals(request) - assert tracker.remove(request).equals(request) - assert !tracker.lookup(0, "host1") + assert tracker.lookupPlacedRequest(0, "host1").equals(request) + assert tracker.removePlacedRequest(request).equals(request) + assert !tracker.lookupPlacedRequest(0, "host1") } @Test @@ -59,11 +63,68 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest { tracker.newRequest(host1, 0) tracker.newRequest(host2, 0) tracker.newRequest(host1, 1) - assert tracker.onContainerAllocated(1, "host1") == ContainerAllocationOutcome.Placed - assert !tracker.lookup(1, "host1") - assert tracker.lookup(0, "host1") + assert tracker.onContainerAllocated(1, "host1", null) == ContainerAllocationOutcome.Placed + assert !tracker.lookupPlacedRequest(1, "host1") + assert tracker.lookupPlacedRequest(0, "host1") } - + + @Test + public void testResetOpenRequests() throws Throwable { + def req1 = tracker.newRequest(null, 0) + assert !req1.located + tracker.newRequest(host1, 0) + def openRequests = tracker.listOpenRequests() + assert openRequests.size() == 1 + tracker.resetOutstandingRequests(0) + assert tracker.listOpenRequests().empty + assert tracker.listPlacedRequests().empty + } + + @Test + public void testRemoveOpenRequestUnissued() throws Throwable { + def req1 = tracker.newRequest(null, 0) + assert tracker.listOpenRequests().size() == 1 + def c1 = factory.newContainer() + c1.setPriority(new MockPriority(0)) + + def resource = factory.newResource() + resource.virtualCores=1 + resource.memory = 48; + c1.setResource(resource) + ContainerAllocationOutcome outcome = tracker.onContainerAllocated(0, "host1", c1) + assert outcome == ContainerAllocationOutcome.Unallocated + assert tracker.listOpenRequests().size() == 1 + } + + @Test + public void testIssuedOpenRequest() throws Throwable { + def req1 = tracker.newRequest(null, 0) + def resource = factory.newResource() + resource.virtualCores = 1 + resource.memory = 48; + def yarnRequest = req1.buildContainerRequest(resource, role0Status, 0, "") + assert tracker.listOpenRequests().size() == 1 + def c1 = factory.newContainer() + + def nodeId = factory.newNodeId() + c1.nodeId = nodeId + nodeId.host ="hostname-1" + + def pri = ContainerPriority.buildPriority(0, false) + assert pri > 0 + c1.setPriority(new MockPriority(pri)) + + c1.setResource(resource) + + def issued = req1.issuedRequest + assert issued.capability == resource + assert issued.priority.priority == c1.getPriority().getPriority() + assert req1.resourceRequirementsMatch(resource) + ContainerAllocationOutcome outcome = tracker.onContainerAllocated(0, nodeId.host, c1) + assert tracker.listOpenRequests().size() == 0 + assert outcome == ContainerAllocationOutcome.Open + } + @Test public void testResetEntries() throws Throwable { tracker.newRequest(host1, 0) @@ -73,14 +134,13 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest { assert canceled.size() == 2 assert canceled.contains(host1) assert canceled.contains(host2) - assert tracker.lookup(1, "host1") - assert !tracker.lookup(0, "host1") + assert tracker.lookupPlacedRequest(1, "host1") + assert !tracker.lookupPlacedRequest(0, "host1") canceled = tracker.resetOutstandingRequests(0) assert canceled.size() == 0 assert tracker.resetOutstandingRequests(1).size() == 1 } - @Test public void testEscalation() throws Throwable { @@ -92,7 +152,7 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest { assert outstanding0.located assert !outstanding0.escalated assert !initialRequest.relaxLocality - assert tracker.listOutstandingRequests().size() == 1 + assert tracker.listPlacedRequests().size() == 1 // second. This one doesn't get launched. This is to verify that the escalation // process skips entries which are in the list but have not been issued. @@ -137,7 +197,7 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest { assert escalations3.size() == 2 assert outstanding2.escalated - // finally add a strict entry to th emix + // finally add a strict entry to the mix def (res3, outstanding3) = newRequest(role1Status) final ProviderRole providerRole1 = role1Status.providerRole assert providerRole1.placementPolicy == PlacementPolicy.STRICT @@ -156,7 +216,8 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest { entry.containerCompleted(false) entry.containerCompleted(false) assert entry.failedRecently == 3 - final AMRMClient.ContainerRequest initialRequest = outstanding0.buildContainerRequest(res0, role0Status, 0, null) + final AMRMClient.ContainerRequest initialRequest = + outstanding0.buildContainerRequest(res0, role0Status, 0, null) assert initialRequest.relaxLocality assert initialRequest.nodes == null } @@ -171,16 +232,16 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest { entry.containerCompleted(false) entry.containerCompleted(false) assert entry.failedRecently == 3 - final AMRMClient.ContainerRequest initialRequest = outstanding0.buildContainerRequest(res0, - roleStatus, 0, null) + final AMRMClient.ContainerRequest initialRequest = + outstanding0.buildContainerRequest(res0, roleStatus, 0, null) assert !initialRequest.relaxLocality assert initialRequest.nodes[0] == host1.hostname } /** * Create a new request (always against host1) - * @param r - * @return + * @param r role status + * @return (resource, oustanding-request) */ public def newRequest(RoleStatus r) { final Resource res2 = new MockResource() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43c61fbb/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy index bab6233..34a0a4d 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy @@ -18,6 +18,7 @@ package org.apache.slider.server.appmaster.model.history +import org.apache.hadoop.yarn.api.records.Container import org.apache.hadoop.yarn.api.records.Resource import org.apache.hadoop.yarn.client.api.AMRMClient import org.apache.slider.providers.ProviderRole @@ -123,7 +124,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { @Test public void testRequestedNodeIntoReqList() throws Throwable { AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource) - List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests() + List<OutstandingRequest> requests = roleHistory.listPlacedRequests() assert requests.size() == 1 assert age3Active0.hostname == requests[0].hostname } @@ -131,44 +132,44 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { @Test public void testCompletedRequestDropsNode() throws Throwable { AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource) - List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests() + List<OutstandingRequest> requests = roleHistory.listPlacedRequests() assert requests.size() == 1 String hostname = requests[0].hostname assert age3Active0.hostname == hostname assert hostname == req.nodes[0] MockContainer container = factory.newContainer(req, hostname) assertOnContainerAllocated(container, 2, 1) - assertNoOutstandingRequests() + assertNoOutstandingPlacedRequests() } - public void assertOnContainerAllocated(MockContainer c1, int p1, int p2) { + public void assertOnContainerAllocated(Container c1, int p1, int p2) { assert ContainerAllocationOutcome.Open != roleHistory.onContainerAllocated(c1, p1, p2) } - public void assertOnContainerAllocationOpen(MockContainer c1, int p1, int p2) { + public void assertOnContainerAllocationOpen(Container c1, int p1, int p2) { assert ContainerAllocationOutcome.Open == roleHistory.onContainerAllocated(c1, p1, p2) } - def assertNoOutstandingRequests() { - assert roleHistory.listOutstandingPlacedRequests().empty + def assertNoOutstandingPlacedRequests() { + assert roleHistory.listPlacedRequests().empty } public void assertOutstandingPlacedRequests(int i) { - assert roleHistory.listOutstandingPlacedRequests().size() == i + assert roleHistory.listPlacedRequests().size() == i } @Test public void testTwoRequests() throws Throwable { AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource) AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource) - List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests() + List<OutstandingRequest> requests = roleHistory.listPlacedRequests() assert requests.size() == 2 MockContainer container = factory.newContainer(req, req.nodes[0]) assertOnContainerAllocated(container, 2, 1) assertOutstandingPlacedRequests(1) container = factory.newContainer(req2, req2.nodes[0]) assertOnContainerAllocated(container, 2, 2) - assertNoOutstandingRequests() + assertNoOutstandingPlacedRequests() } @Test @@ -176,7 +177,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource) AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource) AMRMClient.ContainerRequest req3 = roleHistory.requestNode(roleStatus, resource) - List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests() + List<OutstandingRequest> requests = roleHistory.listPlacedRequests() assert requests.size() == 2 MockContainer container = factory.newContainer(req, req.nodes[0]) assertOnContainerAllocated(container, 2, 1) @@ -189,9 +190,10 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { // the final allocation will trigger a cleanup container = factory.newContainer(req2, "four") // no node dropped - assertOnContainerAllocationOpen(container, 3, 3) + assert ContainerAllocationOutcome.Unallocated == roleHistory.onContainerAllocated(container, 3, 3) // yet the list is now empty - assertNoOutstandingRequests() + assertNoOutstandingPlacedRequests() + roleHistory.listOpenRequests().empty // and the remainder goes onto the available list List<NodeInstance> a2 = roleHistory.cloneAvailableList(0) @@ -211,10 +213,10 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { assertOutstandingPlacedRequests(1) container = factory.newContainer(req2, req2.nodes[0]) assertOnContainerAllocated(container, 3, 2) - assertNoOutstandingRequests() + assertNoOutstandingPlacedRequests() container = factory.newContainer(req3, "three") assertOnContainerAllocationOpen(container, 3, 3) - assertNoOutstandingRequests() + assertNoOutstandingPlacedRequests() } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43c61fbb/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockPriority.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockPriority.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockPriority.groovy new file mode 100644 index 0000000..9737973 --- /dev/null +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockPriority.groovy @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.mock + +import org.apache.hadoop.yarn.api.records.Priority + +class MockPriority extends Priority { + + private int priority; + + MockPriority(int priority) { + this.priority = priority + } + + MockPriority() { + } + + @Override + int getPriority() { + return priority + } + + @Override + void setPriority(int priority) { + + } +}
