SLIDER-966 RoleHistory now creates the resource requests, issued via the factory. To be honest, I'm not 100% sure I like the current design; I may need to do another iteration
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/aa46b473 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/aa46b473 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/aa46b473 Branch: refs/heads/feature/SLIDER-82-pass-3.1 Commit: aa46b473e11e9ec154ee12cf5d5113d7b974f99c Parents: ac98d82 Author: Steve Loughran <[email protected]> Authored: Fri Nov 6 21:19:51 2015 +0000 Committer: Steve Loughran <[email protected]> Committed: Fri Nov 6 21:19:51 2015 +0000 ---------------------------------------------------------------------- .../appmaster/ProtobufClusterServices.java | 6 + .../state/AbstractClusterServices.java | 6 + .../slider/server/appmaster/state/AppState.java | 126 +++++++++++-------- .../server/appmaster/state/RoleHistory.java | 23 ++-- .../appstate/TestMockAppStateAAPlacement.groovy | 29 +++-- .../TestRoleHistoryContainerEvents.groovy | 62 ++++----- .../TestRoleHistoryRequestTracking.groovy | 32 ++--- .../model/mock/MockClusterServices.groovy | 5 + .../appmaster/model/mock/MockFactory.groovy | 4 +- .../appmaster/model/mock/MockRoleHistory.groovy | 3 +- 10 files changed, 182 insertions(+), 114 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aa46b473/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java index 50b5dad..5d52441 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java @@ -20,6 +20,7 @@ package org.apache.slider.server.appmaster; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.util.Records; +import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.slider.server.appmaster.state.AbstractClusterServices; public class ProtobufClusterServices extends AbstractClusterServices { @@ -27,4 +28,9 @@ public class ProtobufClusterServices extends AbstractClusterServices { public Resource newResource() { return Records.newRecord(Resource.class); } + + @Override + public Resource newResource(int memory, int cores) { + return Resources.createResource(memory, cores); + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aa46b473/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java index 27e25f9..eba8c38 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java @@ -19,6 +19,7 @@ package org.apache.slider.server.appmaster.state; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; /** * Cluster services offered by the YARN infrastructure. @@ -30,4 +31,9 @@ public abstract class AbstractClusterServices { */ public abstract Resource newResource(); + public abstract Resource newResource(int memory, int cores); + + public Resource normalize(Resource resource, Resource minR, Resource maxR) { + return new DefaultResourceCalculator().normalize(resource, minR, maxR); + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aa46b473/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index f6fe474..1e23bef 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -289,6 +289,8 @@ public class AppState { * Selector of containers to release; application wide. */ private ContainerReleaseSelector containerReleaseSelector; + private Resource minResource; + private Resource maxResource; /** * Create an instance @@ -309,7 +311,7 @@ public class AppState { register(MetricsConstants.CONTAINERS_START_FAILED, startFailedContainerCount); } - private void register(String name, Counter counter) { + private void register(String name, Metric counter) { this.metricsAndMonitoring.getMetrics().register( MetricRegistry.name(AppState.class, name), counter); } @@ -462,6 +464,8 @@ public class AppState { containerMaxCores = maxCores; containerMinMemory = minMemory; containerMaxMemory = maxMemory; + minResource = recordFactory.newResource(containerMinMemory, containerMinCores); + maxResource = recordFactory.newResource(containerMaxMemory, containerMaxCores); } public ConfTreeOperations getResourcesSnapshot() { @@ -556,7 +560,7 @@ public class AppState { // set up the role history - roleHistory = new RoleHistory(roleStatusMap.values()); + roleHistory = new RoleHistory(roleStatusMap.values(), recordFactory); roleHistory.register(metricsAndMonitoring); roleHistory.onStart(binding.fs, binding.historyPath); // trigger first node update @@ -762,6 +766,9 @@ public class AppState { newRoles.add(dynamicRole); } } + // and fill in all those roles with their requirements + buildRoleResourceRequirements(); + return newRoles; } @@ -812,6 +819,17 @@ public class AppState { } /** + * Build up the requirements of every resource + */ + private void buildRoleResourceRequirements() { + roleStatusMap.values(); + for (RoleStatus role : roleStatusMap.values()) { + role.setResourceRequirements( + buildResourceRequirements(role, recordFactory.newResource())); + } + } + + /** * build up the special master node, which lives * in the live node set but has a lifecycle bonded to the AM * @param containerId the AM master @@ -913,6 +931,12 @@ public class AppState { } + /** + * Look up a role in the map + * @param name role name + * @return the instance + * @throws YarnRuntimeException if not found + */ public RoleStatus lookupRoleStatus(String name) throws YarnRuntimeException { ProviderRole providerRole = roles.get(name); if (providerRole == null) { @@ -928,7 +952,7 @@ public class AppState { */ public synchronized List<RoleInstance> cloneOwnedContainerList() { Collection<RoleInstance> values = ownedContainers.values(); - return new ArrayList<RoleInstance>(values); + return new ArrayList<>(values); } /** @@ -1027,7 +1051,6 @@ public class AppState { } } - public synchronized List<RoleInstance> getLiveInstancesByContainerIDs( Collection<String> containerIDs) { //first, a hashmap of those containerIDs is built up @@ -1081,7 +1104,6 @@ public class AppState { return nodes; } - /** * Build an instance map. * @return the map of Role name to list of role instances @@ -1167,38 +1189,18 @@ public class AppState { /** - * Set up the resource requirements with all that this role needs, - * then create the container request itself. - * @param role role to ask an instance of - * @param capability a resource to set up - * @return the request for a new container - */ - public AMRMClient.ContainerRequest buildContainerResourceAndRequest( - RoleStatus role, - Resource capability) { - buildResourceRequirements(role, capability); - String labelExpression = role.getLabelExpression(); - //get the role history to select a suitable node, if available - AMRMClient.ContainerRequest containerRequest = - createContainerRequest(role, capability); - return containerRequest; - } - - /** * Create a container request. - * Update internal state, such as the role request count - * This is where role history information will be used for placement decisions - - * @param labelExpression label expression to satisfy + * Update internal state, such as the role request count. + * Anti-Affine: the {@link RoleStatus#outstandingAArequest} is set here. + * This is where role history information will be used for placement decisions. * @param role role - * @param resource requirements * @return the container request to submit */ - private AMRMClient.ContainerRequest createContainerRequest(RoleStatus role, - Resource resource) { - AMRMClient.ContainerRequest request; - request = roleHistory.requestNode(role, resource); + private AMRMClient.ContainerRequest createContainerRequest(RoleStatus role) { incrementRequestCount(role); - return request; + OutstandingRequest request = roleHistory.requestContainerForRole(role); + role.setOutstandingAArequest(request); + return request.getIssuedRequest(); } /** @@ -1266,9 +1268,10 @@ public class AppState { * cluster specification, including substituing max allowed values * if the specification asked for it. * @param role role - * @param capability capability to set up + * @param capability capability to set up. A new one may be created + * during normalization */ - public void buildResourceRequirements(RoleStatus role, Resource capability) { + public Resource buildResourceRequirements(RoleStatus role, Resource capability) { // Set up resource requirements from role values String name = role.getName(); ConfTreeOperations resources = getResourcesSnapshot(); @@ -1283,6 +1286,7 @@ public class AppState { DEF_YARN_MEMORY, containerMaxMemory); capability.setMemory(ram); + return recordFactory.normalize(capability,minResource, maxResource); } /** @@ -1748,7 +1752,7 @@ public class AppState { public synchronized List<AbstractRMOperation> reviewRequestAndReleaseNodes() throws SliderInternalStateException, TriggerClusterTeardownException { log.debug("in reviewRequestAndReleaseNodes()"); - List<AbstractRMOperation> allOperations = new ArrayList<AbstractRMOperation>(); + List<AbstractRMOperation> allOperations = new ArrayList<>(); for (RoleStatus roleStatus : getRoleStatusMap().values()) { if (!roleStatus.isExcludeFromFlexing()) { List<AbstractRMOperation> operations = reviewOneRole(roleStatus); @@ -1853,6 +1857,7 @@ public class AppState { long delta; long expected; String name = role.getName(); + boolean isAA = role.isAntiAffinePlacement(); synchronized (role) { delta = role.getDelta(); expected = role.getDesired(); @@ -1871,38 +1876,43 @@ public class AppState { } if (delta > 0) { - log.info("{}: Asking for {} more nodes(s) for a total of {} ", name, - delta, expected); // more workers needed than we have -ask for more - for (int i = 0; i < delta; i++) { - Resource capability = recordFactory.newResource(); - AMRMClient.ContainerRequest containerAsk = - buildContainerResourceAndRequest(role, capability); - log.info("Container ask is {} and label = {}", containerAsk, - containerAsk.getNodeLabelExpression()); - int askMemory = containerAsk.getCapability().getMemory(); - if (askMemory > this.containerMaxMemory) { - log.warn("Memory requested: {} > max of {}", askMemory, containerMaxMemory); + log.info("{}: Asking for {} more nodes(s) for a total of {} ", name, + delta, expected); + + // TODO: AA RH to help here by only allowing one request for an AA + + if (isAA) { + // build one only if there is none outstanding + if (role.getPendingAntiAffineRequests() == 0) { + log.info("Starting an anti-affine request sequence"); + role.incPendingAntiAffineRequests(delta); + addContainerRequest(operations, createContainerRequest(role)); + } else { + log.info("Adding {} more anti-affine requests", delta); + role.incPendingAntiAffineRequests(delta); + } + } else { + + for (int i = 0; i < delta; i++) { + //get the role history to select a suitable node, if available + addContainerRequest(operations, createContainerRequest(role)); } - operations.add(new ContainerRequestOperation(containerAsk)); } } else if (delta < 0) { log.info("{}: Asking for {} fewer node(s) for a total of {}", name, -delta, expected); // reduce the number expected (i.e. subtract the delta) - - // then pick some containers to kill long excess = -delta; - // how many requests are outstanding + // how many requests are outstanding? long outstandingRequests = role.getRequested(); if (outstandingRequests > 0) { // outstanding requests. int toCancel = (int)Math.min(outstandingRequests, excess); // Delegate to Role History - List<AbstractRMOperation> cancellations = roleHistory.cancelRequestsForRole(role, toCancel); log.info("Found {} outstanding requests to cancel", cancellations.size()); operations.addAll(cancellations); @@ -1922,7 +1932,6 @@ public class AppState { } } - // after the cancellation there may be no excess if (excess > 0) { @@ -1936,7 +1945,7 @@ public class AppState { log.info("No containers for component {}", roleId); } - // cut all release-in-progress nodes + // filter out all release-in-progress nodes ListIterator<RoleInstance> li = containersToRelease.listIterator(); while (li.hasNext()) { RoleInstance next = li.next(); @@ -1979,6 +1988,17 @@ public class AppState { return operations; } + private void addContainerRequest(List<AbstractRMOperation> operations, + AMRMClient.ContainerRequest containerAsk) { + log.info("Container ask is {} and label = {}", containerAsk, + containerAsk.getNodeLabelExpression()); + int askMemory = containerAsk.getCapability().getMemory(); + if (askMemory > this.containerMaxMemory) { + log.warn("Memory requested: {} > max of {}", askMemory, containerMaxMemory); + } + operations.add(new ContainerRequestOperation(containerAsk)); + } + /** * Releases a container based on container id * @param containerId http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aa46b473/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java index c93c7f5..f8271a6 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java @@ -73,6 +73,8 @@ public class RoleHistory { private final List<ProviderRole> providerRoles; /** the roles in here are shared with App State */ private final Map<Integer, RoleStatus> roleStatusMap = new HashMap<>(); + private final AbstractClusterServices recordFactory; + private long startTime; /** Time when saved */ @@ -115,9 +117,11 @@ public class RoleHistory { /** * Instantiate * @param roles initial role list + * @param recordFactory yarn record factory * @throws BadConfigException */ - public RoleHistory(Collection<RoleStatus> roles) throws BadConfigException { + public RoleHistory(Collection<RoleStatus> roles, AbstractClusterServices recordFactory) throws BadConfigException { + this.recordFactory = recordFactory; roleSize = roles.size(); providerRoles = new ArrayList<>(roleSize); for (RoleStatus role : roles) { @@ -604,24 +608,27 @@ public class RoleHistory { * * @param node node to target or null for "any" * @param role role to request - * @return the container priority + * @return the request */ - public synchronized AMRMClient.ContainerRequest requestInstanceOnNode( + public synchronized OutstandingRequest requestInstanceOnNode( NodeInstance node, RoleStatus role, Resource resource) { OutstandingRequest outstanding = outstandingRequests.newRequest(node, role.getKey()); - return outstanding.buildContainerRequest(resource, role, now()); + outstanding.buildContainerRequest(resource, role, now()); + return outstanding; } /** * Find a node for a role and request an instance on that (or a location-less * instance) * @param role role status - * @param resource resource capabilities * @return a request ready to go */ - public synchronized AMRMClient.ContainerRequest requestNode(RoleStatus role, - Resource resource) { + public synchronized OutstandingRequest requestContainerForRole(RoleStatus role) { + + Resource resource = recordFactory.newResource(); + role.copyResourceRequirements(resource); NodeInstance node = findNodeForNewInstance(role); + // TODO AA -what if there are no suitable nodes? return requestInstanceOnNode(node, role, resource); } @@ -992,6 +999,8 @@ public class RoleHistory { // ask for some placed nodes requests.addAll(outstandingRequests.extractPlacedRequestsForRole(roleId, remaining)); + // TODO AA: clear anything here? + // build cancellations for (OutstandingRequest request : requests) { results.add(request.createCancelOperation()); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aa46b473/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy index 157870a..42772c5 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy @@ -22,6 +22,8 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import org.apache.hadoop.yarn.api.records.Container import org.apache.hadoop.yarn.client.api.AMRMClient +import org.apache.slider.api.ResourceKeys +import org.apache.slider.providers.PlacementPolicy import org.apache.slider.providers.ProviderRole import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest import org.apache.slider.server.appmaster.model.mock.MockFactory @@ -43,16 +45,29 @@ import org.junit.Test class TestMockAppStateAAPlacement extends BaseMockAppStateTest implements MockRoles { - static private final ProviderRole aaRole = MockFactory.PROVIDER_ROLE2 - private static final int roleId = aaRole.id -/* + /** + * Patch up a "role2" role to have anti-affinity set + */ + public static final ProviderRole AAROLE = new ProviderRole( + MockRoles.ROLE2, + 2, + PlacementPolicy.ANTI_AFFINITY_REQUIRED, + 2, + 2, + null) + @Override AppStateBindingInfo buildBindingInfo() { def bindingInfo = super.buildBindingInfo() - // only have the AA role, to avoid complications/confusion - bindingInfo.roles = [aaRole] + bindingInfo.roles = [ + MockFactory.PROVIDER_ROLE0, + MockFactory.PROVIDER_ROLE1, + AAROLE, + ] bindingInfo - }*/ + } + + private static final int roleId = AAROLE.id /** * Get the single request of a list of operations; includes the check for the size @@ -74,7 +89,7 @@ class TestMockAppStateAAPlacement extends BaseMockAppStateTest @Test public void testAllocateAANoLabel() throws Throwable { - def aaRole = lookupRole(aaRole.name) + def aaRole = lookupRole(AAROLE.name) // want two instances, so there will be two iterations aaRole.desired = 2 http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aa46b473/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy index 5609682..d9cfddb 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy @@ -35,7 +35,6 @@ import org.apache.slider.server.appmaster.model.mock.MockFactory import org.apache.slider.server.appmaster.model.mock.MockNodeId import org.apache.slider.server.appmaster.model.mock.MockRoleHistory import org.apache.slider.server.appmaster.state.* -import org.junit.Before import org.junit.Test /** @@ -61,13 +60,18 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest { String roleName = "test" List<NodeInstance> nodes = [age2Active2, age2Active0, age4Active1, age1Active4, age3Active0] - RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES) + RoleHistory roleHistory Resource resource - @Before - public void setupRH() { - roleHistory.onStart(fs, historyPath) + AMRMClient.ContainerRequest requestContainer(RoleStatus roleStatus) { + roleHistory.requestContainerForRole(roleStatus).issuedRequest + } + + @Override + void setup() { + super.setup() + roleHistory = appState.roleHistory roleHistory.insert(nodes) roleHistory.buildRecentNodeLists(); resource = Resource.newInstance(ResourceKeys.DEF_YARN_CORES, @@ -77,11 +81,10 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest { @Test public void testFindAndCreate() throws Throwable { int role = 0 - ProviderRole provRole = new ProviderRole(roleName, role) - RoleStatus roleStatus = new RoleStatus(provRole) + RoleStatus roleStatus = appState.lookupRoleStatus(role) AMRMClient.ContainerRequest request = - roleHistory.requestNode(roleStatus, resource); + requestContainer(roleStatus); List<String> nodes = request.getNodes() assert nodes != null @@ -114,14 +117,13 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest { @Test public void testCreateAndRelease() throws Throwable { int role = 1 - ProviderRole provRole = new ProviderRole(roleName, role) - RoleStatus roleStatus = new RoleStatus(provRole) + RoleStatus roleStatus = appState.lookupRoleStatus(role) //verify it is empty assert roleHistory.listActiveNodes(role).empty AMRMClient.ContainerRequest request = - roleHistory.requestNode(roleStatus, resource); + requestContainer(roleStatus); assert request.nodes == null @@ -170,7 +172,7 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest { // ask for a container and expect to get the recently released one AMRMClient.ContainerRequest request2 = - roleHistory.requestNode(roleStatus, resource); + requestContainer(roleStatus); List<String> nodes2 = request2.nodes assert nodes2 != null @@ -206,11 +208,10 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest { @Test public void testStartFailed() throws Throwable { int role = 0 - ProviderRole provRole = new ProviderRole(roleName, role) - RoleStatus roleStatus = new RoleStatus(provRole) + RoleStatus roleStatus = appState.lookupRoleStatus(role) AMRMClient.ContainerRequest request = - roleHistory.requestNode(roleStatus, resource); + requestContainer(roleStatus); String hostname = request.getNodes()[0] assert hostname == age3Active0.hostname @@ -240,10 +241,10 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest { @Test public void testStartFailedWithoutWarning() throws Throwable { int role = 0 - ProviderRole provRole = new ProviderRole(roleName, role) - RoleStatus roleStatus = new RoleStatus(provRole) + RoleStatus roleStatus = appState.lookupRoleStatus(role) + AMRMClient.ContainerRequest request = - roleHistory.requestNode(roleStatus, resource); + requestContainer(roleStatus); String hostname = request.getNodes()[0] assert hostname == age3Active0.hostname @@ -271,10 +272,10 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest { describe("fail a container without declaring it as starting") int role = 0 - ProviderRole provRole = new ProviderRole(roleName, role) - RoleStatus roleStatus = new RoleStatus(provRole) + RoleStatus roleStatus = appState.lookupRoleStatus(role) + AMRMClient.ContainerRequest request = - roleHistory.requestNode(roleStatus, resource); + requestContainer(roleStatus); String hostname = request.getNodes()[0] assert hostname == age3Active0.hostname @@ -310,10 +311,10 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest { public void testContainerFailedWithoutWarning() throws Throwable { describe( "fail a container without declaring it as starting") int role = 0 - ProviderRole provRole = new ProviderRole(roleName, role) - RoleStatus roleStatus = new RoleStatus(provRole) + RoleStatus roleStatus = appState.lookupRoleStatus(role) + AMRMClient.ContainerRequest request = - roleHistory.requestNode(roleStatus, resource); + requestContainer(roleStatus); String hostname = request.getNodes()[0] assert hostname == age3Active0.hostname @@ -343,10 +344,10 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest { public void testAllocationListPrep() throws Throwable { describe("test prepareAllocationList") int role = 0 - ProviderRole provRole = new ProviderRole(roleName, role) - RoleStatus roleStatus = new RoleStatus(provRole) + RoleStatus roleStatus = appState.lookupRoleStatus(role) + AMRMClient.ContainerRequest request = - roleHistory.requestNode(roleStatus, resource); + requestContainer(roleStatus); String hostname = request.getNodes()[0] assert hostname == age3Active0.hostname @@ -375,9 +376,10 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest { describe("fail a node") int role = 0 - ProviderRole provRole = new ProviderRole(roleName, role) - RoleStatus roleStatus = new RoleStatus(provRole) - AMRMClient.ContainerRequest request = roleHistory.requestNode(roleStatus, resource); + RoleStatus roleStatus = appState.lookupRoleStatus(role) + + AMRMClient.ContainerRequest request = + roleHistory.requestContainerForRole(roleStatus).issuedRequest; String hostname = request.getNodes()[0] assert age3Active0.hostname == hostname http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aa46b473/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy index 693ea9f..14ac32a 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy @@ -63,6 +63,10 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { ProviderRole provRole = new ProviderRole(roleName, 0) RoleStatus roleStatus = new RoleStatus(provRole) + AMRMClient.ContainerRequest requestContainer(RoleStatus roleStatus) { + roleHistory.requestContainerForRole(roleStatus).issuedRequest + } + @Override String getTestName() { return "TestRoleHistoryAvailableList" @@ -72,6 +76,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { public void setupNodeMap() { roleHistory.insert(nodes) roleHistory.buildRecentNodeLists(); + roleStatus.setResourceRequirements(Resource.newInstance(1, 1)) } @Test @@ -107,8 +112,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { // which is translated to a no-location request AMRMClient.ContainerRequest req = roleHistory.requestInstanceOnNode(ni, roleStatus, - resource - ) + resource).issuedRequest assertNull(req.nodes) @@ -121,7 +125,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { // looking for a node should now find one ni = roleHistory.findNodeForNewInstance(roleStatus) assert ni == age3Active0 - req = roleHistory.requestInstanceOnNode(ni, roleStatus, resource) + req = roleHistory.requestInstanceOnNode(ni, roleStatus, resource).issuedRequest assert 1 == req.nodes.size() } @@ -162,7 +166,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { @Test public void testFindAndRequestNode() throws Throwable { - AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource) + AMRMClient.ContainerRequest req = requestContainer(roleStatus) assert age3Active0.hostname == req.nodes[0] List<NodeInstance> a2 = roleHistory.cloneRecentNodeList(0) @@ -171,7 +175,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { @Test public void testRequestedNodeIntoReqList() throws Throwable { - AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource) + requestContainer(roleStatus) List<OutstandingRequest> requests = roleHistory.listPlacedRequests() assert requests.size() == 1 assert age3Active0.hostname == requests[0].hostname @@ -179,7 +183,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { @Test public void testCompletedRequestDropsNode() throws Throwable { - AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource) + AMRMClient.ContainerRequest req = requestContainer(roleStatus) List<OutstandingRequest> requests = roleHistory.listPlacedRequests() assert requests.size() == 1 String hostname = requests[0].hostname @@ -208,8 +212,8 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { @Test public void testTwoRequests() throws Throwable { - AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource) - AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource) + AMRMClient.ContainerRequest req = requestContainer(roleStatus) + AMRMClient.ContainerRequest req2 = requestContainer(roleStatus) List<OutstandingRequest> requests = roleHistory.listPlacedRequests() assert requests.size() == 2 MockContainer container = factory.newContainer(req, req.nodes[0]) @@ -222,9 +226,9 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { @Test public void testThreeRequestsOneUnsatisified() throws Throwable { - AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource) - AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource) - AMRMClient.ContainerRequest req3 = roleHistory.requestNode(roleStatus, resource) + AMRMClient.ContainerRequest req = requestContainer(roleStatus) + AMRMClient.ContainerRequest req2 = requestContainer(roleStatus) + AMRMClient.ContainerRequest req3 = requestContainer(roleStatus) List<OutstandingRequest> requests = roleHistory.listPlacedRequests() assert requests.size() == 2 MockContainer container = factory.newContainer(req, req.nodes[0]) @@ -251,9 +255,9 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { @Test public void testThreeRequests() throws Throwable { - AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource) - AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource) - AMRMClient.ContainerRequest req3 = roleHistory.requestNode(roleStatus, resource) + AMRMClient.ContainerRequest req = requestContainer(roleStatus) + AMRMClient.ContainerRequest req2 = requestContainer(roleStatus) + AMRMClient.ContainerRequest req3 = requestContainer(roleStatus) assertOutstandingPlacedRequests(2) assert req3.nodes == null MockContainer container = factory.newContainer(req, req.nodes[0]) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aa46b473/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockClusterServices.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockClusterServices.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockClusterServices.groovy index d27a6bb..dfecc94 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockClusterServices.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockClusterServices.groovy @@ -27,4 +27,9 @@ class MockClusterServices extends AbstractClusterServices { Resource newResource() { return new MockResource() } + + @Override + Resource newResource(int memory, int cores) { + return new MockResource(memory, cores) + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aa46b473/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy index 0a4a93e..bbd64f1 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy @@ -59,11 +59,11 @@ class MockFactory implements MockRoles { 2, 1, ResourceKeys.DEF_YARN_LABEL_EXPRESSION) - // role 2: longer delay and anti-affinity + // role 2: longer delay public static final ProviderRole PROVIDER_ROLE2 = new ProviderRole( MockRoles.ROLE2, 2, - PlacementPolicy.ANTI_AFFINITY_REQUIRED, + PlacementPolicy.NO_DATA_LOCALITY, 2, 2, ResourceKeys.DEF_YARN_LABEL_EXPRESSION) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aa46b473/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.groovy index 0a68afb..4553e22 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.groovy @@ -35,7 +35,8 @@ class MockRoleHistory extends RoleHistory { * @throws BadConfigException configuration problem with the role list */ MockRoleHistory(List<ProviderRole> providerRoles) throws BadConfigException { - super(providerRoles.collect { new RoleStatus(it) }) + super(providerRoles.collect { new RoleStatus(it) }, + new MockClusterServices()) } }
