SLIDER-647 allocation requests not being satisfied when a cluster goes to labels
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c7186e13 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c7186e13 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c7186e13 Branch: refs/heads/master Commit: c7186e13ddd2c1e45e4faf84119b5bb2d04fd00e Parents: 0cb6eaf Author: Steve Loughran <[email protected]> Authored: Thu Nov 13 22:35:32 2014 +0000 Committer: Sumit Mohanty <[email protected]> Committed: Thu Nov 13 18:19:30 2014 -0800 ---------------------------------------------------------------------- .../slider/providers/PlacementPolicy.java | 22 ++- .../slider/server/appmaster/state/AppState.java | 22 ++- .../appmaster/state/OutstandingRequest.java | 2 +- .../server/appmaster/state/RoleHistory.java | 6 +- .../server/appmaster/state/RoleStatus.java | 4 + .../TestMockAppStateDynamicHistory.groovy | 3 +- .../TestMockAppStateDynamicRoles.groovy | 172 ++++++++++++++++++- .../TestMockAppStateRolePlacement.groovy | 5 + .../model/mock/BaseMockAppStateTest.groovy | 2 + .../appmaster/model/mock/MockYarnCluster.groovy | 10 +- .../appmaster/model/mock/MockYarnEngine.groovy | 6 + 11 files changed, 229 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c7186e13/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java b/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java index 444c041..dc6c910 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java +++ b/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java @@ -23,9 +23,29 @@ package org.apache.slider.providers; */ public class PlacementPolicy { + /** + * Default values + */ public static final int DEFAULT = 0; - public static final int EXCLUDE_FROM_FLEXING = 1; + + /** + * Strict placement: when asking for an instance for which there is + * history, mandate that it is strict + */ + public static final int STRICT = 1; + + /** + * No data locality; do not bother trying to ask for any location + */ public static final int NO_DATA_LOCALITY = 2; + /** + * Anti-affinity is mandatory. + */ public static final int ANTI_AFFINITY_REQUIRED = 4; + + /** + * Exclude from flexing; used internally to mark AMs. + */ + public static final int EXCLUDE_FROM_FLEXING = 16; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c7186e13/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index 0848173..9956db2 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -58,6 +58,7 @@ import org.apache.slider.core.exceptions.ErrorStrings; import org.apache.slider.core.exceptions.NoSuchNodeException; import org.apache.slider.core.exceptions.SliderInternalStateException; import org.apache.slider.core.exceptions.TriggerClusterTeardownException; +import org.apache.slider.providers.PlacementPolicy; import org.apache.slider.providers.ProviderRole; import org.apache.slider.server.appmaster.operations.AbstractRMOperation; import org.apache.slider.server.appmaster.operations.CancelRequestOperation; @@ -582,13 +583,15 @@ public class AppState { int priority = SliderUtils.parseAndValidate("value of " + name + " " + ResourceKeys.COMPONENT_PRIORITY, priOpt, 0, 1, -1); - log.info("Role {} assigned priority {}", name, priority); String placementOpt = component.getOption( - ResourceKeys.COMPONENT_PLACEMENT_POLICY, "0"); + ResourceKeys.COMPONENT_PLACEMENT_POLICY, + Integer.toString(PlacementPolicy.DEFAULT)); int placement = SliderUtils.parseAndValidate("value of " + name + " " + ResourceKeys.COMPONENT_PLACEMENT_POLICY, placementOpt, 0, 0, -1); - return new ProviderRole(name, priority, placement); + ProviderRole newRole = new ProviderRole(name, priority, placement); + log.info("New {} ", newRole); + return newRole; } @@ -998,17 +1001,17 @@ public class AppState { /** - * enum nodes by role ID, from either the active or live node list + * enum nodes by role ID, from either the owned or live node list * @param roleId role the container must be in - * @param active flag to indicate "use active list" rather than the smaller + * @param owned flag to indicate "use owned list" rather than the smaller * "live" list * @return a list of nodes, may be empty */ public synchronized List<RoleInstance> enumNodesWithRoleId(int roleId, - boolean active) { + boolean owned) { List<RoleInstance> nodes = new ArrayList<RoleInstance>(); Collection<RoleInstance> allRoleInstances; - allRoleInstances = active ? ownedContainers.values() : liveNodes.values(); + allRoleInstances = owned ? ownedContainers.values() : liveNodes.values(); for (RoleInstance node : allRoleInstances) { if (node.roleId == roleId) { nodes.add(node); @@ -1746,6 +1749,9 @@ public class AppState { // enum all active nodes that aren't being released List<RoleInstance> containersToRelease = enumNodesWithRoleId(roleId, true); + if (containersToRelease.isEmpty()) { + log.info("No containers for component {}", roleId); + } // cut all release-in-progress nodes ListIterator<RoleInstance> li = containersToRelease.listIterator(); @@ -1759,7 +1765,7 @@ public class AppState { // warn if the desired state can't be reaced int numberAvailableForRelease = containersToRelease.size(); if (numberAvailableForRelease < excess) { - log.warn("Not enough nodes to release, have {} and need {} more", + log.warn("Not enough containers to release, have {} and need {} more", numberAvailableForRelease, excess - numberAvailableForRelease); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c7186e13/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java index 4a05a1a..d6022e0 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java @@ -108,7 +108,7 @@ public final class OutstandingRequest { if (node != null) { hosts = new String[1]; hosts[0] = node.hostname; - relaxLocality = false; + relaxLocality = !role.isStrictPlacement(); // tell the node it is in play node.getOrCreate(roleId); log.info("Submitting request for container on {}", hosts[0]); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c7186e13/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java index 8de4b92..ce2ab0a 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java @@ -750,11 +750,7 @@ public class RoleHistory { available = false; } else { available = nodeEntry.containerCompleted(wasReleased); - boolean isFailedNode = failedNodes.contains(RoleHistoryUtils - .hostnameOf(container)); - if (!isFailedNode) { - maybeQueueNodeForWork(container, nodeEntry, available); - } + maybeQueueNodeForWork(container, nodeEntry, available); } touch(); return available; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c7186e13/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java index 734bbea..3c860d6 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java @@ -100,6 +100,10 @@ public final class RoleStatus implements Cloneable { public boolean getNoDataLocality() { return 0 != (getPlacementPolicy() & PlacementPolicy.NO_DATA_LOCALITY); } + + public boolean isStrictPlacement() { + return 0 != (getPlacementPolicy() & PlacementPolicy.STRICT); + } public synchronized int getDesired() { return desired; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c7186e13/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy index e06c2cb..aa7bb11 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy @@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId import org.apache.slider.api.ResourceKeys import org.apache.slider.core.conf.ConfTreeOperations import org.apache.slider.core.exceptions.BadConfigException +import org.apache.slider.providers.PlacementPolicy import org.apache.slider.providers.ProviderRole import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest import org.apache.slider.server.appmaster.model.mock.MockAppState @@ -87,7 +88,7 @@ class TestMockAppStateDynamicHistory extends BaseMockAppStateTest def dynamic = "dynamicRole" int role_priority_8 = 8 int desired = 1 - int placementPolicy = 0 + int placementPolicy = PlacementPolicy.DEFAULT // snapshot and patch existing spec def resources = ConfTreeOperations.fromInstance( appState.resourcesSnapshot.confTree) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c7186e13/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy index 902752c..83fb273 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy @@ -21,13 +21,18 @@ package org.apache.slider.server.appmaster.model.appstate import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.yarn.api.records.ContainerId import org.apache.slider.api.ResourceKeys -import org.apache.slider.common.tools.SliderUtils -import org.apache.slider.core.conf.ConfTreeOperations +import org.apache.slider.providers.PlacementPolicy import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest import org.apache.slider.server.appmaster.model.mock.MockAppState import org.apache.slider.server.appmaster.model.mock.MockRoles import org.apache.slider.server.appmaster.model.mock.MockYarnEngine +import org.apache.slider.server.appmaster.operations.AbstractRMOperation +import org.apache.slider.server.appmaster.operations.ContainerRequestOperation +import org.apache.slider.server.appmaster.state.AppState +import org.apache.slider.server.appmaster.state.ContainerPriority +import org.apache.slider.server.appmaster.state.RoleHistoryUtils import org.apache.slider.server.appmaster.state.RoleInstance import org.apache.slider.server.appmaster.state.SimpleReleaseSelector import org.junit.Test @@ -39,6 +44,10 @@ import org.junit.Test @Slf4j class TestMockAppStateDynamicRoles extends BaseMockAppStateTest implements MockRoles { + private static final String ROLE4 = "4" + private static final String ROLE5 = "5" + private static final int ID4 = 4 + private static final int ID5 = 5 @Override String getTestName() { @@ -52,7 +61,7 @@ class TestMockAppStateDynamicRoles extends BaseMockAppStateTest */ @Override MockYarnEngine createYarnEngine() { - return new MockYarnEngine(4, 4) + return new MockYarnEngine(8, 2) } @Override @@ -60,17 +69,25 @@ class TestMockAppStateDynamicRoles extends BaseMockAppStateTest super.initApp() appState = new MockAppState() appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES) - def instance = factory.newInstanceDefinition(0,0,0) def opts = [ + (ResourceKeys.COMPONENT_PRIORITY): ROLE4, (ResourceKeys.COMPONENT_INSTANCES): "1", - (ResourceKeys.COMPONENT_PRIORITY): "4", ] - instance.resourceOperations.components["dynamic"]= opts - - + + instance.resourceOperations.components[ROLE4]= opts + + def opts5 = [ + (ResourceKeys.COMPONENT_PRIORITY) : ROLE5, + (ResourceKeys.COMPONENT_INSTANCES): "1", + (ResourceKeys.COMPONENT_PLACEMENT_POLICY): + Integer.toString(PlacementPolicy.STRICT), + ] + + instance.resourceOperations.components[ROLE5]= opts5 + appState.buildInstance( instance, new Configuration(), @@ -88,7 +105,146 @@ class TestMockAppStateDynamicRoles extends BaseMockAppStateTest createAndStartNodes() appState.reviewRequestAndReleaseNodes() appState.getRoleHistory().dump(); + } + + /** + * Find all allocations for a specific role + * @param role role Id/priority + * @param actions source list + * @return found list + */ + List<ContainerRequestOperation> findAllocationsForRole(int role, + List<AbstractRMOperation> actions) { + List <ContainerRequestOperation > results = [] + actions.each { AbstractRMOperation operation -> + if (operation instanceof ContainerRequestOperation) { + def req = (ContainerRequestOperation) operation; + def reqrole = ContainerPriority.extractRole(req.request.priority) + if (role == reqrole) { + results << req + } + } + } + return results + } + + @Test + public void testStrictPlacementInitialRequest() throws Throwable { + log.info("Initial engine state = $engine") + List<AbstractRMOperation> actions = appState.reviewRequestAndReleaseNodes() + assert actions.size() == 2 + + // neither have locality at this point + assertRelaxLocalityFlag(ID4, null, true, actions) + assertRelaxLocalityFlag(ID5, null, true, actions) + } + + + @Test + public void testPolicyPropagation() throws Throwable { + assert !(appState.lookupRoleStatus(ROLE4).placementPolicy & PlacementPolicy.STRICT) + assert (appState.lookupRoleStatus(ROLE5).placementPolicy & PlacementPolicy.STRICT) + + } + + @Test + public void testLaxPlacementSecondRequestRole4() throws Throwable { + log.info("Initial engine state = $engine") + def role4 = appState.lookupRoleStatus(ROLE4) + def role5 = appState.lookupRoleStatus(ROLE5) + role4.desired = 1 + role5.desired = 0 + + def instances = createStartAndStopNodes([]) + assert instances.size() == 1 + + def instanceA = instances.find { RoleInstance instance -> + instance.roleId = ID4 + } + assert instanceA + def hostname = RoleHistoryUtils.hostnameOf(instanceA.container) + + + log.info("Allocated engine state = $engine") + assert engine.containerCount() == 1 + + assert role4.actual == 1 + // shrinking cluster + + role4.desired = 0 + appState.lookupRoleStatus(ROLE4).desired = 0 + def completionResults = [] + def containersToRelease = [] + instances = createStartAndStopNodes(completionResults) + assert engine.containerCount() == 0 + assert completionResults.size() == 1 + + // expanding: expect hostnames now + role4.desired = 1 + def actions = appState.reviewRequestAndReleaseNodes() + assert actions.size() == 1 + + assertRelaxLocalityFlag(ID4, "", true, actions) + ContainerRequestOperation cro = (ContainerRequestOperation) actions[0] + def nodes = cro.request.nodes + assert nodes.size() == 1 + assert hostname == nodes[0] + } + + @Test + public void testStrictPlacementSecondRequestRole5() throws Throwable { + log.info("Initial engine state = $engine") + def role4 = appState.lookupRoleStatus(ROLE4) + def role5 = appState.lookupRoleStatus(ROLE5) + role4.desired = 0 + role5.desired = 1 + + def instances = createStartAndStopNodes([]) + assert instances.size() == 1 + + def instanceA = instances.find { RoleInstance instance -> + instance.roleId = ID5 + } + assert instanceA + def hostname = RoleHistoryUtils.hostnameOf(instanceA.container) + + + log.info("Allocated engine state = $engine") + assert engine.containerCount() == 1 + + assert role5.actual == 1 + // shrinking cluster + + role5.desired = 0 + def completionResults = [] + def containersToRelease = [] + instances = createStartAndStopNodes(completionResults) + assert engine.containerCount() == 0 + assert completionResults.size() == 1 + assert role5.actual == 0 + + role5.desired = 1 + def actions = appState.reviewRequestAndReleaseNodes() + assert actions.size() == 1 + assertRelaxLocalityFlag(ID5, "", false, actions) + ContainerRequestOperation cro = (ContainerRequestOperation) actions[0] + def nodes = cro.request.nodes + assert nodes.size() == 1 + assert hostname == nodes[0] + + } + + public void assertRelaxLocalityFlag( + int id, + String expectedHost, + boolean expectedRelaxFlag, + List<AbstractRMOperation> actions) { + def requests + requests = findAllocationsForRole(id, actions) + assert requests.size() == 1 + def req = requests[0] + assert expectedRelaxFlag == req.request.relaxLocality } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c7186e13/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy index 6df8910..e9de390 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy @@ -56,6 +56,10 @@ class TestMockAppStateRolePlacement extends BaseMockAppStateTest List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes() ContainerRequestOperation operation = (ContainerRequestOperation) ops[0] AMRMClient.ContainerRequest request = operation.request + assert request.relaxLocality + assert request.nodes == null + assert request.racks == null + Container allocated = engine.allocateContainer(request) List<ContainerAssignment> assignments = []; List<AbstractRMOperation> operations = [] @@ -97,6 +101,7 @@ class TestMockAppStateRolePlacement extends BaseMockAppStateTest AMRMClient.ContainerRequest request2 = operation.request assert request2 != null assert request2.nodes[0] == containerHostname + assert request2.relaxLocality engine.execute(ops) } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c7186e13/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy index 50d7e06..6c83c55 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy @@ -223,6 +223,7 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles List<AppState.NodeCompletionResult> completionResults, List<ContainerId> released) { for (RoleInstance instance : instances) { + log.debug("Started ${instance.role} on ${instance.id} ") assert appState.onNodeManagerContainerStarted(instance.containerId) } releaseContainers(completionResults, @@ -297,6 +298,7 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles RoleInstance ri = roleInstance(assigned) instances << ri //tell the app it arrived + log.debug("Start submitted ${ri.role} on ${container.id} ") appState.containerStartSubmitted(container, ri); } return instances http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c7186e13/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy index fe06d7c..6056e3a 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy @@ -56,6 +56,12 @@ public class MockYarnCluster { build(); } + @Override + String toString() { + return "MockYarnCluster size=$clusterSize, capacity=${totalClusterCapacity()},"+ + " in use=${containersInUse()}" + } + /** * Build the cluster. */ @@ -90,7 +96,9 @@ public class MockYarnCluster { */ MockYarnClusterContainer release(ContainerId cid) { int host = extractHost(cid.id) - return nodeAt(host).release(cid.id) + def inUse = nodeAt(host).release(cid.id) + log.debug("Released $cid inuse=$inUse") + return inUse } int containersInUse() { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c7186e13/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy index f405188..04466c6 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy @@ -50,6 +50,11 @@ class MockYarnEngine { attemptId: 1, ) + @Override + String toString() { + return "MockYarnEngine $cluster + pending=${pending.size()}" + } + int containerCount() { return cluster.containersInUse(); } @@ -109,6 +114,7 @@ class MockYarnEngine { } else { ContainerRequestOperation req = (ContainerRequestOperation) op Container container = allocateContainer(req.request) + log.info("allocated container $container for $req") if (container != null) { allocation.add(container) } else {
