SLIDER-82: label information is set in the ProviderRole, no longer directly managed in AppState. Needed for building a sequence of AA placement requests
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/07fb45df Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/07fb45df Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/07fb45df Branch: refs/heads/feature/SLIDER-82-pass-3.1 Commit: 07fb45dfacf5147fa3eba6ffade3f64ce4a69741 Parents: 138912c Author: Steve Loughran <[email protected]> Authored: Fri Nov 6 12:34:45 2015 +0000 Committer: Steve Loughran <[email protected]> Committed: Fri Nov 6 13:01:24 2015 +0000 ---------------------------------------------------------------------- .../apache/slider/providers/ProviderRole.java | 25 +++-- .../slideram/SliderAMClientProvider.java | 2 +- .../slider/server/appmaster/state/AppState.java | 111 +++++++++---------- .../appmaster/state/OutstandingRequest.java | 15 ++- .../server/appmaster/state/RoleHistory.java | 23 +--- .../server/appmaster/state/RoleStatus.java | 14 ++- .../resources/AggregateConfResource.java | 26 ++--- .../appstate/TestMockAppStateAAPlacement.groovy | 2 +- ...tRoleHistoryOutstandingRequestTracker.groovy | 51 ++++++--- .../model/history/TestRoleHistoryRW.groovy | 4 +- .../TestRoleHistoryRequestTracking.groovy | 10 +- .../model/mock/BaseMockAppStateTest.groovy | 10 +- .../appmaster/model/mock/MockFactory.groovy | 9 +- 13 files changed, 152 insertions(+), 150 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/07fb45df/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java index 3009f50..1b95b42 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java +++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java @@ -32,13 +32,15 @@ public final class ProviderRole { public int placementPolicy; public int nodeFailureThreshold; public final long placementTimeoutSeconds; + public final String labelExpression; public ProviderRole(String name, int id) { this(name, id, PlacementPolicy.DEFAULT, ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD, - ResourceKeys.DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS); + ResourceKeys.DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS, + ResourceKeys.DEF_YARN_LABEL_EXPRESSION); } /** @@ -49,18 +51,20 @@ public final class ProviderRole { * @param nodeFailureThreshold threshold for node failures (within a reset interval) * after which a node failure is considered an app failure * @param placementTimeoutSeconds for lax placement, timeout in seconds before - * a relaxed placement request is generated. + * @param labelExpression label expression for requests; may be null */ public ProviderRole(String name, int id, int policy, int nodeFailureThreshold, - long placementTimeoutSeconds) { + long placementTimeoutSeconds, + String labelExpression) { this.name = name; this.id = id; this.placementPolicy = policy; this.nodeFailureThreshold = nodeFailureThreshold; this.placementTimeoutSeconds = placementTimeoutSeconds; + this.labelExpression = labelExpression; } @Override @@ -83,11 +87,14 @@ public final class ProviderRole { @Override public String toString() { - return "ProviderRole {" + - "name='" + name + '\'' + - ", id=" + id + - ", policy=" + placementPolicy + - ", nodeFailureThreshold=" + nodeFailureThreshold + - '}'; + final StringBuilder sb = new StringBuilder("ProviderRole{"); + sb.append("name='").append(name).append('\''); + sb.append(", id=").append(id); + sb.append(", placementPolicy=").append(placementPolicy); + sb.append(", nodeFailureThreshold=").append(nodeFailureThreshold); + sb.append(", placementTimeoutSeconds=").append(placementTimeoutSeconds); + sb.append(", labelExpression='").append(labelExpression).append('\''); + sb.append('}'); + return sb.toString(); } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/07fb45df/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java index 9bd4dc9..3be0f48 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java +++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java @@ -90,7 +90,7 @@ public class SliderAMClientProvider extends AbstractClientProvider new ProviderRole(COMPONENT_AM, KEY_AM, PlacementPolicy.EXCLUDE_FROM_FLEXING, ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD, - 0); + 0, ""); /** * Initialize role list http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/07fb45df/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index e47ef34..946d45f 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -22,7 +22,6 @@ import com.codahale.metrics.Counter; import com.codahale.metrics.MetricRegistry; import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -141,7 +140,7 @@ public class AppState { private ConfTreeOperations resourcesSnapshot; private ConfTreeOperations appConfSnapshot; private ConfTreeOperations internalsSnapshot; - + /** * This is the status, the live model */ @@ -151,7 +150,7 @@ public class AppState { * Metadata provided by the AM for use in filling in status requests */ private Map<String, String> applicationInfo; - + /** * Client properties created via the provider -static for the life * of the application @@ -278,10 +277,10 @@ public class AppState { private RoleHistory roleHistory; private Configuration publishedProviderConf; private long startTimeThreshold; - + private int failureThreshold = 10; private int nodeFailureThreshold = 3; - + private String logServerURL = ""; /** @@ -618,7 +617,7 @@ public class AppState { ResourceKeys.COMPONENT_PLACEMENT_POLICY, Integer.toString(PlacementPolicy.DEFAULT)); int placement = SliderUtils.parseAndValidate("value of " + name + " " + - ResourceKeys.COMPONENT_PLACEMENT_POLICY, + ResourceKeys.COMPONENT_PLACEMENT_POLICY, placementOpt, 0, 0, -1); int placementTimeout = component.getOptionInt(ResourceKeys.PLACEMENT_ESCALATE_DELAY, @@ -627,7 +626,8 @@ public class AppState { priority, placement, getNodeFailureThresholdForRole(name), - placementTimeout); + placementTimeout, + component.getOption(YARN_LABEL_EXPRESSION, DEF_YARN_LABEL_EXPRESSION)); log.info("New {} ", newRole); return newRole; } @@ -653,7 +653,7 @@ public class AppState { log.debug("Instance definition updated"); //note the time snapshotTime = now(); - + // resolve references if not already done instanceDefinition.resolve(); @@ -682,15 +682,14 @@ public class AppState { clusterStatusTemplate = ClusterDescriptionOperations.buildFromInstanceDefinition( instanceDefinition); - // Add the -site configuration properties for (Map.Entry<String, String> prop : clientProperties.entrySet()) { clusterStatusTemplate.clientProperties.put(prop.getKey(), prop.getValue()); } - + } - + /** * The resource configuration is updated -review and update state. * @param resources updated resources specification @@ -723,9 +722,9 @@ public class AppState { private List<ProviderRole> buildRoleRequirementsFromResources() throws BadConfigException { List<ProviderRole> newRoles = new ArrayList<>(0); - - //now update every role's desired count. - //if there are no instance values, that role count goes to zero + + // now update every role's desired count. + // if there are no instance values, that role count goes to zero ConfTreeOperations resources = instanceDefinition.getResourceOperations(); @@ -751,8 +750,8 @@ public class AppState { } } - //now the dynamic ones. Iterate through the the cluster spec and - //add any role status entries not in the role status + // now the dynamic ones. Iterate through the the cluster spec and + // add any role status entries not in the role status Set<String> roleNames = resources.getComponentNames(); for (String name : roleNames) { if (!roles.containsKey(name)) { @@ -803,7 +802,7 @@ public class AppState { * @throws BadConfigException if a role of that priority already exists */ public RoleStatus buildRole(ProviderRole providerRole) throws BadConfigException { - //build role status map + // build role status map int priority = providerRole.id; if (roleStatusMap.containsKey(priority)) { throw new BadConfigException("Duplicate Provider Key: %s and %s", @@ -843,7 +842,7 @@ public class AppState { //it is also added to the set of live nodes getLiveContainers().put(containerId, am); putOwnedContainer(containerId, am); - + // patch up the role status RoleStatus roleStatus = roleStatusMap.get( (SliderKeys.ROLE_AM_PRIORITY_INDEX)); @@ -876,7 +875,12 @@ public class AppState { return appMasterNode; } - + /** + * Look up the status entry of a role or raise an exception + * @param key role ID + * @return the status entry + * @throws RuntimeException if the role cannot be found + */ public RoleStatus lookupRoleStatus(int key) { RoleStatus rs = getRoleStatusMap().get(key); if (rs == null) { @@ -884,8 +888,15 @@ public class AppState { } return rs; } - - public RoleStatus lookupRoleStatus(Container c) throws YarnRuntimeException { + + /** + * Look up the status entry of a container or raise an exception + * + * @param c container + * @return the status entry + * @throws RuntimeException if the role cannot be found + */ + public RoleStatus lookupRoleStatus(Container c) { return lookupRoleStatus(ContainerPriority.extractRole(c)); } @@ -1093,8 +1104,7 @@ public class AppState { } return map; } - - + /** * Build a map of role->nodename->node-info * @@ -1173,10 +1183,10 @@ public class AppState { RoleStatus role, Resource capability) { buildResourceRequirements(role, capability); - String labelExpression = getLabelExpression(role); + String labelExpression = role.getLabelExpression(); //get the role history to select a suitable node, if available AMRMClient.ContainerRequest containerRequest = - createContainerRequest(role, capability, labelExpression); + createContainerRequest(role, capability); return containerRequest; } @@ -1184,18 +1194,15 @@ public class AppState { * Create a container request. * Update internal state, such as the role request count * This is where role history information will be used for placement decisions - + * @param labelExpression label expression to satisfy * @param role role * @param resource requirements - * @param labelExpression label expression to satisfy * @return the container request to submit */ private AMRMClient.ContainerRequest createContainerRequest(RoleStatus role, - Resource resource, - String labelExpression) { - - + Resource resource) { AMRMClient.ContainerRequest request; - request = roleHistory.requestNode(role, resource, labelExpression); + request = roleHistory.requestNode(role, resource); incrementRequestCount(role); return request; } @@ -1211,7 +1218,6 @@ public class AppState { incOutstandingContainerRequests(); } - /** * dec requested count of a role * <p> @@ -1261,7 +1267,7 @@ public class AppState { String option, int defVal, int maxVal) { - + String val = resources.getComponentOpt(name, option, Integer.toString(defVal)); Integer intVal; @@ -1273,7 +1279,6 @@ public class AppState { return intVal; } - /** * Build up the resource requirements for this role from the * cluster specification, including substituing max allowed values @@ -1299,17 +1304,6 @@ public class AppState { } /** - * Extract the label expression for this role. - * @param role role - */ - public String getLabelExpression(RoleStatus role) { - // Set up resource requirements from role values - String name = role.getName(); - ConfTreeOperations resources = getResourcesSnapshot(); - return resources.getComponentOpt(name, YARN_LABEL_EXPRESSION, DEF_YARN_LABEL_EXPRESSION); - } - - /** * add a launched container to the node map for status responses * @param container id * @param node node details @@ -1471,10 +1465,10 @@ public class AppState { return sb.toString(); } } - + /** * handle completed node in the CD -move something from the live - * server list to the completed server list + * server list to the completed server list. * @param status the node that has just completed * @return NodeCompletionResult */ @@ -1534,7 +1528,7 @@ public class AppState { if (failedContainer != null) { String completedLogsUrl = getLogsURLForContainer(failedContainer); message = String.format("Failure %s on host %s (%d): %s", - roleInstance.getContainerId().toString(), + roleInstance.getContainerId(), failedContainer.getNodeId().getHost(), exitStatus, completedLogsUrl); @@ -1583,10 +1577,10 @@ public class AppState { log.warn("Received notification of completion of unknown node {}", id); completionOfNodeNotInLiveListEvent.incrementAndGet(); } - + // and the active node list if present removeOwnedContainer(containerId); - + // finally, verify the node doesn't exist any more assert !containersBeingReleased.containsKey( containerId) : "container still in release queue"; @@ -1621,9 +1615,6 @@ public class AppState { return completedLogsUrl; } - - - /** * Return the percentage done that Slider is to have YARN display in its * Web UI @@ -1652,7 +1643,7 @@ public class AppState { public ClusterDescription refreshClusterStatus() { return refreshClusterStatus(null); } - + /** * Update the cluster description with the current application state * @param providerStatus status from the provider for the cluster info section @@ -1702,13 +1693,12 @@ public class AppState { cd.statistics.put(rolename, stats); } - Map<String, Integer> sliderstats = getLiveStatistics(); cd.statistics.put(SliderKeys.COMPONENT_AM, sliderstats); - + // liveness cd.liveness = getApplicationLivenessInformation(); - + return cd; } @@ -1723,7 +1713,7 @@ public class AppState { li.allRequestsSatisfied = outstanding <= 0; return li; } - + /** * Get the live statistics map * @return a map of statistics values, defined in the {@link StatusKeys} @@ -1861,7 +1851,7 @@ public class AppState { public List<AbstractRMOperation> escalateOutstandingRequests() { return roleHistory.escalateOutstandingRequests(); } - + /** * Look at the allocation status of one role, and trigger add/release * actions if the number of desired role instances doesn't equal @@ -2103,7 +2093,6 @@ public class AppState { //get the role final ContainerId cid = container.getId(); final RoleStatus role = lookupRoleStatus(container); - //dec requested count decrementRequestCount(role); @@ -2213,7 +2202,7 @@ public class AppState { cid, roleName, containerHostInfo); - + //update app state internal structures and maps RoleInstance instance = new RoleInstance(container); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/07fb45df/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java index 85bd259..38bc96f 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java @@ -71,21 +71,21 @@ public final class OutstandingRequest { /** * Requested time in millis. * <p> - * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long, String)} + * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long)} */ private AMRMClient.ContainerRequest issuedRequest; /** * Requested time in millis. * <p> - * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long, String)} + * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long)} */ private long requestedTimeMillis; /** * Time in millis after which escalation should be triggered.. * <p> - * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long, String)} + * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long)} */ private long escalationTimeoutMillis; @@ -178,16 +178,15 @@ public final class OutstandingRequest { * @param resource resource * @param role role * @param time time in millis to record as request time - * @param labelExpression label to satisfy * @return the request to raise */ public synchronized AMRMClient.ContainerRequest buildContainerRequest( - Resource resource, RoleStatus role, long time, String labelExpression) { + Resource resource, RoleStatus role, long time) { Preconditions.checkArgument(resource != null, "null `resource` arg"); Preconditions.checkArgument(role != null, "null `role` arg"); // cache label for escalation - label = labelExpression; + label = role.getLabelExpression(); requestedTimeMillis = time; escalationTimeoutMillis = time + role.getPlacementTimeoutSeconds() * 1000; String[] hosts; @@ -218,7 +217,7 @@ public final class OutstandingRequest { escalated = true; // and forbid it happening mayEscalate = false; - nodeLabels = labelExpression; + nodeLabels = label; } Priority pri = ContainerPriority.createPriority(roleId, !relaxLocality); priority = pri.getPriority(); @@ -254,7 +253,7 @@ public final class OutstandingRequest { String[] nodes; List<String> issuedRequestNodes = issuedRequest.getNodes(); - if (label == null && issuedRequestNodes != null) { + if (SliderUtils.isUnset(label) && issuedRequestNodes != null) { nodes = issuedRequestNodes.toArray(new String[issuedRequestNodes.size()]); } else { nodes = null; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/07fb45df/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java index a0aa3bc..34340a2 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java @@ -611,31 +611,14 @@ public class RoleHistory { * Returns the request that is now being tracked. * If the node instance is not null, it's details about the role is incremented * - * * @param node node to target or null for "any" * @param role role to request - * @param labelExpression label to satisfy * @return the container priority */ public synchronized AMRMClient.ContainerRequest requestInstanceOnNode( - NodeInstance node, RoleStatus role, Resource resource, String labelExpression) { + NodeInstance node, RoleStatus role, Resource resource) { OutstandingRequest outstanding = outstandingRequests.newRequest(node, role.getKey()); - return outstanding.buildContainerRequest(resource, role, now(), labelExpression); - } - - /** - * Find a node for a role and request an instance on that (or a location-less - * instance) with a label expression - * @param role role status - * @param resource resource capabilities - * @param labelExpression label to satisfy - * @return a request ready to go - */ - public synchronized AMRMClient.ContainerRequest requestNode(RoleStatus role, - Resource resource, - String labelExpression) { - NodeInstance node = findNodeForNewInstance(role); - return requestInstanceOnNode(node, role, resource, labelExpression); + return outstanding.buildContainerRequest(resource, role, now()); } /** @@ -648,7 +631,7 @@ public class RoleHistory { public synchronized AMRMClient.ContainerRequest requestNode(RoleStatus role, Resource resource) { NodeInstance node = findNodeForNewInstance(role); - return requestInstanceOnNode(node, role, resource, null); + return requestInstanceOnNode(node, role, resource); } /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/07fb45df/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java index 52df406..20f5802 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java @@ -41,7 +41,6 @@ public final class RoleStatus implements Cloneable { * Role priority */ private final int key; - private final ProviderRole providerRole; private int desired, actual, requested, releasing; @@ -315,7 +314,7 @@ public final class RoleStatus implements Cloneable { ", actual=" + actual + ", requested=" + requested + ", releasing=" + releasing + - ", pendingAntiAffineRequestCount=" + pendingAntiAffineRequestCount + + ", pendingAntiAffineRequestCount=" + pendingAntiAffineRequestCount + ", failed=" + failed + ", failed recently=" + failedRecently.get() + ", node failed=" + nodeFailed.get() + @@ -324,6 +323,7 @@ public final class RoleStatus implements Cloneable { ", startFailed=" + startFailed + ", completed=" + completed + ", failureMessage='" + failureMessage + '\'' + + ", providerRole=" + providerRole + '}'; } @@ -373,7 +373,15 @@ public final class RoleStatus implements Cloneable { info.pendingAntiAffineRequestCount = pendingAntiAffineRequestCount; return info; } - + + /** + * Get the (possibly null) label expression for this role + * @return a string or null + */ + public String getLabelExpression() { + return providerRole.labelExpression; + } + /** * Compare two role status entries by name */ http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/07fb45df/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java index 75d417b..ebffd1a 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java @@ -37,23 +37,13 @@ public class AggregateConfResource { public AggregateConfResource(AggregateConf conf, UriBuilder uriBuilder) { if (uriBuilder != null) { - this.href = - uriBuilder.build().toASCIIString(); - resources = - ResourceFactory.createConfTreeResource(conf.getAppConf(), - uriBuilder.clone().path( - "configurations").path( - "resources")); - internal = - ResourceFactory.createConfTreeResource(conf.getInternal(), - uriBuilder.clone().path( - "configurations").path( - "internal")); - appConf = - ResourceFactory.createConfTreeResource(conf.getAppConf(), - uriBuilder.clone().path( - "configurations").path( - "appConf")); + this.href = uriBuilder.build().toASCIIString(); + resources = ResourceFactory.createConfTreeResource(conf.getAppConf(), + uriBuilder.clone().path("configurations").path("resources")); + internal = ResourceFactory.createConfTreeResource(conf.getInternal(), + uriBuilder.clone().path("configurations").path("internal")); + appConf = ResourceFactory.createConfTreeResource(conf.getAppConf(), + uriBuilder.clone().path("configurations").path("appConf")); initConfMap(); } else { resources = null; @@ -63,7 +53,7 @@ public class AggregateConfResource { } private void initConfMap() { - confMap = new HashMap<String, ConfTreeResource>(); + confMap = new HashMap<>(); confMap.put("internal", internal); confMap.put("resources", resources); confMap.put("appConf", appConf); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/07fb45df/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy index 0e9fad0..6168146 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy @@ -44,7 +44,7 @@ import static org.apache.slider.server.appmaster.state.ContainerPriority.extract class TestMockAppStateAAPlacement extends BaseMockAppStateTest implements MockRoles { - @Test +// @Test public void testAllocateAA() throws Throwable { def aaRole = role2Status http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/07fb45df/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy index 8399d53..745d40f 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy @@ -27,6 +27,7 @@ import org.apache.slider.server.appmaster.model.mock.MockResource import org.apache.slider.server.appmaster.operations.AbstractRMOperation import org.apache.slider.server.appmaster.operations.CancelSingleRequest import org.apache.slider.server.appmaster.operations.ContainerRequestOperation +import org.apache.slider.server.appmaster.state.AppStateBindingInfo import org.apache.slider.server.appmaster.state.ContainerAllocationOutcome import org.apache.slider.server.appmaster.state.ContainerPriority import org.apache.slider.server.appmaster.state.NodeInstance @@ -37,12 +38,28 @@ import org.junit.Test class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest { + public static final String WORKERS_LABEL = "workers" NodeInstance host1 = new NodeInstance("host1", 3) NodeInstance host2 = new NodeInstance("host2", 3) def resource = factory.newResource(48, 1) OutstandingRequestTracker tracker = new OutstandingRequestTracker() + public static final ProviderRole WORKER = new ProviderRole( + "worker", + 5, + PlacementPolicy.NONE, + 2, + 1, + WORKERS_LABEL) + + @Override + AppStateBindingInfo buildBindingInfo() { + def bindingInfo = super.buildBindingInfo() + bindingInfo.roles = [ WORKER ] + bindingInfo.roles + bindingInfo + } + @Test public void testAddRetrieveEntry() throws Throwable { OutstandingRequest request = tracker.newRequest(host1, 0) @@ -54,10 +71,10 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest { @Test public void testAddCompleteEntry() throws Throwable { def req1 = tracker.newRequest(host1, 0) - req1.buildContainerRequest(resource, role0Status, 0, "") + req1.buildContainerRequest(resource, role0Status, 0) - tracker.newRequest(host2, 0).buildContainerRequest(resource, role0Status, 0, "") - tracker.newRequest(host1, 1).buildContainerRequest(resource, role0Status, 0, "") + tracker.newRequest(host2, 0).buildContainerRequest(resource, role0Status, 0) + tracker.newRequest(host1, 1).buildContainerRequest(resource, role0Status, 0) def allocation = tracker.onContainerAllocated(1, "host1", null) assert allocation.outcome == ContainerAllocationOutcome.Placed @@ -82,7 +99,7 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest { @Test public void testRemoveOpenRequestUnissued() throws Throwable { def req1 = tracker.newRequest(null, 0) - req1.buildContainerRequest(resource, role0Status, 0, "") + req1.buildContainerRequest(resource, role0Status, 0) assert tracker.listOpenRequests().size() == 1 def c1 = factory.newContainer(null, new MockPriority(0)) c1.resource = resource @@ -97,7 +114,7 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest { @Test public void testIssuedOpenRequest() throws Throwable { def req1 = tracker.newRequest(null, 0) - req1.buildContainerRequest(resource, role0Status, 0, "") + req1.buildContainerRequest(resource, role0Status, 0) assert tracker.listOpenRequests().size() == 1 def pri = ContainerPriority.buildPriority(0, false) @@ -142,7 +159,7 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest { // first request: default placement assert role0Status.placementPolicy == PlacementPolicy.DEFAULT; final def (res0, outstanding0) = newRequest(role0Status) - final def initialRequest = outstanding0.buildContainerRequest(res0, role0Status, 0, null) + final def initialRequest = outstanding0.buildContainerRequest(res0, role0Status, 0) assert outstanding0.issuedRequest != null; assert outstanding0.located assert !outstanding0.escalated @@ -178,7 +195,7 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest { // build that second request from an anti-affine entry // these get placed as well now += interval - final def containerReq2 = outstanding2.buildContainerRequest(res2, role2Status, now, null) + final def containerReq2 = outstanding2.buildContainerRequest(res2, role2Status, now) // escalate a little bit more final List<AbstractRMOperation> escalations2 = tracker.escalateOutstandingRequests(now) // and expect no new entries @@ -219,18 +236,18 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest { resource.virtualCores = 1 resource.memory = 48; - def label = "workers" + def workerRole = lookupRole(WORKER.name) // initial request - def yarnRequest = req1.buildContainerRequest(resource, role0Status, 0, label) + def yarnRequest = req1.buildContainerRequest(resource, workerRole, 0) assert (yarnRequest.nodeLabelExpression == null) assert (!yarnRequest.relaxLocality) def yarnRequest2 = req1.escalate() - assert (yarnRequest2.nodeLabelExpression == label) + assert (yarnRequest2.nodeLabelExpression == WORKERS_LABEL) assert (yarnRequest2.relaxLocality) } /** - * If the placement doesnt include a lablel, then the escalation request + * If the placement doesnt include a label, then the escalation request * retains the node list, but sets relaxLocality==true * @throws Throwable */ @@ -244,13 +261,13 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest { def label = null // initial request - def yarnRequest = req1.buildContainerRequest(resource, role0Status, 0, label) - assert (yarnRequest.nodes != null) - assert (yarnRequest.nodeLabelExpression == null) - assert (!yarnRequest.relaxLocality) + def yarnRequest = req1.buildContainerRequest(resource, role0Status, 0) + assert yarnRequest.nodes != null + assert !yarnRequest.nodeLabelExpression + assert !yarnRequest.relaxLocality def yarnRequest2 = req1.escalate() - assert (yarnRequest2.nodes != null) - assert (yarnRequest2.relaxLocality) + assert yarnRequest2.nodes != null + assert yarnRequest2.relaxLocality } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/07fb45df/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy index a1e424f..254c0b6 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy @@ -22,6 +22,7 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import org.apache.hadoop.fs.FSDataOutputStream import org.apache.hadoop.fs.Path +import org.apache.slider.api.ResourceKeys import org.apache.slider.providers.PlacementPolicy import org.apache.slider.providers.ProviderRole import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest @@ -46,7 +47,8 @@ class TestRoleHistoryRW extends BaseMockAppStateTest { 3, PlacementPolicy.STRICT, 3, - 3) + 3, + ResourceKeys.DEF_YARN_LABEL_EXPRESSION) @Override String getTestName() { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/07fb45df/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy index 42d0c50..c6dcb07 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy @@ -86,8 +86,8 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { assertListEquals([age2Active0], roleHistory.cloneRecentNodeList(0)) roleHistory.requestInstanceOnNode(ni, roleStatus, - resource, - "") + resource + ) } @Test @@ -106,8 +106,8 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { // which is translated to a no-location request AMRMClient.ContainerRequest req = roleHistory.requestInstanceOnNode(ni, roleStatus, - resource, - "") + resource + ) assertNull(req.nodes) @@ -120,7 +120,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { // looking for a node should now find one ni = roleHistory.findNodeForNewInstance(roleStatus) assert ni == age3Active0 - req = roleHistory.requestInstanceOnNode(ni, roleStatus, resource, "") + req = roleHistory.requestInstanceOnNode(ni, roleStatus, resource) assert 1 == req.nodes.size() } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/07fb45df/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy index a065518..40d7fd7 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy @@ -128,15 +128,19 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles } public RoleStatus getRole0Status() { - return appState.lookupRoleStatus(ROLE0) + lookupRole(ROLE0) + } + + public RoleStatus lookupRole(String role) { + appState.lookupRoleStatus(role) } public RoleStatus getRole1Status() { - return appState.lookupRoleStatus(ROLE1) + lookupRole(ROLE1) } public RoleStatus getRole2Status() { - return appState.lookupRoleStatus(ROLE2) + lookupRole(ROLE2) } /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/07fb45df/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy index f7fd641..0a4a93e 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy @@ -49,21 +49,24 @@ class MockFactory implements MockRoles { 0, PlacementPolicy.DEFAULT, 2, - 1) + 1, + ResourceKeys.DEF_YARN_LABEL_EXPRESSION) // role 1 is strict. timeout should be irrelevant; same as failures public static final ProviderRole PROVIDER_ROLE1 = new ProviderRole( MockRoles.ROLE1, 1, PlacementPolicy.STRICT, 2, - 1) + 1, + ResourceKeys.DEF_YARN_LABEL_EXPRESSION) // role 2: longer delay and anti-affinity public static final ProviderRole PROVIDER_ROLE2 = new ProviderRole( MockRoles.ROLE2, 2, PlacementPolicy.ANTI_AFFINITY_REQUIRED, 2, - 2) + 2, + ResourceKeys.DEF_YARN_LABEL_EXPRESSION) int appIdCount; int attemptIdCount;
