SLIDER-799 track outcome of allocation: whether an assignment was "open", "placed", or "escalated"; this info is included in serialized/JSON views of container state so can be retrieved by client APIs
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7d9a9e94 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7d9a9e94 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7d9a9e94 Branch: refs/heads/feature/SLIDER-799-AM-managed-relax Commit: 7d9a9e942ead8343bc4e2c52419c1b258292ca15 Parents: ad41b24 Author: Steve Loughran <[email protected]> Authored: Tue Mar 24 11:05:39 2015 +0000 Committer: Steve Loughran <[email protected]> Committed: Tue Mar 24 11:05:39 2015 +0000 ---------------------------------------------------------------------- .../org/apache/slider/api/ResourceKeys.java | 8 +- .../org/apache/slider/api/proto/Messages.java | 223 ++++++++++++++++--- .../slider/api/proto/RestTypeMarshalling.java | 8 +- .../slider/api/types/ContainerInformation.java | 4 +- .../apache/slider/providers/ProviderRole.java | 2 +- .../server/appmaster/RoleLaunchService.java | 38 ++-- .../server/appmaster/SliderAppMaster.java | 8 +- .../slider/server/appmaster/state/AppState.java | 14 +- .../state/ContainerAllocationOutcome.java | 29 +++ .../appmaster/state/ContainerAssignment.java | 31 ++- .../server/appmaster/state/NodeEntry.java | 7 +- .../server/appmaster/state/NodeInstance.java | 2 +- .../state/OutstandingRequestTracker.java | 12 +- .../server/appmaster/state/RoleHistory.java | 13 +- .../server/appmaster/state/RoleInstance.java | 21 +- .../resources/LiveContainersRefresher.java | 2 +- .../src/main/proto/SliderClusterMessages.proto | 1 + ...tRoleHistoryOutstandingRequestTracker.groovy | 4 +- .../TestRoleHistoryRequestTracking.groovy | 79 ++++--- 19 files changed, 385 insertions(+), 121 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java index 94ce681..9066a52 100644 --- a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java +++ b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java @@ -142,10 +142,10 @@ public interface ResourceKeys { /** - * Time in seconds to relax placement delay + * Time in seconds to escalate placement delay */ - String PLACEMENT_RELAX_DELAY = - "yarn.placement.relax.seconds"; + String PLACEMENT_ESCALATE_DELAY = + "yarn.placement.escalate.seconds"; /** * Time to have a strict placement policy outstanding before @@ -156,7 +156,7 @@ public interface ResourceKeys { * </ol> * */ - int DEFAULT_PLACEMENT_RELAX_DELAY_SECONDS = 30; + int DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS = 30; /** * Log aggregation include, exclude patterns http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/api/proto/Messages.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/proto/Messages.java b/slider-core/src/main/java/org/apache/slider/api/proto/Messages.java index 53c7d45..44771e5 100644 --- a/slider-core/src/main/java/org/apache/slider/api/proto/Messages.java +++ b/slider-core/src/main/java/org/apache/slider/api/proto/Messages.java @@ -15438,6 +15438,21 @@ public final class Messages { */ com.google.protobuf.ByteString getHostURLBytes(); + + // optional string placement = 12; + /** + * <code>optional string placement = 12;</code> + */ + boolean hasPlacement(); + /** + * <code>optional string placement = 12;</code> + */ + java.lang.String getPlacement(); + /** + * <code>optional string placement = 12;</code> + */ + com.google.protobuf.ByteString + getPlacementBytes(); } /** * Protobuf type {@code org.apache.slider.api.ContainerInformationProto} @@ -15553,6 +15568,11 @@ public final class Messages { hostURL_ = input.readBytes(); break; } + case 98: { + bitField0_ |= 0x00000400; + placement_ = input.readBytes(); + break; + } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { @@ -15921,6 +15941,49 @@ public final class Messages { } } + // optional string placement = 12; + public static final int PLACEMENT_FIELD_NUMBER = 12; + private java.lang.Object placement_; + /** + * <code>optional string placement = 12;</code> + */ + public boolean hasPlacement() { + return ((bitField0_ & 0x00000400) == 0x00000400); + } + /** + * <code>optional string placement = 12;</code> + */ + public java.lang.String getPlacement() { + java.lang.Object ref = placement_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + placement_ = s; + } + return s; + } + } + /** + * <code>optional string placement = 12;</code> + */ + public com.google.protobuf.ByteString + getPlacementBytes() { + java.lang.Object ref = placement_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + placement_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + private void initFields() { containerId_ = ""; component_ = ""; @@ -15933,6 +15996,7 @@ public final class Messages { output_ = com.google.protobuf.LazyStringArrayList.EMPTY; host_ = ""; hostURL_ = ""; + placement_ = ""; } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -15979,6 +16043,9 @@ public final class Messages { if (((bitField0_ & 0x00000200) == 0x00000200)) { output.writeBytes(11, getHostURLBytes()); } + if (((bitField0_ & 0x00000400) == 0x00000400)) { + output.writeBytes(12, getPlacementBytes()); + } getUnknownFields().writeTo(output); } @@ -16037,6 +16104,10 @@ public final class Messages { size += com.google.protobuf.CodedOutputStream .computeBytesSize(11, getHostURLBytes()); } + if (((bitField0_ & 0x00000400) == 0x00000400)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(12, getPlacementBytes()); + } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; return size; @@ -16112,6 +16183,11 @@ public final class Messages { result = result && getHostURL() .equals(other.getHostURL()); } + result = result && (hasPlacement() == other.hasPlacement()); + if (hasPlacement()) { + result = result && getPlacement() + .equals(other.getPlacement()); + } result = result && getUnknownFields().equals(other.getUnknownFields()); return result; @@ -16169,6 +16245,10 @@ public final class Messages { hash = (37 * hash) + HOSTURL_FIELD_NUMBER; hash = (53 * hash) + getHostURL().hashCode(); } + if (hasPlacement()) { + hash = (37 * hash) + PLACEMENT_FIELD_NUMBER; + hash = (53 * hash) + getPlacement().hashCode(); + } hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; @@ -16305,6 +16385,8 @@ public final class Messages { bitField0_ = (bitField0_ & ~0x00000200); hostURL_ = ""; bitField0_ = (bitField0_ & ~0x00000400); + placement_ = ""; + bitField0_ = (bitField0_ & ~0x00000800); return this; } @@ -16379,6 +16461,10 @@ public final class Messages { to_bitField0_ |= 0x00000200; } result.hostURL_ = hostURL_; + if (((from_bitField0_ & 0x00000800) == 0x00000800)) { + to_bitField0_ |= 0x00000400; + } + result.placement_ = placement_; result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -16445,6 +16531,11 @@ public final class Messages { hostURL_ = other.hostURL_; onChanged(); } + if (other.hasPlacement()) { + bitField0_ |= 0x00000800; + placement_ = other.placement_; + onChanged(); + } this.mergeUnknownFields(other.getUnknownFields()); return this; } @@ -17100,6 +17191,80 @@ public final class Messages { return this; } + // optional string placement = 12; + private java.lang.Object placement_ = ""; + /** + * <code>optional string placement = 12;</code> + */ + public boolean hasPlacement() { + return ((bitField0_ & 0x00000800) == 0x00000800); + } + /** + * <code>optional string placement = 12;</code> + */ + public java.lang.String getPlacement() { + java.lang.Object ref = placement_; + if (!(ref instanceof java.lang.String)) { + java.lang.String s = ((com.google.protobuf.ByteString) ref) + .toStringUtf8(); + placement_ = s; + return s; + } else { + return (java.lang.String) ref; + } + } + /** + * <code>optional string placement = 12;</code> + */ + public com.google.protobuf.ByteString + getPlacementBytes() { + java.lang.Object ref = placement_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + placement_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + /** + * <code>optional string placement = 12;</code> + */ + public Builder setPlacement( + java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000800; + placement_ = value; + onChanged(); + return this; + } + /** + * <code>optional string placement = 12;</code> + */ + public Builder clearPlacement() { + bitField0_ = (bitField0_ & ~0x00000800); + placement_ = getDefaultInstance().getPlacement(); + onChanged(); + return this; + } + /** + * <code>optional string placement = 12;</code> + */ + public Builder setPlacementBytes( + com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000800; + placement_ = value; + onChanged(); + return this; + } + // @@protoc_insertion_point(builder_scope:org.apache.slider.api.ContainerInformationProto) } @@ -26869,38 +27034,38 @@ public final class Messages { "ed\030\t \001(\005\022\021\n\tcompleted\030\n \001(\005\022\026\n\016totalRequ" + "ested\030\013 \001(\005\022\026\n\016failureMessage\030\014 \001(\t\022\027\n\017p" + "lacementPolicy\030\r \001(\005\022\022\n\ncontainers\030\016 \003(\t" + - "\"\341\001\n\031ContainerInformationProto\022\023\n\013contai" + + "\"\364\001\n\031ContainerInformationProto\022\023\n\013contai" + "nerId\030\001 \001(\t\022\021\n\tcomponent\030\002 \001(\t\022\020\n\010releas" + "ed\030\003 \001(\010\022\r\n\005state\030\004 \001(\005\022\020\n\010exitCode\030\005 \001(" + "\005\022\023\n\013diagnostics\030\006 \001(\t\022\022\n\ncreateTime\030\007 \001" + "(\003\022\021\n\tstartTime\030\010 \001(\003\022\016\n\006output\030\t \003(\t\022\014\n" + - "\004host\030\n \001(\t\022\017\n\007hostURL\030\013 \001(\t\"N\n\024PingInfo" + - "rmationProto\022\014\n\004text\030\001 \001(\t\022\014\n\004verb\030\002 \001(\t", - "\022\014\n\004body\030\003 \001(\t\022\014\n\004time\030\004 \001(\003\"\026\n\024GetModel" + - "RequestProto\"\035\n\033GetModelDesiredRequestPr" + - "oto\"$\n\"GetModelDesiredAppconfRequestProt" + - "o\"&\n$GetModelDesiredResourcesRequestProt" + - "o\"%\n#GetModelResolvedAppconfRequestProto" + - "\"\'\n%GetModelResolvedResourcesRequestProt" + - "o\"#\n!GetModelLiveResourcesRequestProto\"\037" + - "\n\035GetLiveContainersRequestProto\"u\n\036GetLi" + - "veContainersResponseProto\022\r\n\005names\030\001 \003(\t" + - "\022D\n\ncontainers\030\002 \003(\01320.org.apache.slider", - ".api.ContainerInformationProto\"3\n\034GetLiv" + - "eContainerRequestProto\022\023\n\013containerId\030\001 " + - "\002(\t\"\037\n\035GetLiveComponentsRequestProto\"u\n\036" + - "GetLiveComponentsResponseProto\022\r\n\005names\030" + - "\001 \003(\t\022D\n\ncomponents\030\002 \003(\01320.org.apache.s" + - "lider.api.ComponentInformationProto\",\n\034G" + - "etLiveComponentRequestProto\022\014\n\004name\030\001 \002(" + - "\t\"$\n\"GetApplicationLivenessRequestProto\"" + - "\023\n\021EmptyPayloadProto\" \n\020WrappedJsonProto" + - "\022\014\n\004json\030\001 \002(\t\"h\n\037GetCertificateStoreReq", - "uestProto\022\020\n\010hostname\030\001 \001(\t\022\023\n\013requester" + - "Id\030\002 \002(\t\022\020\n\010password\030\003 \002(\t\022\014\n\004type\030\004 \002(\t" + - "\"1\n GetCertificateStoreResponseProto\022\r\n\005" + - "store\030\001 \002(\014B-\n\033org.apache.slider.api.pro" + - "toB\010Messages\210\001\001\240\001\001" + "\004host\030\n \001(\t\022\017\n\007hostURL\030\013 \001(\t\022\021\n\tplacemen" + + "t\030\014 \001(\t\"N\n\024PingInformationProto\022\014\n\004text\030", + "\001 \001(\t\022\014\n\004verb\030\002 \001(\t\022\014\n\004body\030\003 \001(\t\022\014\n\004tim" + + "e\030\004 \001(\003\"\026\n\024GetModelRequestProto\"\035\n\033GetMo" + + "delDesiredRequestProto\"$\n\"GetModelDesire" + + "dAppconfRequestProto\"&\n$GetModelDesiredR" + + "esourcesRequestProto\"%\n#GetModelResolved" + + "AppconfRequestProto\"\'\n%GetModelResolvedR" + + "esourcesRequestProto\"#\n!GetModelLiveReso" + + "urcesRequestProto\"\037\n\035GetLiveContainersRe" + + "questProto\"u\n\036GetLiveContainersResponseP" + + "roto\022\r\n\005names\030\001 \003(\t\022D\n\ncontainers\030\002 \003(\0132", + "0.org.apache.slider.api.ContainerInforma" + + "tionProto\"3\n\034GetLiveContainerRequestProt" + + "o\022\023\n\013containerId\030\001 \002(\t\"\037\n\035GetLiveCompone" + + "ntsRequestProto\"u\n\036GetLiveComponentsResp" + + "onseProto\022\r\n\005names\030\001 \003(\t\022D\n\ncomponents\030\002" + + " \003(\01320.org.apache.slider.api.ComponentIn" + + "formationProto\",\n\034GetLiveComponentReques" + + "tProto\022\014\n\004name\030\001 \002(\t\"$\n\"GetApplicationLi" + + "venessRequestProto\"\023\n\021EmptyPayloadProto\"" + + " \n\020WrappedJsonProto\022\014\n\004json\030\001 \002(\t\"h\n\037Get", + "CertificateStoreRequestProto\022\020\n\010hostname" + + "\030\001 \001(\t\022\023\n\013requesterId\030\002 \002(\t\022\020\n\010password\030" + + "\003 \002(\t\022\014\n\004type\030\004 \002(\t\"1\n GetCertificateSto" + + "reResponseProto\022\r\n\005store\030\001 \002(\014B-\n\033org.ap" + + "ache.slider.api.protoB\010Messages\210\001\001\240\001\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -27050,7 +27215,7 @@ public final class Messages { internal_static_org_apache_slider_api_ContainerInformationProto_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_org_apache_slider_api_ContainerInformationProto_descriptor, - new java.lang.String[] { "ContainerId", "Component", "Released", "State", "ExitCode", "Diagnostics", "CreateTime", "StartTime", "Output", "Host", "HostURL", }); + new java.lang.String[] { "ContainerId", "Component", "Released", "State", "ExitCode", "Diagnostics", "CreateTime", "StartTime", "Output", "Host", "HostURL", "Placement", }); internal_static_org_apache_slider_api_PingInformationProto_descriptor = getDescriptor().getMessageTypes().get(24); internal_static_org_apache_slider_api_PingInformationProto_fieldAccessorTable = new http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java b/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java index 84a950d..4264582 100644 --- a/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java +++ b/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java @@ -157,7 +157,10 @@ public class RestTypeMarshalling { info.startTime = wire.getStartTime(); info.output = wire.getOutputList().toArray( new String[wire.getOutputCount()] - ); + ); + if (wire.hasPlacement()) { + info.placement = wire.getPlacement(); + } return info; } @@ -199,6 +202,9 @@ public class RestTypeMarshalling { if (info.released != null) { builder.setReleased(info.released); } + if (info.placement != null) { + builder.setPlacement(info.placement); + } builder.setStartTime(info.startTime); builder.setState(info.state); return builder.build(); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/api/types/ContainerInformation.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/types/ContainerInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/ContainerInformation.java index ccadd9c..8c26bfe 100644 --- a/slider-core/src/main/java/org/apache/slider/api/types/ContainerInformation.java +++ b/slider-core/src/main/java/org/apache/slider/api/types/ContainerInformation.java @@ -22,7 +22,6 @@ import org.apache.hadoop.registry.client.binding.JsonSerDeser; import org.codehaus.jackson.annotate.JsonIgnoreProperties; import org.codehaus.jackson.map.annotate.JsonSerialize; - /** * Serializable version of component instance data */ @@ -41,6 +40,7 @@ public class ContainerInformation { public String host; public String hostURL; + public String placement; /** * What is the tail output from the executed process (or [] if not started * or the log cannot be picked up @@ -50,7 +50,7 @@ public class ContainerInformation { @Override public String toString() { JsonSerDeser<ContainerInformation> serDeser = - new JsonSerDeser<ContainerInformation>( + new JsonSerDeser<>( ContainerInformation.class); return serDeser.toString(this); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java index 7caae48..294fe89 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java +++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java @@ -38,7 +38,7 @@ public final class ProviderRole { id, PlacementPolicy.DEFAULT, ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD, - ResourceKeys.DEFAULT_PLACEMENT_RELAX_DELAY_SECONDS); + ResourceKeys.DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS); } /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java index 4333a09..df65ff4 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java @@ -32,6 +32,7 @@ import org.apache.slider.providers.ProviderService; import org.apache.slider.providers.agent.AgentKeys; import org.apache.slider.server.appmaster.actions.ActionStartContainer; import org.apache.slider.server.appmaster.actions.QueueAccess; +import org.apache.slider.server.appmaster.state.ContainerAssignment; import org.apache.slider.server.appmaster.state.RoleInstance; import org.apache.slider.server.appmaster.state.RoleStatus; import org.apache.slider.server.services.workflow.WorkflowExecutorService; @@ -121,16 +122,15 @@ public class RoleLaunchService * @param role role * @param clusterSpec cluster spec to use for template */ - public void launchRole(Container container, - RoleStatus role, + public void launchRole(ContainerAssignment assignment, AggregateConf clusterSpec) { + RoleStatus role = assignment.role; String roleName = role.getName(); // prelaunch safety check Preconditions.checkArgument(provider.isSupportedRole(roleName)); RoleLaunchService.RoleLauncher launcher = - new RoleLaunchService.RoleLauncher(container, - role.getProviderRole(), - clusterSpec, + new RoleLaunchService.RoleLauncher(assignment, + clusterSpec, clusterSpec.getResourceOperations().getOrAddComponent(roleName), clusterSpec.getAppConfOperations().getOrAddComponent(roleName)); execute(launcher); @@ -141,27 +141,30 @@ public class RoleLaunchService */ private class RoleLauncher implements Runnable { + private final ContainerAssignment assignment; // Allocated container public final Container container; - public final String containerRole; + public final String containerRole; private final MapOperations resourceComponent; private final MapOperations appComponent; private final AggregateConf instanceDefinition; public final ProviderRole role; private Exception raisedException; - public RoleLauncher(Container container, - ProviderRole role, - AggregateConf instanceDefinition, - MapOperations resourceComponent, - MapOperations appComponent) { - assert container != null; - assert role != null; + public RoleLauncher(ContainerAssignment assignment, + AggregateConf instanceDefinition, + MapOperations resourceComponent, + MapOperations appComponent) { + this.assignment = assignment; + this.container = assignment.container; + RoleStatus roleStatus = assignment.role; + assert resourceComponent != null; assert appComponent != null; - this.container = container; - this.containerRole = role.name; - this.role = role; + ProviderRole providerRole = roleStatus.getProviderRole(); + assert providerRole != null; + this.containerRole = providerRole.name; + this.role = providerRole; this.resourceComponent = resourceComponent; this.appComponent = appComponent; this.instanceDefinition = instanceDefinition; @@ -228,8 +231,7 @@ public class RoleLaunchService } log.info("Container launch delay for {} set to {} seconds", role.name, delay); - actionQueue.schedule(new ActionStartContainer("starting " - + containerRole, + actionQueue.schedule(new ActionStartContainer("starting " + containerRole, container, containerLauncher.completeContainerLaunch(), instance, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index fd420fd..ab6b55c 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -1567,17 +1567,15 @@ public class SliderAppMaster extends AbstractSliderLaunchedService @Override //AMRMClientAsync public void onContainersAllocated(List<Container> allocatedContainers) { LOG_YARN.info("onContainersAllocated({})", allocatedContainers.size()); - List<ContainerAssignment> assignments = new ArrayList<ContainerAssignment>(); - List<AbstractRMOperation> operations = new ArrayList<AbstractRMOperation>(); + List<ContainerAssignment> assignments = new ArrayList<>(); + List<AbstractRMOperation> operations = new ArrayList<>(); //app state makes all the decisions appState.onContainersAllocated(allocatedContainers, assignments, operations); //for each assignment: instantiate that role for (ContainerAssignment assignment : assignments) { - RoleStatus role = assignment.role; - Container container = assignment.container; - launchService.launchRole(container, role, getInstanceDefinition()); + launchService.launchRole(assignment, getInstanceDefinition()); } //for all the operations, exec them http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index 0f07ee9..20e2fc0 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -655,8 +655,8 @@ public class AppState { ResourceKeys.COMPONENT_PLACEMENT_POLICY, placementOpt, 0, 0, -1); int placementTimeout = - component.getOptionInt(ResourceKeys.PLACEMENT_RELAX_DELAY, - ResourceKeys.DEFAULT_PLACEMENT_RELAX_DELAY_SECONDS); + component.getOptionInt(ResourceKeys.PLACEMENT_ESCALATE_DELAY, + ResourceKeys.DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS); ProviderRole newRole = new ProviderRole(name, priority, placement, @@ -1135,7 +1135,7 @@ public class AppState { * @return the map of Role name to list of Cluster Nodes */ public synchronized Map<String, Map<String, ClusterNode>> createRoleToClusterNodeMap() { - Map<String, Map<String, ClusterNode>> map = new HashMap<String, Map<String, ClusterNode>>(); + Map<String, Map<String, ClusterNode>> map = new HashMap<>(); for (RoleInstance node : getLiveNodes().values()) { Map<String, ClusterNode> containers = map.get(node.role); @@ -2090,7 +2090,7 @@ public class AppState { /** * Event handler for allocated containers: builds up the lists * of assignment actions (what to run where), and possibly - * a list of release operations + * a list of operations to perform * @param allocatedContainers the containers allocated * @param assignments the assignments of roles to containers * @param releaseOperations any release operations @@ -2121,7 +2121,9 @@ public class AppState { //look for (race condition) where we get more back than we asked desired = role.getDesired(); - roleHistory.onContainerAllocated( container, desired, allocated ); + ContainerAllocationOutcome outcome = roleHistory.onContainerAllocated(container, + desired, + allocated); if (allocated > desired) { log.info("Discarding surplus container {} on {}", cid, @@ -2146,7 +2148,7 @@ public class AppState { container.getNodeId().getPort() ); - assignments.add(new ContainerAssignment(container, role)); + assignments.add(new ContainerAssignment(container, role, outcome)); //add to the history roleHistory.onContainerAssigned(container); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationOutcome.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationOutcome.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationOutcome.java new file mode 100644 index 0000000..6639300 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationOutcome.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.state; + +/** + * Outcome of the assignment + */ +public enum ContainerAllocationOutcome { + Unallocated, + Open, + Placed, + Escalated +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAssignment.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAssignment.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAssignment.java index 3be3777..3e8a3c3 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAssignment.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAssignment.java @@ -20,14 +20,41 @@ package org.apache.slider.server.appmaster.state; import org.apache.hadoop.yarn.api.records.Container; +/** + * Static assignment structure + */ public class ContainerAssignment { - + + /** + * Container that has been allocated + */ public final Container container; + + /** + * Role to assign to it + */ public final RoleStatus role; + /** + * Placement outcome: was this from history or not + */ + public final ContainerAllocationOutcome placement; + public ContainerAssignment(Container container, - RoleStatus role) { + RoleStatus role, + ContainerAllocationOutcome placement) { this.container = container; this.role = role; + this.placement = placement; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("ContainerAssignment{"); + sb.append("container=").append(container); + sb.append(", role=").append(role); + sb.append(", placement=").append(placement); + sb.append('}'); + return sb.toString(); } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java index c2e203a..0aa2d42 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java @@ -21,7 +21,7 @@ package org.apache.slider.server.appmaster.state; /** * Information about the state of a role on a specific node instance. * No fields are synchronized; sync on the instance to work with it - * + <p> The two fields `releasing` and `requested` are used to track the ongoing state of YARN requests; they do not need to be persisted across stop/start cycles. They may be relevant across AM restart, but without other data @@ -29,10 +29,11 @@ package org.apache.slider.server.appmaster.state; it was restarted. The strategy will be to ignore unexpected allocation responses (which may come from pre-restart) requests, while treating unexpected container release responses as failures. - + <p> The `active` counter is only decremented after a container release response has been received. - + <p> + Accesses are synchronized. */ public class NodeEntry { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java index 68c8a15..ed039f9 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java @@ -26,7 +26,7 @@ import java.util.ListIterator; /** * A node instance -stores information about a node in the cluster. - * + * <p> * Operations on the array/set of roles are synchronized. */ public class NodeInstance { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java index 959cb1f..e226a22 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java @@ -103,18 +103,22 @@ public class OutstandingRequestTracker { * from the {@link #placedRequests} structure. * @param role role index * @param hostname hostname - * @return true if an entry was found and removed + * @return the allocation outcome */ - public synchronized boolean onContainerAllocated(int role, String hostname) { + public synchronized ContainerAllocationOutcome onContainerAllocated(int role, String hostname) { OutstandingRequest request = placedRequests.remove(new OutstandingRequest(role, hostname)); if (request == null) { - return false; + // not in the list; this is an open placement + return ContainerAllocationOutcome.Open; } else { //satisfied request request.completed(); + // derive outcome from status of tracked request + return request.isEscalated() + ? ContainerAllocationOutcome.Escalated + : ContainerAllocationOutcome.Placed; } - return true; } /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java index a1b54c7..99108fe 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java @@ -636,20 +636,21 @@ public class RoleHistory { * @param container container * @param desiredCount desired #of instances * @param actualCount current count of instances - * @return true if an entry was found and dropped + * @return The allocation outcome */ - public synchronized boolean onContainerAllocated(Container container, int desiredCount, int actualCount) { + public synchronized ContainerAllocationOutcome onContainerAllocated(Container container, + int desiredCount, + int actualCount) { int role = ContainerPriority.extractRole(container); String hostname = RoleHistoryUtils.hostnameOf(container); LinkedList<NodeInstance> nodeInstances = getOrCreateNodesForRoleId(role); - boolean requestFound = - outstandingRequests.onContainerAllocated(role, hostname); + ContainerAllocationOutcome outcome = outstandingRequests.onContainerAllocated(role, hostname); if (desiredCount <= actualCount) { // all outstanding requests have been satisfied // tag nodes as available List<NodeInstance> - hosts = outstandingRequests.resetOutstandingRequests(role); + hosts = outstandingRequests.resetOutstandingRequests(role); if (!hosts.isEmpty()) { //add the list log.info("Adding {} hosts for role {}", hosts.size(), role); @@ -657,7 +658,7 @@ public class RoleHistory { sortAvailableNodeList(role); } } - return requestFound; + return outcome; } /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java index 4f9b222..726ee08 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java @@ -19,12 +19,12 @@ package org.apache.slider.server.appmaster.state; import com.google.common.base.Preconditions; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.registry.client.binding.RegistryTypeUtils; import org.apache.hadoop.registry.client.types.Endpoint; import org.apache.hadoop.registry.client.types.ProtocolTypes; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.slider.api.ClusterNode; import org.apache.slider.api.proto.Messages; import org.apache.slider.api.types.ContainerInformation; @@ -95,14 +95,23 @@ public final class RoleInstance implements Cloneable { public String host; public String hostURL; + public ContainerAllocationOutcome placement; /** * A list of registered endpoints. */ private List<Endpoint> endpoints = - new ArrayList<Endpoint>(2); + new ArrayList<>(2); + public RoleInstance(ContainerAssignment assignment) { + this(assignment.container); + placement = assignment.placement; + } + /** + * Create an instance to track an allocated container + * @param container a container which must be non null, and have a non-null Id field. + */ public RoleInstance(Container container) { Preconditions.checkNotNull(container, "Null container"); Preconditions.checkState(container.getId() != null, @@ -140,6 +149,7 @@ public final class RoleInstance implements Cloneable { sb.append(", host=").append(host); sb.append(", hostURL=").append(hostURL); sb.append(", state=").append(state); + sb.append(", placement=").append(placement); sb.append(", exitCode=").append(exitCode); sb.append(", command='").append(command).append('\''); sb.append(", diagnostics='").append(diagnostics).append('\''); @@ -288,6 +298,9 @@ public final class RoleInstance implements Cloneable { info.host = host; info.hostURL = hostURL; info.released = released ? Boolean.TRUE : null; + if (placement != null) { + info.placement = placement.toString(); + } if (output != null) { info.output = output; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java index 276e8cc..68bd8a2 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java @@ -42,7 +42,7 @@ public class LiveContainersRefresher implements ResourceRefresher<Map<String, Co Exception { List<RoleInstance> containerList = state.cloneOwnedContainerList(); - Map<String, ContainerInformation> map = new HashMap<String, ContainerInformation>(); + Map<String, ContainerInformation> map = new HashMap<>(); for (RoleInstance instance : containerList) { ContainerInformation serialized = instance.serialize(); map.put(serialized.containerId, serialized); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/main/proto/SliderClusterMessages.proto ---------------------------------------------------------------------- diff --git a/slider-core/src/main/proto/SliderClusterMessages.proto b/slider-core/src/main/proto/SliderClusterMessages.proto index 9f8f20f..730a49b 100644 --- a/slider-core/src/main/proto/SliderClusterMessages.proto +++ b/slider-core/src/main/proto/SliderClusterMessages.proto @@ -244,6 +244,7 @@ message ContainerInformationProto { repeated string output = 9; optional string host = 10; optional string hostURL = 11; + optional string placement = 12; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy index 97d970d..c30537a 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy @@ -27,7 +27,7 @@ import org.apache.slider.server.appmaster.model.mock.MockResource import org.apache.slider.server.appmaster.operations.AbstractRMOperation import org.apache.slider.server.appmaster.operations.CancelSingleRequest import org.apache.slider.server.appmaster.operations.ContainerRequestOperation -import org.apache.slider.server.appmaster.state.NodeEntry +import org.apache.slider.server.appmaster.state.ContainerAllocationOutcome import org.apache.slider.server.appmaster.state.NodeInstance import org.apache.slider.server.appmaster.state.OutstandingRequest import org.apache.slider.server.appmaster.state.OutstandingRequestTracker @@ -59,7 +59,7 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest { tracker.newRequest(host1, 0) tracker.newRequest(host2, 0) tracker.newRequest(host1, 1) - assert tracker.onContainerAllocated(1, "host1") + assert tracker.onContainerAllocated(1, "host1") == ContainerAllocationOutcome.Placed assert !tracker.lookup(1, "host1") assert tracker.lookup(0, "host1") } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d9a9e94/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy index e84dfce..bab6233 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy @@ -24,6 +24,7 @@ import org.apache.slider.providers.ProviderRole import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest import org.apache.slider.server.appmaster.model.mock.MockContainer import org.apache.slider.server.appmaster.model.mock.MockFactory +import org.apache.slider.server.appmaster.state.ContainerAllocationOutcome import org.apache.slider.server.appmaster.state.NodeInstance import org.apache.slider.server.appmaster.state.OutstandingRequest import org.apache.slider.server.appmaster.state.RoleHistory @@ -38,7 +39,7 @@ import org.junit.Test class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { String roleName = "test" - + NodeInstance age1Active4 = nodeInstance(1, 4, 0, 0) NodeInstance age2Active2 = nodeInstance(2, 2, 0, 1) NodeInstance age3Active0 = nodeInstance(3, 0, 0, 0) @@ -52,12 +53,12 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { ProviderRole provRole = new ProviderRole(roleName, 0) RoleStatus roleStatus = new RoleStatus(provRole) - + @Override String getTestName() { return "TestRoleHistoryAvailableList" } - + @Before public void setupNodeMap() { roleHistory.insert(nodes) @@ -69,16 +70,16 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { List<NodeInstance> available0 = roleHistory.cloneAvailableList(0) assertListEquals([age3Active0, age2Active0], available0) } - + @Test public void testRequestedNodeOffList() throws Throwable { List<NodeInstance> available0 = roleHistory.cloneAvailableList(0) NodeInstance ni = roleHistory.findNodeForNewInstance(roleStatus) assert age3Active0 == ni AMRMClient.ContainerRequest req = roleHistory.requestInstanceOnNode(ni, - roleStatus, - resource, - "") + roleStatus, + resource, + "") List<NodeInstance> a2 = roleHistory.cloneAvailableList(0) assertListEquals([age2Active0], a2) } @@ -118,7 +119,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { List<NodeInstance> a2 = roleHistory.cloneAvailableList(0) assertListEquals([age2Active0], a2) } - + @Test public void testRequestedNodeIntoReqList() throws Throwable { AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource) @@ -126,7 +127,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { assert requests.size() == 1 assert age3Active0.hostname == requests[0].hostname } - + @Test public void testCompletedRequestDropsNode() throws Throwable { AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource) @@ -136,10 +137,26 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { assert age3Active0.hostname == hostname assert hostname == req.nodes[0] MockContainer container = factory.newContainer(req, hostname) - assert roleHistory.onContainerAllocated(container , 2, 1) + assertOnContainerAllocated(container, 2, 1) + assertNoOutstandingRequests() + } + + public void assertOnContainerAllocated(MockContainer c1, int p1, int p2) { + assert ContainerAllocationOutcome.Open != roleHistory.onContainerAllocated(c1, p1, p2) + } + + public void assertOnContainerAllocationOpen(MockContainer c1, int p1, int p2) { + assert ContainerAllocationOutcome.Open == roleHistory.onContainerAllocated(c1, p1, p2) + } + + def assertNoOutstandingRequests() { assert roleHistory.listOutstandingPlacedRequests().empty } - + + public void assertOutstandingPlacedRequests(int i) { + assert roleHistory.listOutstandingPlacedRequests().size() == i + } + @Test public void testTwoRequests() throws Throwable { AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource) @@ -147,14 +164,13 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests() assert requests.size() == 2 MockContainer container = factory.newContainer(req, req.nodes[0]) - assert roleHistory.onContainerAllocated(container , 2, 1) - assert roleHistory.listOutstandingPlacedRequests().size() == 1 + assertOnContainerAllocated(container, 2, 1) + assertOutstandingPlacedRequests(1) container = factory.newContainer(req2, req2.nodes[0]) - assert roleHistory.onContainerAllocated(container, 2, 2) - assert roleHistory.listOutstandingPlacedRequests().empty + assertOnContainerAllocated(container, 2, 2) + assertNoOutstandingRequests() } - @Test public void testThreeRequestsOneUnsatisified() throws Throwable { AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource) @@ -163,19 +179,19 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests() assert requests.size() == 2 MockContainer container = factory.newContainer(req, req.nodes[0]) - assert roleHistory.onContainerAllocated(container , 2, 1) - assert roleHistory.listOutstandingPlacedRequests().size() == 1 - + assertOnContainerAllocated(container, 2, 1) + assertOutstandingPlacedRequests(1) + container = factory.newContainer(req3, "three") - assert !roleHistory.onContainerAllocated(container, 3, 2) - assert roleHistory.listOutstandingPlacedRequests().size() == 1 - + assertOnContainerAllocationOpen(container, 3, 2) + assertOutstandingPlacedRequests(1) + // the final allocation will trigger a cleanup container = factory.newContainer(req2, "four") // no node dropped - assert !roleHistory.onContainerAllocated(container, 3, 3) + assertOnContainerAllocationOpen(container, 3, 3) // yet the list is now empty - assert roleHistory.listOutstandingPlacedRequests().empty + assertNoOutstandingRequests() // and the remainder goes onto the available list List<NodeInstance> a2 = roleHistory.cloneAvailableList(0) @@ -183,23 +199,22 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { } - @Test public void testThreeRequests() throws Throwable { AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource) AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource) AMRMClient.ContainerRequest req3 = roleHistory.requestNode(roleStatus, resource) - assert roleHistory.listOutstandingPlacedRequests().size() == 2 + assertOutstandingPlacedRequests(2) assert req3.nodes == null MockContainer container = factory.newContainer(req, req.nodes[0]) - assert roleHistory.onContainerAllocated(container , 3, 1) - assert roleHistory.listOutstandingPlacedRequests().size() == 1 + assertOnContainerAllocated(container, 3, 1) + assertOutstandingPlacedRequests(1) container = factory.newContainer(req2, req2.nodes[0]) - assert roleHistory.onContainerAllocated(container, 3, 2) - assert roleHistory.listOutstandingPlacedRequests().empty + assertOnContainerAllocated(container, 3, 2) + assertNoOutstandingRequests() container = factory.newContainer(req3, "three") - assert !roleHistory.onContainerAllocated(container, 3, 3) - assert roleHistory.listOutstandingPlacedRequests().empty + assertOnContainerAllocationOpen(container, 3, 3) + assertNoOutstandingRequests() } }
