Repository: hadoop Updated Branches: refs/heads/yarn-native-services 633eb661f -> d23a97d4e
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index 43c7ead..9f7b4a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -20,7 +20,13 @@ package org.apache.slider.server.appmaster.state; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.metrics2.lib.MutableGaugeInt; import org.apache.hadoop.yarn.api.records.Container; @@ -42,6 +48,7 @@ import org.apache.slider.api.proto.Messages.ComponentCountProto; import org.apache.slider.api.resource.Application; import org.apache.slider.api.resource.ApplicationState; import org.apache.slider.api.resource.Component; +import org.apache.slider.api.resource.ConfigFile; import org.apache.slider.api.types.ApplicationLivenessInformation; import org.apache.slider.api.types.ComponentInformation; import org.apache.slider.api.types.RoleStatistics; @@ -79,6 +86,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import static org.apache.slider.api.ResourceKeys.*; @@ -99,7 +107,6 @@ public class AppState { private final AbstractClusterServices recordFactory; private final MetricsAndMonitoring metricsAndMonitoring; - /** * Flag set to indicate the application is live -this only happens * after the buildInstance operation @@ -108,9 +115,11 @@ public class AppState { private Application app; + // priority_id -> RoleStatus private final Map<Integer, RoleStatus> roleStatusMap = new ConcurrentSkipListMap<>(); + // component_name -> ProviderRole private final Map<String, ProviderRole> roles = new ConcurrentHashMap<>(); @@ -202,6 +211,10 @@ public class AppState { private SliderMetrics appMetrics; private ServiceTimelinePublisher serviceTimelinePublisher; + + // A cache for loading config files from remote such as hdfs + public LoadingCache<ConfigFile, Object> configFileCache = null; + /** * Create an instance * @param recordFactory factory for YARN records @@ -304,8 +317,6 @@ public class AppState { public synchronized void buildInstance(AppStateBindingInfo binding) throws BadClusterStateException, BadConfigException, IOException { binding.validate(); - - log.debug("Building application state"); containerReleaseSelector = binding.releaseSelector; // set the cluster specification (once its dependency the client properties @@ -313,10 +324,8 @@ public class AppState { this.app = binding.application; appMetrics = SliderMetrics.register(app.getName(), "Metrics for service"); - appMetrics - .tag("type", "Metrics type [component or service]", "service"); - appMetrics - .tag("appId", "Application id for service", app.getId()); + appMetrics.tag("type", "Metrics type [component or service]", "service"); + appMetrics.tag("appId", "Application id for service", app.getId()); org.apache.slider.api.resource.Configuration conf = app.getConfiguration(); startTimeThreshold = @@ -327,12 +336,7 @@ public class AppState { nodeFailureThreshold = conf.getPropertyInt(NODE_FAILURE_THRESHOLD, DEFAULT_NODE_FAILURE_THRESHOLD); - //build the initial role list - List<ProviderRole> roleList = new ArrayList<>(binding.roles); - for (ProviderRole providerRole : roleList) { - buildRole(providerRole); - } - + //build the initial component list int priority = 1; for (Component component : app.getComponents()) { priority = getNewPriority(priority); @@ -340,25 +344,18 @@ public class AppState { if (roles.containsKey(name)) { continue; } - if (component.getUniqueComponentSupport()) { - log.info("Skipping group " + name + ", as it's unique component"); - continue; - } log.info("Adding component: " + name); - ProviderRole dynamicRole = - createComponent(name, name, component, priority); - buildRole(dynamicRole); - roleList.add(dynamicRole); + createComponent(name, name, component, priority++); } + //then pick up the requirements - buildRoleRequirementsFromResources(); +// buildRoleRequirementsFromResources(); // set up the role history roleHistory = new RoleHistory(roleStatusMap.values(), recordFactory); roleHistory.onStart(binding.fs, binding.historyPath); // trigger first node update roleHistory.onNodesUpdated(binding.nodeReports); - //rebuild any live containers rebuildModelFromRestart(binding.liveContainers); @@ -367,9 +364,39 @@ public class AppState { //mark as live applicationLive = true; app.setState(STARTED); + createConfigFileCache(binding.fs); + } + + private void createConfigFileCache(final FileSystem fileSystem) { + this.configFileCache = + CacheBuilder.newBuilder().expireAfterAccess(10, TimeUnit.MINUTES) + .build(new CacheLoader<ConfigFile, Object>() { + @Override public Object load(ConfigFile key) throws Exception { + switch (key.getType()) { + case HADOOP_XML: + try (FSDataInputStream input = fileSystem + .open(new Path(key.getSrcFile()))) { + org.apache.hadoop.conf.Configuration confRead = + new org.apache.hadoop.conf.Configuration(false); + confRead.addResource(input); + Map<String, String> map = new HashMap<>(confRead.size()); + for (Map.Entry<String, String> entry : confRead) { + map.put(entry.getKey(), entry.getValue()); + } + return map; + } + case TEMPLATE: + try (FSDataInputStream fileInput = fileSystem + .open(new Path(key.getSrcFile()))) { + return IOUtils.toString(fileInput); + } + default: + return null; + } + } + }); } - //TODO WHY do we need to create the component for AM ? public ProviderRole createComponent(String name, String group, Component component, int priority) throws BadConfigException { org.apache.slider.api.resource.Configuration conf = @@ -384,26 +411,28 @@ public class AppState { DEF_YARN_LABEL_EXPRESSION); ProviderRole newRole = new ProviderRole(name, group, priority, (int)placementPolicy, threshold, - placementTimeout, label, component); - + placementTimeout, label, component, this); + buildRole(newRole, component); log.info("Created a new role " + newRole); return newRole; } @VisibleForTesting - public synchronized List<ProviderRole> updateComponents(Map<String, Long> + public synchronized void updateComponents(Map<String, Long> componentCounts) throws BadConfigException { for (Component component : app.getComponents()) { if (componentCounts.containsKey(component.getName())) { - component.setNumberOfContainers(componentCounts.get(component - .getName())); + long count = componentCounts.get(component.getName()); + component.setNumberOfContainers(count); + ProviderRole role = roles.get(component.getName()); + if (role != null && roleStatusMap.get(role.id) != null) { + setDesiredContainers(roleStatusMap.get(role.id), (int) count); + } } } - //TODO update cluster description - return buildRoleRequirementsFromResources(); } - public synchronized List<ProviderRole> updateComponents( + public synchronized void updateComponents( Messages.FlexComponentsRequestProto requestProto) throws BadConfigException { Map<String, Long> componentCounts = new HashMap<>(); @@ -412,116 +441,119 @@ public class AppState { componentCounts.put(componentCount.getName(), componentCount .getNumberOfContainers()); } - return updateComponents(componentCounts); + updateComponents(componentCounts); } /** * build the role requirements from the cluster specification * @return a list of any dynamically added provider roles */ - private List<ProviderRole> buildRoleRequirementsFromResources() - throws BadConfigException { - - List<ProviderRole> newRoles = new ArrayList<>(0); - - // now update every role's desired count. - // if there are no instance values, that role count goes to zero - // Add all the existing roles - // component name -> number of containers - Map<String, Integer> groupCounts = new HashMap<>(); - - for (RoleStatus roleStatus : getRoleStatusMap().values()) { - if (roleStatus.isExcludeFromFlexing()) { - // skip inflexible roles, e.g AM itself - continue; - } - long currentDesired = roleStatus.getDesired(); - String role = roleStatus.getName(); - String roleGroup = roleStatus.getGroup(); - Component component = roleStatus.getProviderRole().component; - int desiredInstanceCount = component.getNumberOfContainers().intValue(); - - int newDesired = desiredInstanceCount; - if (component.getUniqueComponentSupport()) { - Integer groupCount = 0; - if (groupCounts.containsKey(roleGroup)) { - groupCount = groupCounts.get(roleGroup); - } - - newDesired = desiredInstanceCount - groupCount; - - if (newDesired > 0) { - newDesired = 1; - groupCounts.put(roleGroup, groupCount + newDesired); - } else { - newDesired = 0; - } - } - - if (newDesired == 0) { - log.info("Role {} has 0 instances specified", role); - } - if (currentDesired != newDesired) { - log.info("Role {} flexed from {} to {}", role, currentDesired, - newDesired); - setDesiredContainers(roleStatus, newDesired); - } - } - - // now the dynamic ones. Iterate through the the cluster spec and - // add any role status entries not in the role status - - for (Component component : app.getComponents()) { - String name = component.getName(); - if (roles.containsKey(name)) { - continue; - } - if (component.getUniqueComponentSupport()) { - // THIS NAME IS A GROUP - int desiredInstanceCount = component.getNumberOfContainers().intValue(); - Integer groupCount = 0; - if (groupCounts.containsKey(name)) { - groupCount = groupCounts.get(name); - } - for (int i = groupCount + 1; i <= desiredInstanceCount; i++) { - // this is a new instance of an existing group - String newName = String.format("%s%d", name, i); - if (roles.containsKey(newName)) { - continue; - } - int newPriority = getNewPriority(i); - log.info("Adding new role {}", newName); - ProviderRole dynamicRole = - createComponent(newName, name, component, newPriority); - RoleStatus newRole = buildRole(dynamicRole); - incDesiredContainers(newRole); - log.info("New role {}", newRole); - if (roleHistory != null) { - roleHistory.addNewRole(newRole); - } - newRoles.add(dynamicRole); - } - } else { - // this is a new value - log.info("Adding new role {}, num containers {}", name, - component.getNumberOfContainers()); - ProviderRole dynamicRole = - createComponent(name, name, component, getNewPriority(1)); - RoleStatus newRole = buildRole(dynamicRole); - incDesiredContainers(newRole, - component.getNumberOfContainers().intValue()); - log.info("New role {}", newRole); - if (roleHistory != null) { - roleHistory.addNewRole(newRole); - } - newRoles.add(dynamicRole); - } - } - // and fill in all those roles with their requirements - buildRoleResourceRequirements(); - return newRoles; - } +// private List<ProviderRole> buildRoleRequirementsFromResources() +// throws BadConfigException { +// +// List<ProviderRole> newRoles = new ArrayList<>(0); +// +// // now update every role's desired count. +// // if there are no instance values, that role count goes to zero +// // Add all the existing roles +// // component name -> number of containers +// Map<String, Integer> groupCounts = new HashMap<>(); +// +// for (RoleStatus roleStatus : getRoleStatusMap().values()) { +// if (roleStatus.isExcludeFromFlexing()) { +// // skip inflexible roles, e.g AM itself +// continue; +// } +// long currentDesired = roleStatus.getDesired(); +// String role = roleStatus.getName(); +// String roleGroup = roleStatus.getGroup(); +// Component component = roleStatus.getProviderRole().component; +// int desiredInstanceCount = component.getNumberOfContainers().intValue(); +// +// int newDesired = desiredInstanceCount; +// if (component.getUniqueComponentSupport()) { +// Integer groupCount = 0; +// if (groupCounts.containsKey(roleGroup)) { +// groupCount = groupCounts.get(roleGroup); +// } +// +// newDesired = desiredInstanceCount - groupCount; +// +// if (newDesired > 0) { +// newDesired = 1; +// groupCounts.put(roleGroup, groupCount + newDesired); +// } else { +// newDesired = 0; +// } +// } +// +// if (newDesired == 0) { +// log.info("Role {} has 0 instances specified", role); +// } +// if (currentDesired != newDesired) { +// log.info("Role {} flexed from {} to {}", role, currentDesired, +// newDesired); +// setDesiredContainers(roleStatus, newDesired); +// } +// } +// +// log.info("Counts per component: " + groupCounts); +// // now the dynamic ones. Iterate through the the cluster spec and +// // add any role status entries not in the role status +// +// List<RoleStatus> list = new ArrayList<>(getRoleStatusMap().values()); +// for (RoleStatus roleStatus : list) { +// String name = roleStatus.getName(); +// Component component = roleStatus.getProviderRole().component; +// if (roles.containsKey(name)) { +// continue; +// } +// if (component.getUniqueComponentSupport()) { +// // THIS NAME IS A GROUP +// int desiredInstanceCount = component.getNumberOfContainers().intValue(); +// Integer groupCount = 0; +// if (groupCounts.containsKey(name)) { +// groupCount = groupCounts.get(name); +// } +// log.info("Component " + component.getName() + ", current count = " +// + groupCount + ", desired count = " + desiredInstanceCount); +// for (int i = groupCount + 1; i <= desiredInstanceCount; i++) { +// int priority = roleStatus.getPriority(); +// // this is a new instance of an existing group +// String newName = String.format("%s%d", name, i); +// int newPriority = getNewPriority(priority + i - 1); +// log.info("Adding new role {}", newName); +// ProviderRole dynamicRole = +// createComponent(newName, name, component, newPriority); +// RoleStatus newRole = buildRole(dynamicRole); +// incDesiredContainers(newRole); +// log.info("New role {}", newRole); +// if (roleHistory != null) { +// roleHistory.addNewRole(newRole); +// } +// newRoles.add(dynamicRole); +// } +// } else { +// // this is a new value +// log.info("Adding new role {}", name); +// ProviderRole dynamicRole = +// createComponent(name, name, component, roleStatus.getPriority()); +// RoleStatus newRole = buildRole(dynamicRole); +// incDesiredContainers(roleStatus, +// component.getNumberOfContainers().intValue()); +// log.info("New role {}", newRole); +// if (roleHistory != null) { +// roleHistory.addNewRole(newRole); +// } +// newRoles.add(dynamicRole); +// } +// } +// // and fill in all those roles with their requirements +// buildRoleResourceRequirements(); +// +// return newRoles; +// } private int getNewPriority(int start) { if (!rolePriorityMap.containsKey(start)) { @@ -539,16 +571,20 @@ public class AppState { * @return the role status built up * @throws BadConfigException if a role of that priority already exists */ - public RoleStatus buildRole(ProviderRole providerRole) throws BadConfigException { + public RoleStatus buildRole(ProviderRole providerRole, Component component) + throws BadConfigException { // build role status map int priority = providerRole.id; if (roleStatusMap.containsKey(priority)) { - throw new BadConfigException("Duplicate Provider Key: %s and %s", - providerRole, - roleStatusMap.get(priority) - .getProviderRole()); + throw new BadConfigException("Duplicate component priority Key: %s and %s", + providerRole, roleStatusMap.get(priority)); } RoleStatus roleStatus = new RoleStatus(providerRole); + roleStatus.setResourceRequirements(buildResourceRequirements(roleStatus)); + long prev = roleStatus.getDesired(); + setDesiredContainers(roleStatus, component.getNumberOfContainers().intValue()); + log.info("Set desired containers for component " + component.getName() + + " from " + prev + " to " + roleStatus.getDesired()); roleStatusMap.put(priority, roleStatus); String name = providerRole.name; roles.put(name, providerRole); @@ -559,16 +595,6 @@ public class AppState { } /** - * Build up the requirements of every resource - */ - private void buildRoleResourceRequirements() { - for (RoleStatus role : roleStatusMap.values()) { - role.setResourceRequirements(buildResourceRequirements(role)); - log.info("Setting resource requirements for {} to {}", role.getName(), - role.getResourceRequirements()); - } - } - /** * Look up the status entry of a role or raise an exception * @param key role ID * @return the status entry @@ -731,7 +757,7 @@ public class AppState { } /** - * Enum all nodes by role. + * Enum all nodes by role. * @param role role, or "" for all roles * @return a list of nodes, may be empty */ @@ -785,7 +811,7 @@ public class AppState { } /** - * Build a map of role->nodename->node-info + * Build a map of Component_name -> ContainerId -> ClusterNode * * @return the map of Role name to list of Cluster Nodes */ @@ -850,7 +876,7 @@ public class AppState { /** * Create a container request. - * Update internal state, such as the role request count. + * Update internal state, such as the role request count. * Anti-Affine: the {@link RoleStatus#outstandingAArequest} is set here. * This is where role history information will be used for placement decisions. * @param role role @@ -942,18 +968,9 @@ public class AppState { } private void setDesiredContainers(RoleStatus role, int n) { + int delta = n - role.getComponentMetrics().containersDesired.value(); role.getComponentMetrics().containersDesired.set(n); - appMetrics.containersDesired.set(n); - } - - private void incDesiredContainers(RoleStatus role) { - role.getComponentMetrics().containersDesired.incr(); - appMetrics.containersDesired.incr(); - } - - private void incDesiredContainers(RoleStatus role, int n) { - role.getComponentMetrics().containersDesired.incr(n); - appMetrics.containersDesired.incr(n); + appMetrics.containersDesired.incr(delta); } private void incCompletedContainers(RoleStatus role) { @@ -1001,7 +1018,8 @@ public class AppState { * Build up the resource requirements for this role from the cluster * specification, including substituting max allowed values if the * specification asked for it (except when - * {@link ResourceKeys#YARN_RESOURCE_NORMALIZATION_ENABLED} is set to false). + * {@link org.apache.slider.api.ResourceKeys#YARN_RESOURCE_NORMALIZATION_ENABLED} + * is set to false). * @param role role * during normalization */ @@ -1009,11 +1027,6 @@ public class AppState { // Set up resource requirements from role values String name = role.getName(); Component component = role.getProviderRole().component; - if (component == null) { - // this is for AM container - // TODO why do we need to create the component for AM ? - return Resource.newInstance(1, 512); - } int cores = DEF_YARN_CORES; if (component.getResource() != null && component.getResource().getCpus() != null) { @@ -1282,10 +1295,13 @@ public class AppState { if (roleInstance != null) { int roleId = roleInstance.roleId; String rolename = roleInstance.role; - log.info("Failed container in role[{}] : {}", roleId, rolename); + log.info("Failed container in role[{}] : {}", roleId, + roleInstance.getCompInstanceName()); try { RoleStatus roleStatus = lookupRoleStatus(roleInstance.roleId); decRunningContainers(roleStatus); + roleStatus.getProviderRole().failedInstanceName + .offer(roleInstance.compInstanceName); boolean shortLived = isShortLived(roleInstance); String message; Container failedContainer = roleInstance.container; @@ -1571,7 +1587,7 @@ public class AppState { /** * Look at the allocation status of one role, and trigger add/release - * actions if the number of desired role instances doesn't equal + * actions if the number of desired role instances doesn't equal * (actual + pending). * <p> * MUST be executed from within a synchronized method @@ -1584,7 +1600,6 @@ public class AppState { @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter") private List<AbstractRMOperation> reviewOneRole(RoleStatus role) throws SliderInternalStateException, TriggerClusterTeardownException { - log.info("review one role " + role.getName()); List<AbstractRMOperation> operations = new ArrayList<>(); long delta; long expected; @@ -1594,9 +1609,7 @@ public class AppState { expected = role.getDesired(); } - log.info("Reviewing {} : ", role); - log.debug("Expected {}, Requested/Running {}, Delta: {}", expected, - role.getActualAndRequested(), delta); + log.info("Reviewing " + role.getName() + ": " + role.getComponentMetrics()); checkFailureThreshold(role); if (expected < 0 ) { @@ -1729,7 +1742,9 @@ public class AppState { for (RoleInstance possible : finalCandidates) { log.info("Targeting for release: {}", possible); containerReleaseSubmitted(possible.container); - operations.add(new ContainerReleaseOperation(possible.getId())); + role.getProviderRole().failedInstanceName + .offer(possible.compInstanceName); + operations.add(new ContainerReleaseOperation(possible.getContainerId())); } } @@ -1783,7 +1798,7 @@ public class AppState { for (RoleInstance role : activeRoleInstances) { if (role.container.getId().equals(containerId)) { containerReleaseSubmitted(role.container); - operations.add(new ContainerReleaseOperation(role.getId())); + operations.add(new ContainerReleaseOperation(role.getContainerId())); } } @@ -1907,17 +1922,6 @@ public class AppState { } /** - * Get diagnostics info about containers - */ - public String getContainerDiagnosticInfo() { - StringBuilder builder = new StringBuilder(); - for (RoleStatus roleStatus : getRoleStatusMap().values()) { - builder.append(roleStatus).append('\n'); - } - return builder.toString(); - } - - /** * Event handler for the list of active containers on restart. * Sets the info key {@link StatusKeys#INFO_CONTAINERS_AM_RESTART} * to the size of the list passed down (and does not set it if none were) @@ -1965,10 +1969,10 @@ public class AppState { //update app state internal structures and maps + //TODO recover the component instance name from zk registry ? RoleInstance instance = new RoleInstance(container); instance.command = roleName; instance.role = roleName; - instance.group = role.getGroup(); instance.roleId = roleId; instance.environment = new String[0]; instance.container = container; http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java index de52f4e..736dfd1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java @@ -19,6 +19,7 @@ package org.apache.slider.server.appmaster.state; import com.google.common.base.Preconditions; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.registry.client.binding.RegistryTypeUtils; import org.apache.hadoop.registry.client.types.Endpoint; import org.apache.hadoop.registry.client.types.ProtocolTypes; @@ -27,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.slider.api.ClusterNode; import org.apache.slider.api.proto.Messages; +import org.apache.slider.api.resource.ConfigFile; import org.apache.slider.api.types.ContainerInformation; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.providers.ProviderRole; @@ -42,6 +44,8 @@ public final class RoleInstance implements Cloneable { public Container container; public ProviderRole providerRole; + public long componentId = -1; + public String compInstanceName = null; /** * Container ID */ @@ -58,7 +62,6 @@ public final class RoleInstance implements Cloneable { * Name of the role */ public String role; - public String group; /** * Version of the app @@ -106,7 +109,7 @@ public final class RoleInstance implements Cloneable { public String host; public String hostURL; public ContainerAllocationOutcome placement; - + public Path compInstanceDir; /** * A list of registered endpoints. @@ -114,10 +117,24 @@ public final class RoleInstance implements Cloneable { private List<Endpoint> endpoints = new ArrayList<>(2); - public RoleInstance(ContainerAssignment assignment) { - this(assignment.container); - placement = assignment.placement; + public RoleInstance(Container container, ProviderRole role) { + this(container); + if (role.componentIdCounter != null) { + componentId = role.componentIdCounter.getAndIncrement(); + compInstanceName = role.name + componentId; + } else { + compInstanceName = role.name; + } + this.providerRole = role; + } + + public RoleInstance(Container container, ProviderRole role, + String compInstanceName) { + this(container); + this.compInstanceName = compInstanceName; + this.providerRole = role; } + /** * Create an instance to track an allocated container * @param container a container which must be non null, and have a non-null Id field. @@ -136,10 +153,6 @@ public final class RoleInstance implements Cloneable { hostURL = "http://" + container.getNodeHttpAddress(); } } - - public ContainerId getId() { - return container.getId(); - } public NodeId getHost() { return container.getNodeId(); @@ -151,6 +164,7 @@ public final class RoleInstance implements Cloneable { new StringBuilder("RoleInstance{"); sb.append("role='").append(role).append('\''); sb.append(", id='").append(id).append('\''); + sb.append(", instanceName='").append(compInstanceName).append('\''); sb.append(", container=").append(SliderUtils.containerToString(container)); sb.append(", createTime=").append(createTime); sb.append(", startTime=").append(startTime); @@ -170,7 +184,7 @@ public final class RoleInstance implements Cloneable { } public ContainerId getContainerId() { - return container != null ? container.getId() : null; + return container.getId(); } /** @@ -322,4 +336,8 @@ public final class RoleInstance implements Cloneable { } return info; } + + public String getCompInstanceName() { + return compInstanceName; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java index 5051aee..9842481 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java @@ -272,6 +272,7 @@ public final class RoleStatus implements MetricSet { // containers -- maybe we need releasing //if we are releasing, remove the number that are already released. //but never switch to a positive + // TODO, WHY is this min operation even needed ??? if delta is negative, it's always < 0 ??? delta = Math.min(delta, 0); } return delta; http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java index 118ca9d..5bc6dce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java @@ -214,6 +214,7 @@ public interface StateAccessForProviders { /** * Find out about the nodes for specific roles + * Component_name -> ContainerId -> ClusterNode * @return */ Map<String, Map<String, ClusterNode>> getRoleClusterNodeMapping(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java index 0f6247d..ac89ed8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java @@ -59,7 +59,7 @@ public interface RestApiErrorMessages { ERROR_RESOURCE_CPUS_INVALID_RANGE + " for component %s (or at the global level)"; String ERROR_CONTAINERS_COUNT_INVALID = - "Required no of containers not specified"; + "Invalid no of containers specified"; String ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID = ERROR_CONTAINERS_COUNT_INVALID + ERROR_SUFFIX_FOR_COMPONENT; http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java index 776ce00..d7c72a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java @@ -20,17 +20,30 @@ package org.apache.slider.util; import com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.slider.api.resource.Application; import org.apache.slider.api.resource.Artifact; import org.apache.slider.api.resource.Component; +import org.apache.slider.api.resource.ConfigFile; import org.apache.slider.api.resource.Configuration; import org.apache.slider.api.resource.Resource; import org.apache.slider.common.tools.SliderUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class ServiceApiUtil { +import java.io.IOException; +import java.nio.file.Paths; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +public class ServiceApiUtil { + private static final Logger log = + LoggerFactory.getLogger(ServiceApiUtil.class); @VisibleForTesting - public static void validateApplicationPostPayload(Application application) { + public static void validateApplicationPayload(Application application, + FileSystem fs) throws IOException { if (StringUtils.isEmpty(application.getName())) { throw new IllegalArgumentException( RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID); @@ -64,11 +77,13 @@ public class ServiceApiUtil { application.getArtifact().getType()); // container size - if (application.getNumberOfContainers() == null) { + if (application.getNumberOfContainers() == null + || application.getNumberOfContainers() < 0) { throw new IllegalArgumentException( - RestApiErrorMessages.ERROR_CONTAINERS_COUNT_INVALID); + RestApiErrorMessages.ERROR_CONTAINERS_COUNT_INVALID + ": " + + application.getNumberOfContainers()); } - + validateConfigFile(application.getConfiguration().getFiles(), fs); // Since it is a simple app with no components, create a default component application.getComponents().add(createDefaultComponent(application)); } else { @@ -114,11 +129,13 @@ public class ServiceApiUtil { if (comp.getNumberOfContainers() == null) { comp.setNumberOfContainers(globalNumberOfContainers); } - if (comp.getNumberOfContainers() == null) { + if (comp.getNumberOfContainers() == null + || comp.getNumberOfContainers() < 0) { throw new IllegalArgumentException(String.format( - RestApiErrorMessages.ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID, - comp.getName())); + RestApiErrorMessages.ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID + + ": " + comp.getNumberOfContainers(), comp.getName())); } + validateConfigFile(comp.getConfiguration().getFiles(), fs); } } @@ -128,6 +145,46 @@ public class ServiceApiUtil { } } + // 1) Verify the src_file exists and non-empty for template + // 2) dest_file is absolute path + private static void validateConfigFile(List<ConfigFile> list, FileSystem fs) + throws IOException { + Set<String> destFileSet = new HashSet<>(); + + for (ConfigFile file : list) { + if (file.getType().equals(ConfigFile.TypeEnum.TEMPLATE) && StringUtils + .isEmpty(file.getSrcFile())) { + throw new IllegalArgumentException( + "Src_file is empty for " + ConfigFile.TypeEnum.TEMPLATE); + + } + if (!StringUtils.isEmpty(file.getSrcFile())) { + Path p = new Path(file.getSrcFile()); + if (!fs.exists(p)) { + throw new IllegalArgumentException( + "Src_file does not exist for config file: " + file + .getSrcFile()); + } + } + + if (StringUtils.isEmpty(file.getDestFile())) { + throw new IllegalArgumentException("Dest_file is empty."); + } + // validate dest_file is absolute + if (!Paths.get(file.getDestFile()).isAbsolute()) { + throw new IllegalArgumentException( + "Dest_file must be absolute path: " + file.getDestFile()); + } + + if (destFileSet.contains(file.getDestFile())) { + throw new IllegalArgumentException( + "Duplicated ConfigFile exists: " + file.getDestFile()); + } + destFileSet.add(file.getDestFile()); + } + } + + private static void validateApplicationResource(Resource resource, Component comp, Artifact.TypeEnum artifactType) { // Only apps/components of type APPLICATION can skip resource requirement @@ -200,4 +257,8 @@ public class ServiceApiUtil { comp.setLaunchCommand(app.getLaunchCommand()); return comp; } + + public static String $(String s) { + return "${" + s +"}"; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.java index da2ed0d..9e79821 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.java @@ -67,7 +67,7 @@ public class TestMockAppStateDynamicHistory extends BaseMockAppStateTest return new MockYarnEngine(8, 1); } - @Test + // TODO does not support adding new components dynamically public void testDynamicRoleHistory() throws Throwable { String dynamic = "dynamicRole"; @@ -81,12 +81,8 @@ public class TestMockAppStateDynamicHistory extends BaseMockAppStateTest .COMPONENT_PLACEMENT_POLICY, "" + placementPolicy); application.getComponents().add(component); - // write the definitions - List<ProviderRole> updates = appState.updateComponents( + appState.updateComponents( Collections.singletonMap(dynamic, desired)); - assertEquals(1, updates.size()); - ProviderRole updatedRole = updates.get(0); - assertEquals(updatedRole.placementPolicy, placementPolicy); // now look at the role map assertNotNull(appState.getRoleMap().get(dynamic)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java index 01bf9bd..6d8e963 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java @@ -87,7 +87,7 @@ public class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest createAndStartNodes(); } - @Test + // TODO does not support adding new components dynamically public void testDynamicFlexAddRole() throws Throwable { Application application = appState.getClusterStatus(); Component component = new Component().name("dynamicAdd7") @@ -96,16 +96,12 @@ public class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest appState.updateComponents(Collections.singletonMap(component.getName(), component.getNumberOfContainers())); createAndStartNodes(); - dumpClusterDescription("updated CD", appState.getClusterStatus()); appState.lookupRoleStatus("dynamicAdd7"); } @Test public void testDynamicFlexDropRole() throws Throwable { appState.updateComponents(Collections.singletonMap("dynamic-6", 0L)); - - Application getCD = appState.getClusterStatus(); - dumpClusterDescription("updated CD", getCD); //status is retained for future appState.lookupRoleStatus("dynamic-6"); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java index eaf5271..54ffe17 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java @@ -26,10 +26,15 @@ import org.apache.slider.server.appmaster.model.mock.MockRoles; import org.apache.slider.server.appmaster.model.mock.MockYarnEngine; import org.apache.slider.server.appmaster.state.AppStateBindingInfo; import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector; +import org.apache.slider.server.appmaster.state.RoleInstance; import org.apache.slider.server.appmaster.state.RoleStatus; import org.junit.Test; import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.TreeMap; /** * Test that if you have more than one role, the right roles are chosen for @@ -72,40 +77,76 @@ public class TestMockAppStateUniqueNames extends BaseMockAppStateTest return application; } + public static Map<String, RoleInstance> organize(List<RoleInstance> + instances) { + Map<String, RoleInstance> map = new TreeMap<>(); + for (RoleInstance instance : instances) { + assertFalse("Multiple role instances for unique name " + instance + .compInstanceName, map.containsKey(instance.compInstanceName)); + System.out.println("Adding to map " + instance.compInstanceName + " for" + + instance.role); + map.put(instance.compInstanceName, instance); + } + return map; + } + + public static void verifyInstances(List<RoleInstance> instances, String + group, String... roles) { + assertEquals(roles.length, instances.size()); + Map<String, RoleInstance> map = organize(instances); + int i = 0; + for (Entry<String, RoleInstance> entry : map.entrySet()) { + assertEquals(roles[i], entry.getKey()); + RoleInstance instance = entry.getValue(); + assertEquals(roles[i], instance.compInstanceName); + assertEquals(group, instance.role); + assertEquals(group, instance.providerRole.name); + assertEquals(group, instance.providerRole.group); + // TODO remove group from provider role if it continues to be unused + i++; + } + } + @Test public void testDynamicFlexDown() throws Throwable { createAndStartNodes(); + List<RoleInstance> instances = appState.cloneOwnedContainerList(); + verifyInstances(instances, "group1", "group10", "group11"); + appState.updateComponents(Collections.singletonMap("group1", 0L)); createAndStartNodes(); - RoleStatus roleStatus = appState.lookupRoleStatus("group11"); + instances = appState.cloneOwnedContainerList(); + assertEquals(0, instances.size()); + + RoleStatus roleStatus = appState.lookupRoleStatus("group1"); assertEquals(0, roleStatus.getDesired()); assertEquals(1024L, roleStatus.getResourceRequirements().getMemorySize()); assertEquals(2, roleStatus.getResourceRequirements().getVirtualCores()); assertEquals("group1", roleStatus.getGroup()); + + // now flex back up + appState.updateComponents(Collections.singletonMap("group1", 3L)); + createAndStartNodes(); + instances = appState.cloneOwnedContainerList(); + verifyInstances(instances, "group1", "group10", "group11", "group12"); + // fails because the names continue at N+1, with group12, group13, group14 } @Test public void testDynamicFlexUp() throws Throwable { createAndStartNodes(); + List<RoleInstance> instances = appState.cloneOwnedContainerList(); + verifyInstances(instances, "group1", "group10", "group11"); + appState.updateComponents(Collections.singletonMap("group1", 3L)); createAndStartNodes(); - RoleStatus group11 = appState.lookupRoleStatus("group11"); - RoleStatus group12 = appState.lookupRoleStatus("group12"); - RoleStatus group13 = appState.lookupRoleStatus("group13"); - assertEquals(1, group11.getDesired()); - assertEquals(1, group12.getDesired()); - assertEquals(1, group13.getDesired()); - assertEquals(1024L, group11.getResourceRequirements().getMemorySize()); - assertEquals(1024L, group12.getResourceRequirements().getMemorySize()); - assertEquals(1024L, group13.getResourceRequirements().getMemorySize()); - assertEquals(2, group11.getResourceRequirements().getVirtualCores()); - assertEquals(2, group12.getResourceRequirements().getVirtualCores()); - assertEquals(2, group13.getResourceRequirements().getVirtualCores()); - assertEquals("group1", group11.getGroup()); - assertEquals("group1", group12.getGroup()); - assertEquals("group1", group13.getGroup()); - - appState.refreshClusterStatus(); + instances = appState.cloneOwnedContainerList(); + verifyInstances(instances, "group1", "group10", "group11", "group12"); + + RoleStatus group1 = appState.lookupRoleStatus("group1"); + assertEquals(3, group1.getDesired()); + assertEquals(1024L, group1.getResourceRequirements().getMemorySize()); + assertEquals("group1", group1.getGroup()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java index 046bd83..d382c8a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java @@ -27,6 +27,7 @@ import org.apache.slider.server.appmaster.model.mock.MockAppState; import org.apache.slider.server.appmaster.model.mock.MockRoles; import org.apache.slider.server.appmaster.operations.AbstractRMOperation; import org.apache.slider.server.appmaster.operations.ContainerRequestOperation; +import org.apache.slider.server.appmaster.state.RoleStatus; import org.junit.Test; import java.util.Collections; @@ -47,6 +48,11 @@ public class TestMockContainerResourceAllocations extends BaseMockAppStateTest { Component role0 = appState.getClusterStatus().getComponent(MockRoles.ROLE0); role0.resource(new org.apache.slider.api.resource.Resource().memory("512") .cpus(2)); + // hack - because role0 is created before the test run + RoleStatus role0Status = + appState.getRoleStatusMap().get(appState.getRoleMap().get(ROLE0).id); + role0Status.setResourceRequirements( + appState.buildResourceRequirements(role0Status)); appState.updateComponents(Collections.singletonMap(role0.getName(), role0.getNumberOfContainers())); List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes(); @@ -58,12 +64,17 @@ public class TestMockContainerResourceAllocations extends BaseMockAppStateTest { assertEquals(2, requirements.getVirtualCores()); } + //TODO replace with resource profile feature in yarn @Test public void testMaxMemAllocations() throws Throwable { // max core allocations no longer supported Component role0 = appState.getClusterStatus().getComponent(MockRoles.ROLE0); role0.resource(new org.apache.slider.api.resource.Resource() .memory(ResourceKeys.YARN_RESOURCE_MAX).cpus(2)); + RoleStatus role0Status = + appState.getRoleStatusMap().get(appState.getRoleMap().get(ROLE0).id); + role0Status.setResourceRequirements( + appState.buildResourceRequirements(role0Status)); appState.updateComponents(Collections.singletonMap(role0.getName(), role0.getNumberOfContainers())); List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java index eca8401..4352959 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java @@ -176,7 +176,14 @@ public abstract class BaseMockAppStateTest extends SliderTestBase implements */ public RoleInstance roleInstance(ContainerAssignment assigned) { Container target = assigned.container; - RoleInstance ri = new RoleInstance(target); + String failedInstance = + assigned.role.getProviderRole().failedInstanceName.poll(); + RoleInstance ri; + if (failedInstance != null) { + ri = new RoleInstance(target, assigned.role.getProviderRole(), failedInstance); + } else { + ri = new RoleInstance(target, assigned.role.getProviderRole()); + } ri.roleId = assigned.role.getPriority(); ri.role = assigned.role.getName(); return ri; http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockProviderService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockProviderService.java index 112a5ac..4098cf7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockProviderService.java @@ -30,6 +30,7 @@ import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.core.launch.ContainerLauncher; import org.apache.slider.providers.ProviderRole; import org.apache.slider.providers.ProviderService; +import org.apache.slider.server.appmaster.state.RoleInstance; import org.apache.slider.server.appmaster.state.StateAccessForProviders; import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders; @@ -118,7 +119,8 @@ public class MockProviderService implements ProviderService { @Override public void buildContainerLaunchContext(ContainerLauncher containerLauncher, Application application, Container container, ProviderRole providerRole, - SliderFileSystem sliderFileSystem) throws IOException, SliderException { + SliderFileSystem sliderFileSystem, RoleInstance roleInstance) + throws IOException, SliderException { } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org