SLIDER-875 better handling and support of nested external components through addition of role.prefix
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a4dc574c Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a4dc574c Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a4dc574c Branch: refs/heads/develop Commit: a4dc574c9f60e2f07906b37f8200e9488269c20d Parents: 000b38d Author: Billie Rinaldi <billie.rina...@gmail.com> Authored: Wed Jul 13 09:53:47 2016 -0700 Committer: Billie Rinaldi <billie.rina...@gmail.com> Committed: Wed Jul 13 09:53:47 2016 -0700 ---------------------------------------------------------------------- .../java/org/apache/slider/api/RoleKeys.java | 5 + .../apache/slider/common/tools/SliderUtils.java | 9 ++ .../slider/core/build/InstanceBuilder.java | 108 +++++++++++---- .../slider/core/conf/ConfTreeOperations.java | 39 ++++++ .../providers/agent/AgentClientProvider.java | 6 +- .../slider/providers/agent/AgentKeys.java | 1 + .../providers/agent/AgentProviderService.java | 42 +++--- .../slider/providers/agent/AgentUtils.java | 30 ++-- .../providers/agent/ComponentCommandOrder.java | 110 ++++++++++----- .../slider/server/appmaster/state/AppState.java | 5 + .../appConfig_external_component_nested.json | 12 ++ .../test_min_pkg/sleep_cmd/metainfo.json | 6 + .../metainfo_external_component_nested.json | 14 ++ .../resources_external_component_nested.json | 12 ++ .../TestBuildExternalComponent.groovy | 138 +++++++++++++++++++ .../agent/TestAgentProviderService.java | 35 ++--- .../agent/TestComponentCommandOrder.java | 107 +++++++++++++- .../apache/slider/funtest/ResourcePaths.groovy | 4 + .../funtest/misc/ExternalComponentIT.groovy | 38 +++++ 19 files changed, 589 insertions(+), 132 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java b/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java index 812a6b3..ce413ff 100644 --- a/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java +++ b/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java @@ -35,6 +35,11 @@ public interface RoleKeys { String ROLE_GROUP = "role.group"; /** + * The prefix of a role: {@value} + */ + String ROLE_PREFIX = "role.prefix"; + + /** * Status report: number actually granted : {@value} */ String ROLE_ACTUAL_INSTANCES = "role.actual.instances"; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index c48e109..b798d4d 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -123,6 +123,8 @@ import java.util.zip.GZIPOutputStream; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; +import static org.apache.slider.common.SliderKeys.COMPONENT_SEPARATOR; + /** * These are slider-specific Util methods */ @@ -2612,4 +2614,11 @@ public final class SliderUtils { } return buffer.toString(); } + + public static String trimPrefix(String prefix) { + if (prefix != null && prefix.endsWith(COMPONENT_SEPARATOR)) { + return prefix.substring(0, prefix.length()-1); + } + return prefix; + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java index 4afdf7c..6a7975e 100644 --- a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java +++ b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java @@ -69,6 +69,7 @@ import static org.apache.slider.api.OptionKeys.INTERNAL_SNAPSHOT_CONF_PATH; import static org.apache.slider.api.OptionKeys.ZOOKEEPER_HOSTS; import static org.apache.slider.api.OptionKeys.ZOOKEEPER_PATH; import static org.apache.slider.api.OptionKeys.ZOOKEEPER_QUORUM; +import static org.apache.slider.api.RoleKeys.ROLE_PREFIX; import static org.apache.slider.common.SliderKeys.COMPONENT_AM; import static org.apache.slider.common.SliderKeys.COMPONENT_SEPARATOR; import static org.apache.slider.common.SliderKeys.COMPONENT_TYPE; @@ -85,7 +86,7 @@ public class InstanceBuilder { private final CoreFileSystem coreFS; private final InstancePaths instancePaths; private AggregateConf instanceDescription; - private Map<Path, Path> externalAppDefs = new HashMap<>(); + private Map<String, Path> externalAppDefs = new HashMap<>(); private TreeSet<Integer> priorities = new TreeSet<>(); private static final Logger log = @@ -272,8 +273,7 @@ public class InstanceBuilder { continue; } Map<String, String> options = entry.getValue(); - if (options.containsKey(COMPONENT_TYPE) && - EXTERNAL_COMPONENT.equals(options.get(COMPONENT_TYPE))) { + if (EXTERNAL_COMPONENT.equals(options.get(COMPONENT_TYPE))) { externalComponents.add(entry.getKey()); } } @@ -284,17 +284,37 @@ public class InstanceBuilder { private void mergeExternalComponent(ConfTreeOperations ops, ConfTreeOperations externalOps, String externalComponent, - Integer priority) { + Integer priority) throws BadConfigException { for (String subComponent : externalOps.getComponentNames()) { if (COMPONENT_AM.equals(subComponent)) { continue; } + String prefix = externalComponent + COMPONENT_SEPARATOR; log.debug("Merging options for {} into {}", subComponent, - externalComponent + COMPONENT_SEPARATOR + subComponent); - MapOperations subComponentOps = ops.getOrAddComponent(externalComponent + - COMPONENT_SEPARATOR + subComponent); + prefix + subComponent); + MapOperations subComponentOps = ops.getOrAddComponent( + prefix + subComponent); + if (priority == null) { + SliderUtils.mergeMaps(subComponentOps, + ops.getComponent(externalComponent).options); + subComponentOps.remove(COMPONENT_TYPE); + } + SliderUtils.mergeMapsIgnorePrefixes(subComponentOps, externalOps.getComponent(subComponent), PREFIXES_TO_SKIP); + + // add prefix to existing prefix + String existingPrefix = subComponentOps.get(ROLE_PREFIX); + if (existingPrefix != null) { + if (!subComponent.startsWith(existingPrefix)) { + throw new BadConfigException("Bad prefix " + existingPrefix + + " for subcomponent " + subComponent + " of " + externalComponent); + } + prefix = prefix + existingPrefix; + } + subComponentOps.set(ROLE_PREFIX, prefix); + + // adjust priority if (priority != null) { subComponentOps.put(ResourceKeys.COMPONENT_PRIORITY, Integer.toString(priority)); @@ -322,10 +342,6 @@ public class InstanceBuilder { if (COMPONENT_AM.equals(entry.getKey())) { continue; } - if (entry.getKey().contains(COMPONENT_SEPARATOR)) { - throw new BadConfigException("Components must not contain " + - COMPONENT_SEPARATOR + ": " + entry.getKey()); - } if (entry.getValue().containsKey(ResourceKeys.COMPONENT_PRIORITY)) { priorities.add(Integer.parseInt(entry.getValue().get( ResourceKeys.COMPONENT_PRIORITY))); @@ -358,31 +374,73 @@ public class InstanceBuilder { throw new BadConfigException("Couldn't read configuration for " + "external component " + component); } - String externalAppDef = componentConf.getAppConfOperations() - .getGlobalOptions().get(AgentKeys.APP_DEF); + + ConfTreeOperations componentAppConf = componentConf.getAppConfOperations(); + String externalAppDef = componentAppConf.get(AgentKeys.APP_DEF); if (SliderUtils.isSet(externalAppDef)) { Path newAppDef = new Path(coreFS.buildAppDefDirPath(clustername), component + "_" + SliderKeys.DEFAULT_APP_PKG); - componentConf.getAppConfOperations().set(AgentKeys.APP_DEF, newAppDef); - externalAppDefs.put(newAppDef, new Path(externalAppDef)); + componentAppConf.set(AgentKeys.APP_DEF, newAppDef); + componentAppConf.append(AgentKeys.APP_DEF_ORIGINAL, externalAppDef); + log.info("Copying external appdef {} to {} for {}", externalAppDef, + newAppDef, component); + externalAppDefs.put(externalAppDef, newAppDef); + externalAppDef = newAppDef.toString(); } + for (String rcomp : componentConf.getResourceOperations() .getComponentNames()) { if (COMPONENT_AM.equals(rcomp)) { continue; } log.debug("Adding component {} to appConf for {}", rcomp, component); - componentConf.getAppConfOperations().getOrAddComponent(rcomp); + componentAppConf.getOrAddComponent(rcomp); } - SliderUtils.mergeMaps( - componentConf.getAppConfOperations().getGlobalOptions().options, - appConf.getComponent(component).options); - componentConf.getAppConfOperations().getGlobalOptions() - .remove(COMPONENT_TYPE); componentConf.resolve(); - mergeExternalComponent(appConf, componentConf.getAppConfOperations(), - component, null); + for (String rcomp : componentConf.getResourceOperations() + .getComponentNames()) { + if (COMPONENT_AM.equals(rcomp)) { + continue; + } + String componentAppDef = componentAppConf.getComponentOpt( + rcomp, AgentKeys.APP_DEF, null); + if (SliderUtils.isUnset(componentAppDef) || + componentAppDef.equals(externalAppDef)) { + continue; + } + if (externalAppDefs.containsKey(componentAppDef)) { + log.info("Using external appdef {} for {}", + externalAppDefs.get(componentAppDef), rcomp); + } else { + String existingPrefix = componentAppConf.getComponentOpt(rcomp, + ROLE_PREFIX, null); + if (SliderUtils.isUnset(existingPrefix)) { + existingPrefix = ""; + } else { + existingPrefix = COMPONENT_SEPARATOR + SliderUtils.trimPrefix( + existingPrefix); + } + Path newAppDef = new Path(coreFS.buildAppDefDirPath(clustername), + component + existingPrefix + "_" + SliderKeys.DEFAULT_APP_PKG); + externalAppDefs.put(componentAppDef, newAppDef); + log.info("Copying external appdef {} to {} for {}", componentAppDef, + newAppDef, component + COMPONENT_SEPARATOR + rcomp); + } + componentAppConf.setComponentOpt(rcomp, AgentKeys.APP_DEF, + externalAppDefs.get(componentAppDef).toString()); + componentAppConf.appendComponentOpt(rcomp, + AgentKeys.APP_DEF_ORIGINAL, componentAppDef); + } + Set<Path> newAppDefs = new HashSet<>(); + newAppDefs.addAll(externalAppDefs.values()); + if (newAppDefs.size() != externalAppDefs.size()) { + throw new IllegalStateException("Values repeat in external appdefs " + + externalAppDefs); + } + log.info("External appdefs after {}: {}", component, externalAppDefs); + + mergeExternalComponent(appConf, componentAppConf, component, null); mergeExternalComponent(resources, componentConf.getResourceOperations(), component, getNextPriority()); } @@ -411,8 +469,8 @@ public class InstanceBuilder { action = new ConfDirSnapshotAction(appconfdir); } persister.save(instanceDescription, action); - for (Entry<Path, Path> appDef : externalAppDefs.entrySet()) { - SliderUtils.copy(conf, appDef.getValue(), appDef.getKey()); + for (Entry<String, Path> appDef : externalAppDefs.entrySet()) { + SliderUtils.copy(conf, new Path(appDef.getKey()), appDef.getValue()); } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java index 4a0ae41..038513e 100644 --- a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java +++ b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java @@ -214,6 +214,23 @@ public class ConfTreeOperations { public String get(String key) { return globalOptions.get(key); } + /** + * append to a global option + * @param key key + * @return value + * + */ + public String append(String key, String value) { + if (SliderUtils.isUnset(value)) { + return null; + } + if (globalOptions.containsKey(key)) { + globalOptions.put(key, globalOptions.get(key) + "," + value); + } else { + globalOptions.put(key, value); + } + return globalOptions.get(key); + } /** * Propagate all global keys matching a prefix @@ -471,4 +488,26 @@ public class ConfTreeOperations { setComponentOpt(role, option, Long.toString(val)); } + /** + * append to a component option + * @param key key + * @return value + * + */ + public String appendComponentOpt(String role, String key, String value) { + if (SliderUtils.isUnset(value)) { + return null; + } + MapOperations roleopts = getComponent(role); + if (roleopts == null) { + return null; + } + + if (roleopts.containsKey(key)) { + roleopts.put(key, roleopts.get(key) + "," + value); + } else { + roleopts.put(key, value); + } + return roleopts.get(key); + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java index 6eae75e..8c0a2e4 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java @@ -184,7 +184,8 @@ public class AgentClientProvider extends AbstractClientProvider if (metaInfo != null) { Component componentDef = metaInfo.getApplicationComponent( - getMetainfoComponentName(name)); + getMetainfoComponentName(name, + instanceDefinition.getAppConfOperations())); if (componentDef == null) { throw new BadConfigException( "Component %s is not a member of application.", name); @@ -214,7 +215,8 @@ public class AgentClientProvider extends AbstractClientProvider // fileSystem may be null for tests if (metaInfo != null) { Component componentDef = metaInfo.getApplicationComponent( - getMetainfoComponentName(name)); + getMetainfoComponentName(name, + instanceDefinition.getAppConfOperations())); // already checked it wasn't null // ensure that intance count is 0 for client components http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java index 8514401..565d73d 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java @@ -68,6 +68,7 @@ public interface AgentKeys { String AGENT_MAIN_SCRIPT = "agent/main.py"; String APP_DEF = "application.def"; + String APP_DEF_ORIGINAL = "application.def.original"; String ADDON_PREFIX = "application.addon."; String ADDONS = "application.addons"; String AGENT_VERSION = "agent.version"; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java index 66fac99..452122f 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java @@ -133,7 +133,7 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import static org.apache.slider.providers.agent.AgentUtils.getMetainfoMapKey; +import static org.apache.slider.api.RoleKeys.ROLE_PREFIX; import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_AGENTS; /** @@ -288,7 +288,8 @@ public class AgentProviderService extends AbstractProviderService implements SliderFileSystem fileSystem, String roleGroup) throws IOException, SliderException { - String mapKey = getMetainfoMapKey(roleGroup); + String mapKey = instanceDefinition.getAppConfOperations() + .getComponentOpt(roleGroup, ROLE_PREFIX, DEFAULT_METAINFO_MAP_KEY); String appDef = SliderUtils.getApplicationDefinitionPath( instanceDefinition.getAppConfOperations(), roleGroup); MapOperations component = null; @@ -319,16 +320,17 @@ public class AgentProviderService extends AbstractProviderService implements log.info("Modifying external metainfo component name to {}", comp.getName()); } - String commandPrefix = mapKey.substring(0, - mapKey.indexOf(COMPONENT_SEPARATOR)+1); for (CommandOrder co : commandOrders) { log.info("Adding prefix {} to command order {}", - commandPrefix, co); - co.setCommand(commandPrefix + co.getCommand()); - co.setRequires(commandPrefix + co.getRequires()); + mapKey, co); + co.setCommand(mapKey + co.getCommand()); + co.setRequires(mapKey + co.getRequires()); } } - commandOrder.mergeCommandOrders(commandOrders); + log.debug("Merging command orders {} for {}", commandOrders, + roleGroup); + commandOrder.mergeCommandOrders(commandOrders, + instanceDefinition.getResourceOperations()); Map<String, DefaultConfig> defaultConfigs = initializeDefaultConfigs(fileSystem, appDef, metaInfo); metaInfoMap.put(mapKey, new MetainfoHolder(metaInfo, defaultConfigs)); @@ -1393,7 +1395,10 @@ public class AgentProviderService extends AbstractProviderService implements @VisibleForTesting protected Metainfo getMetaInfo(String roleGroup) { - MetainfoHolder mh = this.metaInfoMap.get(getMetainfoMapKey(roleGroup)); + ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); + String mapKey = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, + DEFAULT_METAINFO_MAP_KEY); + MetainfoHolder mh = this.metaInfoMap.get(mapKey); if (mh == null) { return null; } @@ -1468,7 +1473,10 @@ public class AgentProviderService extends AbstractProviderService implements } protected Map<String, DefaultConfig> getDefaultConfigs(String roleGroup) { - return metaInfoMap.get(getMetainfoMapKey(roleGroup)).defaultConfigs; + ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); + String mapKey = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, + DEFAULT_METAINFO_MAP_KEY); + return metaInfoMap.get(mapKey).defaultConfigs; } private int getHeartbeatMonitorInterval() { @@ -2900,14 +2908,14 @@ public class AgentProviderService extends AbstractProviderService implements tokens.put("${NN_HOST}", URI.create(nnuri).getHost()); tokens.put("${ZK_HOST}", appConf.get(OptionKeys.ZOOKEEPER_HOSTS)); tokens.put("${DEFAULT_ZK_PATH}", appConf.get(OptionKeys.ZOOKEEPER_PATH)); - String mapKey = getMetainfoMapKey(componentGroup); + String prefix = appConf.getComponentOpt(componentGroup, ROLE_PREFIX, + null); String dataDirSuffix = ""; - if (!DEFAULT_METAINFO_MAP_KEY.equals(mapKey)) { - dataDirSuffix = "_" + mapKey.substring(0, mapKey.length()-1); + if (prefix == null) { + prefix = ""; } else { - mapKey = ""; + dataDirSuffix = "_" + SliderUtils.trimPrefix(prefix); } - mapKey = mapKey.toLowerCase(); tokens.put("${DEFAULT_DATA_DIR}", getAmState() .getInternalsSnapshot() .getGlobalOptions() @@ -2915,8 +2923,8 @@ public class AgentProviderService extends AbstractProviderService implements tokens.put("${JAVA_HOME}", appConf.get(AgentKeys.JAVA_HOME)); tokens.put("${COMPONENT_NAME}", componentName); tokens.put("${COMPONENT_NAME.lc}", componentName.toLowerCase()); - tokens.put("${COMPONENT_PREFIX}", mapKey); - tokens.put("${COMPONENT_PREFIX.lc}", mapKey.toLowerCase()); + tokens.put("${COMPONENT_PREFIX}", prefix); + tokens.put("${COMPONENT_PREFIX.lc}", prefix.toLowerCase()); if (!componentName.equals(componentGroup) && componentName.startsWith(componentGroup)) { tokens.put("${COMPONENT_ID}", componentName.substring(componentGroup.length())); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java index ed5108c..23e05a3 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java @@ -21,6 +21,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.core.conf.ConfTreeOperations; import org.apache.slider.core.exceptions.BadConfigException; import org.apache.slider.providers.agent.application.metadata.AbstractMetainfoParser; import org.apache.slider.providers.agent.application.metadata.AddonPackageMetainfoParser; @@ -35,8 +36,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; -import static org.apache.slider.common.SliderKeys.COMPONENT_SEPARATOR; -import static org.apache.slider.providers.agent.AgentKeys.DEFAULT_METAINFO_MAP_KEY; +import static org.apache.slider.api.RoleKeys.ROLE_PREFIX; /** * @@ -135,24 +135,16 @@ public class AgentUtils { return new DefaultConfigParser().parse(configStream); } - static String getMetainfoMapKey(String roleGroup) { - if (roleGroup == null) { - return DEFAULT_METAINFO_MAP_KEY; - } - int lastIndex = roleGroup.lastIndexOf(COMPONENT_SEPARATOR); - if (lastIndex == -1) { - return DEFAULT_METAINFO_MAP_KEY; - } else { - return roleGroup.substring(0, lastIndex+1); - } - } - - static String getMetainfoComponentName(String roleGroup) { - int lastIndex = roleGroup.lastIndexOf(COMPONENT_SEPARATOR); - if (lastIndex == -1) { + static String getMetainfoComponentName(String roleGroup, + ConfTreeOperations appConf) throws BadConfigException { + String prefix = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, null); + if (prefix == null) { return roleGroup; - } else { - return roleGroup.substring(lastIndex+1); } + if (!roleGroup.startsWith(prefix)) { + throw new BadConfigException("Component " + roleGroup + " doesn't start" + + " with prefix " + prefix); + } + return roleGroup.substring(prefix.length()); } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java index e2c879e..4abac7a 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java @@ -18,6 +18,8 @@ package org.apache.slider.providers.agent; +import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.core.conf.ConfTreeOperations; import org.apache.slider.providers.agent.application.metadata.CommandOrder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,9 +27,12 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import static org.apache.slider.api.RoleKeys.ROLE_PREFIX; + /** * Stores the command dependency order for all components in a service. <commandOrder> * <command>SUPERVISOR-START</command> <requires>NIMBUS-STARTED</requires> </commandOrder> Means, SUPERVISOR START @@ -39,19 +44,36 @@ public class ComponentCommandOrder { private static char SPLIT_CHAR = '-'; Map<Command, Map<String, List<ComponentState>>> dependencies = new HashMap<Command, Map<String, List<ComponentState>>>(); + Map<String, Collection<String>> prefixRoleMap = new HashMap<>(); + Map<String, String> rolePrefixMap = new HashMap<>(); public ComponentCommandOrder() {} - public ComponentCommandOrder(List<CommandOrder> commandOrders) { - mergeCommandOrders(commandOrders); + public ComponentCommandOrder(List<CommandOrder> commandOrders, + ConfTreeOperations resources) { + mergeCommandOrders(commandOrders, resources); } - void mergeCommandOrders(List<CommandOrder> commandOrders) { + void mergeCommandOrders(List<CommandOrder> commandOrders, + ConfTreeOperations resources) { + for (String component : resources.getComponentNames()) { + String prefix = SliderUtils.trimPrefix( + resources.getComponentOpt(component, ROLE_PREFIX, null)); + if (prefix != null) { + rolePrefixMap.put(component, prefix); + if (!prefixRoleMap.containsKey(prefix)) { + prefixRoleMap.put(prefix, new HashSet<String>()); + } + prefixRoleMap.get(prefix).add(component); + } + } if (commandOrders != null && commandOrders.size() > 0) { for (CommandOrder commandOrder : commandOrders) { - ComponentCommand componentCmd = getComponentCommand(commandOrder.getCommand()); + ComponentCommand componentCmd = getComponentCommand( + commandOrder.getCommand(), resources); String requires = commandOrder.getRequires(); - List<ComponentState> requiredStates = parseRequiredStates(requires); + List<ComponentState> requiredStates = parseRequiredStates(requires, + resources); if (requiredStates.size() > 0) { Map<String, List<ComponentState>> compDep = dependencies.get(componentCmd.command); if (compDep == null) { @@ -71,7 +93,8 @@ public class ComponentCommandOrder { } } - private List<ComponentState> parseRequiredStates(String requires) { + private List<ComponentState> parseRequiredStates(String requires, + ConfTreeOperations resources) { if (requires == null || requires.length() < 2) { throw new IllegalArgumentException("Input cannot be null and must contain component and state."); } @@ -79,13 +102,14 @@ public class ComponentCommandOrder { String[] componentStates = requires.split(","); List<ComponentState> retList = new ArrayList<ComponentState>(); for (String componentStateStr : componentStates) { - retList.add(getComponentState(componentStateStr)); + retList.add(getComponentState(componentStateStr, resources)); } return retList; } - private ComponentCommand getComponentCommand(String compCmdStr) { + private ComponentCommand getComponentCommand(String compCmdStr, + ConfTreeOperations resources) { if (compCmdStr == null || compCmdStr.trim().length() < 2) { throw new IllegalArgumentException("Input cannot be null and must contain component and command."); } @@ -98,6 +122,11 @@ public class ComponentCommandOrder { String compStr = compCmdStr.substring(0, splitIndex); String cmdStr = compCmdStr.substring(splitIndex + 1); + if (resources.getComponent(compStr) == null && !prefixRoleMap.containsKey(compStr)) { + throw new IllegalArgumentException("Component " + compStr + " specified" + + " in command order does not exist"); + } + Command cmd = Command.valueOf(cmdStr); if (cmd != Command.START) { @@ -106,7 +135,8 @@ public class ComponentCommandOrder { return new ComponentCommand(compStr, cmd); } - private ComponentState getComponentState(String compStStr) { + private ComponentState getComponentState(String compStStr, + ConfTreeOperations resources) { if (compStStr == null || compStStr.trim().length() < 2) { throw new IllegalArgumentException("Input cannot be null."); } @@ -119,6 +149,11 @@ public class ComponentCommandOrder { String compStr = compStStr.substring(0, splitIndex); String stateStr = compStStr.substring(splitIndex + 1); + if (resources.getComponent(compStr) == null && !prefixRoleMap.containsKey(compStr)) { + throw new IllegalArgumentException("Component " + compStr + " specified" + + " in command order does not exist"); + } + State state = State.valueOf(stateStr); if (state != State.STARTED && state != State.INSTALLED) { throw new IllegalArgumentException("Dependency order can only be specified against STARTED/INSTALLED."); @@ -129,40 +164,43 @@ public class ComponentCommandOrder { // dependency is still on component level, but not package level // so use component name to check dependency, not component-package public boolean canExecute(String component, Command command, Collection<ComponentInstanceState> currentStates) { - boolean canExecute = true; - if (dependencies.containsKey(command) && dependencies.get(command).containsKey(component)) { - List<ComponentState> required = dependencies.get(command).get(component); - for (ComponentState stateToMatch : required) { - for (ComponentInstanceState currState : currentStates) { - log.debug("Checking schedule {} {} against dependency {} is {}", - component, command, currState.getComponentName(), currState.getState()); - if (currState.getComponentName().equals(stateToMatch.componentName)) { - if (currState.getState() != stateToMatch.state) { - if (stateToMatch.state == State.STARTED) { + if (!dependencies.containsKey(command)) { + return true; + } + List<ComponentState> required = new ArrayList<>(); + if (dependencies.get(command).containsKey(component)) { + required.addAll(dependencies.get(command).get(component)); + } + String prefix = rolePrefixMap.get(component); + if (prefix != null && dependencies.get(command).containsKey(prefix)) { + required.addAll(dependencies.get(command).get(prefix)); + } + + for (ComponentState stateToMatch : required) { + for (ComponentInstanceState currState : currentStates) { + log.debug("Checking schedule {} {} against dependency {} is {}", + component, command, currState.getComponentName(), currState.getState()); + if (currState.getComponentName().equals(stateToMatch.componentName) || + (prefixRoleMap.containsKey(stateToMatch.componentName) && + prefixRoleMap.get(stateToMatch.componentName).contains(currState.getComponentName()))) { + if (currState.getState() != stateToMatch.state) { + if (stateToMatch.state == State.STARTED) { + log.info("Cannot schedule {} {} as dependency {} is {}", + component, command, currState.getComponentName(), currState.getState()); + return false; + } else { + //state is INSTALLED + if (currState.getState() != State.STARTING && currState.getState() != State.STARTED) { log.info("Cannot schedule {} {} as dependency {} is {}", - component, command, currState.getComponentName(), currState.getState()); - canExecute = false; - } else { - //state is INSTALLED - if (currState.getState() != State.STARTING && currState.getState() != State.STARTED) { - log.info("Cannot schedule {} {} as dependency {} is {}", - component, command, currState.getComponentName(), currState.getState()); - canExecute = false; - } + component, command, currState.getComponentName(), currState.getState()); + return false; } } } - if (!canExecute) { - break; - } - } - if (!canExecute) { - break; } } } - - return canExecute; + return true; } static class ComponentState { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index 3213d93..3ba766f 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -1808,6 +1808,11 @@ public class AppState { SliderUtils.mergeMapsIgnoreDuplicateKeys(cd.getRole(rolename), groupOptions.options); } + String prefix = instanceDefinition.getAppConfOperations() + .getComponentOpt(role.getGroup(), ROLE_PREFIX, null); + if (SliderUtils.isSet(prefix)) { + cd.setRoleOpt(rolename, ROLE_PREFIX, SliderUtils.trimPrefix(prefix)); + } List<String> instances = instanceMap.get(rolename); int nodeCount = instances != null ? instances.size(): 0; cd.setRoleOpt(rolename, COMPONENT_INSTANCES, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component_nested.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component_nested.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component_nested.json new file mode 100644 index 0000000..f67f83a --- /dev/null +++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component_nested.json @@ -0,0 +1,12 @@ +{ + "schema": "http://example.org/specification/v2.0.0";, + "metadata": { + }, + "global": { + }, + "components": { + "test-external-component": { + "site.global.component_type": "external" + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo.json index 073d1ff..bfe2afb 100644 --- a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo.json +++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo.json @@ -2,6 +2,12 @@ "schemaVersion": "2.1", "application": { "name": "SLEEPER", + "commandOrders": [ + { + "command": "SLEEP_100-START", + "requires": "SLEEP_LONG-STARTED" + } + ], "components": [ { "name": "SLEEP_100", http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo_external_component_nested.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo_external_component_nested.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo_external_component_nested.json new file mode 100644 index 0000000..43a2200 --- /dev/null +++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo_external_component_nested.json @@ -0,0 +1,14 @@ +{ + "schemaVersion": "2.1", + "application": { + "name": "SLEEPER", + "commandOrders": [ + { + "command": "test-external-component-test_sleep-SLEEP_100-START", + "requires": "test-external-component-SLEEP_100-STARTED" + } + ], + "components": [ + ] + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources_external_component_nested.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources_external_component_nested.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources_external_component_nested.json new file mode 100644 index 0000000..8aa86b4 --- /dev/null +++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources_external_component_nested.json @@ -0,0 +1,12 @@ +{ + "schema" : "http://example.org/specification/v2.0.0";, + "metadata" : { + }, + "global" : { + }, + "components": { + "slider-appmaster": { + "yarn.memory": "384" + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildExternalComponent.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildExternalComponent.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildExternalComponent.groovy new file mode 100644 index 0000000..8dd693e --- /dev/null +++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildExternalComponent.groovy @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.agent.standalone + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import org.apache.hadoop.fs.Path +import org.apache.slider.agent.AgentMiniClusterTestBase +import org.apache.slider.api.ResourceKeys +import org.apache.slider.api.RoleKeys +import org.apache.slider.client.SliderClient +import org.apache.slider.common.SliderKeys +import org.apache.slider.common.params.SliderActions +import org.apache.slider.common.tools.SliderFileSystem +import org.apache.slider.common.tools.SliderUtils +import org.apache.slider.core.conf.AggregateConf +import org.apache.slider.core.main.ServiceLauncher +import org.apache.slider.providers.agent.AgentKeys +import org.junit.Test + +import static org.apache.slider.common.params.Arguments.* + +@CompileStatic +@Slf4j + +class TestBuildExternalComponent extends AgentMiniClusterTestBase { + + @Test + public void testExternalComponentBuild() throws Throwable { + String clustername = createMiniCluster("", configuration, 1, true) + + describe "verify external components" + + String echo = "echo" + ServiceLauncher<SliderClient> launcher = createOrBuildCluster( + SliderActions.ACTION_BUILD, + clustername, + [(echo): 1], + [ARG_RES_COMP_OPT, echo, ResourceKeys.COMPONENT_PRIORITY, "2"], + true, + false, + agentDefOptions) + SliderClient sliderClient = launcher.service + addToTeardown(sliderClient); + + // verify the cluster exists + assert 0 == sliderClient.actionExists(clustername, false) + + String parent1 = clustername + "_ext" + launcher = createOrBuildCluster( + SliderActions.ACTION_BUILD, + parent1, + [(echo): 1], + [ARG_COMP_OPT, clustername, COMPONENT_TYPE, EXTERNAL_COMPONENT, + ARG_RES_COMP_OPT, echo, ResourceKeys.COMPONENT_PRIORITY, "3"], + true, + false, + agentDefOptions) + sliderClient = launcher.service + addToTeardown(sliderClient); + + // verify the cluster exists + assert 0 == sliderClient.actionExists(parent1, false) + // verify generated conf + def aggregateConf = sliderClient.loadPersistedClusterDescription(parent1) + assert 3 == aggregateConf.resourceOperations.componentNames.size() + assert aggregateConf.resourceOperations.componentNames.contains(COMPONENT_AM) + assert aggregateConf.resourceOperations.componentNames.contains(echo) + assert aggregateConf.resourceOperations.componentNames.contains(clustername + COMPONENT_SEPARATOR + echo) + + aggregateConf.resolve() + assert aggregateConf.appConfOperations.get(AgentKeys.APP_DEF).equals( + aggregateConf.appConfOperations.getComponentOpt(echo, AgentKeys.APP_DEF, + aggregateConf.appConfOperations.get(AgentKeys.APP_DEF))) + SliderFileSystem sfs = createSliderFileSystem() + String appdefdir = sfs.buildAppDefDirPath(parent1) + checkComponent(aggregateConf, clustername + COMPONENT_SEPARATOR + echo, appdefdir) + + String parent2 = "parent" + launcher = createOrBuildCluster( + SliderActions.ACTION_BUILD, + parent2, + [(echo): 1], + [ARG_COMP_OPT, clustername, COMPONENT_TYPE, EXTERNAL_COMPONENT, + ARG_COMP_OPT, parent1, COMPONENT_TYPE, EXTERNAL_COMPONENT, + ARG_RES_COMP_OPT, echo, ResourceKeys.COMPONENT_PRIORITY, "4"], + true, + false, + agentDefOptions) + sliderClient = launcher.service + addToTeardown(sliderClient); + + // verify the cluster exists + assert 0 == sliderClient.actionExists(parent2, false) + // verify generated conf + aggregateConf = sliderClient.loadPersistedClusterDescription(parent2) + assert 5 == aggregateConf.resourceOperations.componentNames.size() + assert aggregateConf.resourceOperations.componentNames.contains(COMPONENT_AM) + assert aggregateConf.resourceOperations.componentNames.contains(echo) + assert aggregateConf.resourceOperations.componentNames.contains(clustername + COMPONENT_SEPARATOR + echo) + assert aggregateConf.resourceOperations.componentNames.contains(parent1 + COMPONENT_SEPARATOR + echo) + assert aggregateConf.resourceOperations.componentNames.contains(parent1 + COMPONENT_SEPARATOR + clustername + COMPONENT_SEPARATOR + echo) + + aggregateConf.resolve() + assert aggregateConf.appConfOperations.get(AgentKeys.APP_DEF).equals( + aggregateConf.appConfOperations.getComponentOpt(echo, AgentKeys.APP_DEF, + aggregateConf.appConfOperations.get(AgentKeys.APP_DEF))) + appdefdir = sfs.buildAppDefDirPath(parent2) + checkComponent(aggregateConf, clustername + COMPONENT_SEPARATOR + echo, appdefdir) + checkComponent(aggregateConf, parent1 + COMPONENT_SEPARATOR + echo, appdefdir) + checkComponent(aggregateConf, parent1 + COMPONENT_SEPARATOR + clustername + COMPONENT_SEPARATOR + echo, appdefdir) + } + + private void checkComponent(AggregateConf aggConf, String component, + String appdefdir) { + String path = new Path(appdefdir, SliderUtils.trimPrefix( + aggConf.appConfOperations.getComponentOpt(component, RoleKeys + .ROLE_PREFIX, null)) + "_" + SliderKeys.DEFAULT_APP_PKG).toString() + assert path.equals(aggConf.appConfOperations.getComponentOpt(component, + AgentKeys.APP_DEF, null)) + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java index 07d21d7..a6d52b4 100644 --- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java +++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java @@ -152,10 +152,6 @@ public class TestAgentProviderService { + " <command>HBASE_REGIONSERVER-START</command>\n" + " <requires>HBASE_MASTER-STARTED</requires>\n" + " </commandOrder>\n" - + " <commandOrder>\n" - + " <command>A-START</command>\n" - + " <requires>B-STARTED</requires>\n" - + " </commandOrder>\n" + " </commandOrders>\n" + " <components>\n" + " <component>\n" @@ -434,6 +430,11 @@ public class TestAgentProviderService { .put(AgentKeys.AGENT_CONF, "."); instanceDefinition.getAppConfOperations().getGlobalOptions() .put(AgentKeys.AGENT_VERSION, "."); + + instanceDefinition.getResourceOperations().getOrAddComponent( + "HBASE_MASTER"); + instanceDefinition.getResourceOperations().getOrAddComponent( + "HBASE_REGIONSERVER"); return instanceDefinition; } @@ -530,13 +531,7 @@ public class TestAgentProviderService { assertNotNull(registryViewForProviders); ContainerLaunchContext ctx = createNiceMock(ContainerLaunchContext.class); - AggregateConf instanceDefinition = new AggregateConf(); - - instanceDefinition.setInternal(tree); - instanceDefinition.setAppConf(tree); - instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.APP_DEF, "."); - instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.AGENT_CONF, "."); - instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.AGENT_VERSION, "."); + AggregateConf instanceDefinition = prepareConfForAgentStateTests(); Container container = createNiceMock(Container.class); ProviderRole role_hm = new ProviderRole("HBASE_MASTER", 1); @@ -1116,19 +1111,15 @@ public class TestAgentProviderService { Assert.assertEquals(found, 2); List<CommandOrder> cmdOrders = application.getCommandOrders(); - Assert.assertEquals(cmdOrders.size(), 2); + Assert.assertEquals(cmdOrders.size(), 1); found = 0; for (CommandOrder co : application.getCommandOrders()) { if (co.getCommand().equals("HBASE_REGIONSERVER-START")) { Assert.assertTrue(co.getRequires().equals("HBASE_MASTER-STARTED")); found++; } - if (co.getCommand().equals("A-START")) { - Assert.assertEquals(co.getRequires(), "B-STARTED"); - found++; - } } - Assert.assertEquals(found, 2); + Assert.assertEquals(found, 1); List<ConfigFile> configFiles = application.getConfigFiles(); Assert.assertEquals(configFiles.size(), 2); @@ -1204,8 +1195,6 @@ public class TestAgentProviderService { // Start of HBASE_RS depends on the start of HBASE_MASTER InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes()); Metainfo metainfo = new MetainfoParser().fromXmlStream(metainfo_1); - ConfTree tree = new ConfTree(); - tree.global.put(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, "."); Configuration conf = new Configuration(); AgentProviderService aps = createAgentProviderService(conf); @@ -1213,13 +1202,7 @@ public class TestAgentProviderService { assertNotNull(registryViewForProviders); ContainerLaunchContext ctx = createNiceMock(ContainerLaunchContext.class); - AggregateConf instanceDefinition = new AggregateConf(); - - instanceDefinition.setInternal(tree); - instanceDefinition.setAppConf(tree); - instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.APP_DEF, "."); - instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.AGENT_CONF, "."); - instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.AGENT_VERSION, "."); + AggregateConf instanceDefinition = prepareConfForAgentStateTests(); Container container = createNiceMock(Container.class); ProviderRole role_hm = new ProviderRole("HBASE_MASTER", 1); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentCommandOrder.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentCommandOrder.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentCommandOrder.java index c123fbb..0e3e3ad 100644 --- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentCommandOrder.java +++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentCommandOrder.java @@ -18,20 +18,38 @@ package org.apache.slider.providers.agent; +import org.apache.slider.core.conf.ConfTree; +import org.apache.slider.core.conf.ConfTreeOperations; import org.apache.slider.providers.agent.application.metadata.CommandOrder; import org.apache.slider.server.appmaster.model.mock.MockContainerId; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Arrays; +import java.util.List; + +import static org.apache.slider.api.RoleKeys.ROLE_PREFIX; public class TestComponentCommandOrder { protected static final Logger log = LoggerFactory.getLogger(TestComponentCommandOrder.class); private final MockContainerId containerId = new MockContainerId(1); + private static ConfTreeOperations resources = new ConfTreeOperations( + new ConfTree()); + + @BeforeClass + public static void init() { + resources.getOrAddComponent("A"); + resources.getOrAddComponent("B"); + resources.getOrAddComponent("C"); + resources.getOrAddComponent("D"); + resources.getOrAddComponent("E"); + } + @Test public void testComponentCommandOrder() throws Exception { CommandOrder co1 = new CommandOrder(); @@ -44,7 +62,8 @@ public class TestComponentCommandOrder { co3.setCommand("B-START"); co3.setRequires("C-STARTED,D-STARTED,E-INSTALLED"); - ComponentCommandOrder cco = new ComponentCommandOrder(Arrays.asList(co1, co2, co3)); + ComponentCommandOrder cco = new ComponentCommandOrder( + Arrays.asList(co1, co2, co3), resources); ComponentInstanceState cisB = new ComponentInstanceState("B", containerId, "aid"); ComponentInstanceState cisC = new ComponentInstanceState("C", containerId, "aid"); @@ -100,13 +119,14 @@ public class TestComponentCommandOrder { cisB.setState(State.STARTED); cisC.setState(State.STARTED); - ComponentCommandOrder cco = new ComponentCommandOrder(Arrays.asList(co)); + ComponentCommandOrder cco = new ComponentCommandOrder(Arrays.asList(co), + resources); Assert.assertTrue(cco.canExecute("A", Command.START, Arrays.asList(cisB, cisC))); co.setCommand(" A-STAR"); co.setRequires("B-STARTED , C-STARTED"); try { - cco = new ComponentCommandOrder(Arrays.asList(co)); + cco = new ComponentCommandOrder(Arrays.asList(co), resources); Assert.fail("Instantiation should have failed."); } catch (IllegalArgumentException ie) { log.info(ie.getMessage()); @@ -115,7 +135,7 @@ public class TestComponentCommandOrder { co.setCommand(" -START"); co.setRequires("B-STARTED , C-STARTED"); try { - cco = new ComponentCommandOrder(Arrays.asList(co)); + cco = new ComponentCommandOrder(Arrays.asList(co), resources); Assert.fail("Instantiation should have failed."); } catch (IllegalArgumentException ie) { log.info(ie.getMessage()); @@ -124,7 +144,7 @@ public class TestComponentCommandOrder { co.setCommand(" A-START"); co.setRequires("B-STRTED , C-STARTED"); try { - cco = new ComponentCommandOrder(Arrays.asList(co)); + cco = new ComponentCommandOrder(Arrays.asList(co), resources); Assert.fail("Instantiation should have failed."); } catch (IllegalArgumentException ie) { log.info(ie.getMessage()); @@ -133,7 +153,7 @@ public class TestComponentCommandOrder { co.setCommand(" A-START"); co.setRequires("B-STARTED , C-"); try { - cco = new ComponentCommandOrder(Arrays.asList(co)); + cco = new ComponentCommandOrder(Arrays.asList(co), resources); Assert.fail("Instantiation should have failed."); } catch (IllegalArgumentException ie) { log.info(ie.getMessage()); @@ -142,10 +162,83 @@ public class TestComponentCommandOrder { co.setCommand(" A-INSTALL"); co.setRequires("B-STARTED"); try { - cco = new ComponentCommandOrder(Arrays.asList(co)); + cco = new ComponentCommandOrder(Arrays.asList(co), resources); + Assert.fail("Instantiation should have failed."); + } catch (IllegalArgumentException ie) { + log.info(ie.getMessage()); + } + } + + @Test + public void testComponentCommandOrderBadComponent() throws Exception { + ConfTreeOperations resourcesGood = new ConfTreeOperations(new ConfTree()); + resourcesGood.getOrAddComponent("A"); + resourcesGood.getOrAddComponent("Z"); + ConfTreeOperations resourcesBad = new ConfTreeOperations(new ConfTree()); + + CommandOrder co1 = new CommandOrder(); + co1.setCommand("A-START"); + co1.setRequires("Z-STARTED"); + CommandOrder co2 = new CommandOrder(); + co2.setCommand("Z-START"); + co2.setRequires("A-STARTED"); + + ComponentCommandOrder cco = new ComponentCommandOrder( + Arrays.asList(co1), resourcesGood); + try { + cco = new ComponentCommandOrder(Arrays.asList(co1), resourcesBad); + Assert.fail("Instantiation should have failed."); + } catch (IllegalArgumentException ie) { + log.info(ie.getMessage()); + } + + cco = new ComponentCommandOrder(Arrays.asList(co2), resourcesGood); + try { + cco = new ComponentCommandOrder(Arrays.asList(co2), resourcesBad); Assert.fail("Instantiation should have failed."); } catch (IllegalArgumentException ie) { log.info(ie.getMessage()); } } + + @Test + public void testComponentCommandOrderPrefixes() throws Exception { + ConfTreeOperations resources = new ConfTreeOperations(new ConfTree()); + resources.getOrAddComponent("a-A").put(ROLE_PREFIX, "a-"); + resources.getOrAddComponent("b-B1").put(ROLE_PREFIX, "b-"); + resources.getOrAddComponent("b-B2").put(ROLE_PREFIX, "b-"); + resources.getOrAddComponent("c-C").put(ROLE_PREFIX, "c-"); + + CommandOrder co1 = new CommandOrder(); + co1.setCommand("b-START"); + co1.setRequires("a-STARTED"); + CommandOrder co2 = new CommandOrder(); + co2.setCommand("c-START"); + co2.setRequires("b-STARTED"); + + ComponentCommandOrder cco = new ComponentCommandOrder( + Arrays.asList(co1, co2), resources); + + ComponentInstanceState cisA = new ComponentInstanceState("a-A", containerId, "aid"); + ComponentInstanceState cisB1 = new ComponentInstanceState("b-B1", containerId, "aid"); + ComponentInstanceState cisB2 = new ComponentInstanceState("b-B2", containerId, "aid"); + ComponentInstanceState cisC = new ComponentInstanceState("c-C", containerId, "aid"); + cisA.setState(State.INSTALLED); + cisB1.setState(State.INSTALLED); + cisB2.setState(State.INSTALLED); + cisC.setState(State.INSTALLED); + List<ComponentInstanceState> states = Arrays.asList(cisA, cisB1, cisB2, cisC); + Assert.assertTrue(cco.canExecute("a-A", Command.START, states)); + Assert.assertFalse(cco.canExecute("b-B1", Command.START, states)); + Assert.assertFalse(cco.canExecute("b-B2", Command.START, states)); + Assert.assertFalse(cco.canExecute("c-C", Command.START, states)); + cisA.setState(State.STARTED); + Assert.assertTrue(cco.canExecute("b-B1", Command.START, states)); + Assert.assertTrue(cco.canExecute("b-B2", Command.START, states)); + Assert.assertFalse(cco.canExecute("c-C", Command.START, states)); + cisB1.setState(State.STARTED); + Assert.assertFalse(cco.canExecute("c-C", Command.START, states)); + cisB2.setState(State.STARTED); + Assert.assertTrue(cco.canExecute("c-C", Command.START, states)); + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy index 1cb6f0f..13919df 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy @@ -40,4 +40,8 @@ interface ResourcePaths { String EXTERNAL_RESOURCES = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/resources_external_component.json" String EXTERNAL_APPCONFIG = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/appConfig_external_component.json" + + String NESTED_RESOURCES = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/resources_external_component_nested.json" + String NESTED_META = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/metainfo_external_component_nested.json" + String NESTED_APPCONFIG = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/appConfig_external_component_nested.json" } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/ExternalComponentIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/ExternalComponentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/ExternalComponentIT.groovy index b5e0270..292508a 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/ExternalComponentIT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/ExternalComponentIT.groovy @@ -40,6 +40,7 @@ public class ExternalComponentIT extends AgentCommandTestBase static String NAME = "test-external-component" static String EXT_NAME = "test_sleep" + static String NESTED_NAME = "test-external-component-nested" static String BUILD_APPCONFIG = ResourcePaths.SLEEP_APPCONFIG static String BUILD_RESOURCES = ResourcePaths.EXTERNAL_RESOURCES @@ -47,23 +48,30 @@ public class ExternalComponentIT extends AgentCommandTestBase static String TEST_APPCONFIG = ResourcePaths.EXTERNAL_APPCONFIG static String TEST_RESOURCES = ResourcePaths.EXTERNAL_RESOURCES static String TEST_METAINFO = ResourcePaths.SLEEP_META + static String NEST_APPCONFIG = ResourcePaths.NESTED_APPCONFIG + static String NEST_RESOURCES = ResourcePaths.NESTED_RESOURCES + static String NEST_METAINFO = ResourcePaths.NESTED_META public static final String SLEEP_100 = "SLEEP_100" public static final String SLEEP_LONG = "SLEEP_LONG" public static final String EXT_SLEEP_100 = EXT_NAME + SliderKeys.COMPONENT_SEPARATOR + SLEEP_100 public static final String EXT_SLEEP_LONG = EXT_NAME + SliderKeys.COMPONENT_SEPARATOR + SLEEP_LONG + public static final String NESTED_PREFIX = NAME + + SliderKeys.COMPONENT_SEPARATOR @Before public void prepareCluster() { setupCluster(NAME) setupCluster(EXT_NAME) + setupCluster(NESTED_NAME) } @After public void destroyCluster() { cleanup(NAME) cleanup(EXT_NAME) + cleanup(NESTED_NAME) } @Test @@ -129,5 +137,35 @@ public class ExternalComponentIT extends AgentCommandTestBase expectLiveContainerCountReached(NAME, EXT_SLEEP_100, 0, CONTAINER_LAUNCH_TIMEOUT) + cleanup(NAME) + + describe NESTED_NAME + + slider(0, [ACTION_BUILD, NAME, ARG_METAINFO, TEST_METAINFO, + ARG_TEMPLATE, TEST_APPCONFIG, ARG_RESOURCES, TEST_RESOURCES]) + + slider(0, [ACTION_CREATE, NESTED_NAME, ARG_METAINFO, NEST_METAINFO, + ARG_TEMPLATE, NEST_APPCONFIG, ARG_RESOURCES, NEST_RESOURCES]) + + ensureApplicationIsUp(NESTED_NAME) + status(0, NESTED_NAME) + + cd = execStatus(NESTED_NAME) + + assert 5 == cd.statistics.size() + assert cd.statistics.keySet().containsAll([SliderKeys.COMPONENT_AM, + NESTED_PREFIX + SLEEP_100, + NESTED_PREFIX + SLEEP_LONG, + NESTED_PREFIX + EXT_SLEEP_100, + NESTED_PREFIX + EXT_SLEEP_LONG]) + + expectLiveContainerCountReached(NESTED_NAME, NESTED_PREFIX + SLEEP_LONG, 1, + CONTAINER_LAUNCH_TIMEOUT) + expectLiveContainerCountReached(NESTED_NAME, NESTED_PREFIX + EXT_SLEEP_LONG, 1, + CONTAINER_LAUNCH_TIMEOUT) + expectLiveContainerCountReached(NESTED_NAME, NESTED_PREFIX + SLEEP_100, 0, + CONTAINER_LAUNCH_TIMEOUT) + expectLiveContainerCountReached(NESTED_NAME, NESTED_PREFIX + EXT_SLEEP_100, 0, + CONTAINER_LAUNCH_TIMEOUT) } }
