SLIDER-875 initial uber app support
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/17096ffd Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/17096ffd Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/17096ffd Branch: refs/heads/feature/SLIDER-1107_AM_config_generation Commit: 17096ffd0562279aaf082004e2f774711b37c56f Parents: 3170e62 Author: Billie Rinaldi <billie.rina...@gmail.com> Authored: Tue Jun 21 16:59:34 2016 -0700 Committer: Billie Rinaldi <billie.rina...@gmail.com> Committed: Thu Jun 23 08:30:35 2016 -0700 ---------------------------------------------------------------------- .../org/apache/slider/client/SliderClient.java | 4 +- .../org/apache/slider/common/SliderKeys.java | 7 + .../apache/slider/common/tools/SliderUtils.java | 79 ++++++++ .../slider/core/build/InstanceBuilder.java | 148 +++++++++++++++ .../slider/core/conf/ConfTreeOperations.java | 11 ++ .../providers/AbstractProviderService.java | 3 +- .../slider/providers/ProviderService.java | 4 +- .../providers/agent/AgentClientProvider.java | 30 +-- .../slider/providers/agent/AgentKeys.java | 2 + .../providers/agent/AgentProviderService.java | 181 +++++++++++++------ .../slider/providers/agent/AgentUtils.java | 24 +++ .../providers/agent/ComponentCommandOrder.java | 6 + .../server/appmaster/SliderAppMaster.java | 3 +- .../appmaster/web/rest/agent/AgentResource.java | 4 +- .../sleep_cmd/appConfig_external_component.json | 12 ++ .../sleep_cmd/resources_external_component.json | 22 +++ .../slider/client/TestReplaceTokens.groovy | 5 +- .../model/mock/MockProviderService.groovy | 3 +- .../agent/TestAgentProviderService.java | 36 ++-- .../apache/slider/funtest/ResourcePaths.groovy | 2 + .../funtest/misc/ExternalComponentIT.groovy | 133 ++++++++++++++ 21 files changed, 626 insertions(+), 93 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/17096ffd/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index dd90e46..372ec48 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -1597,6 +1597,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe resources.mergeComponents(buildInfo.getResourceCompOptionMap()); builder.init(providerName, instanceDefinition); + builder.resolve(); builder.propagateFilename(); builder.propagatePrincipals(); builder.setImageDetailsIfAvailable(buildInfo.getImage(), @@ -1786,8 +1787,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe private static String replaceTokens(String s, String userName, String clusterName) throws IOException { return s.replaceAll(Pattern.quote("${USER}"), userName) - .replaceAll(Pattern.quote("${USER_NAME}"), userName) - .replaceAll(Pattern.quote("${CLUSTER_NAME}"), clusterName); + .replaceAll(Pattern.quote("${USER_NAME}"), userName); } public FsPermission getClusterDirectoryPermissions(Configuration conf) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/17096ffd/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java index 05c7048..3921324 100644 --- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java +++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java @@ -70,6 +70,13 @@ public interface SliderKeys extends SliderXmlConfKeys { String APP_TYPE = "org-apache-slider"; /** + * Key for component type. This MUST NOT be set in app_config/global {@value} + */ + String COMPONENT_TYPE = "site.global.component_type"; + String EXTERNAL_COMPONENT = "external"; + String COMPONENT_SEPARATOR = "-"; + + /** * Key for application version. This must be set in app_config/global {@value} */ String APP_VERSION = "site.global.app_version"; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/17096ffd/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index 746e468..c48e109 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -69,6 +69,7 @@ import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.core.launch.ClasspathConstructor; import org.apache.slider.core.main.LauncherExitCodes; import org.apache.slider.providers.agent.AgentKeys; +import org.apache.slider.providers.agent.application.metadata.Component; import org.apache.slider.server.services.utility.PatternValidator; import org.apache.slider.server.services.workflow.ForkedProcessService; import org.apache.zookeeper.server.util.KerberosUtil; @@ -475,6 +476,32 @@ public final class SliderUtils { return srcFileCount; } + /** + * Copy a file to a new FS -both paths must be qualified. + * @param conf conf file + * @param srcFile src file + * @param destFile dest file + */ + public static void copy(Configuration conf, + Path srcFile, + Path destFile) throws + IOException, + BadClusterStateException { + FileSystem srcFS = FileSystem.get(srcFile.toUri(), conf); + //list all paths in the src. + if (!srcFS.exists(srcFile)) { + throw new FileNotFoundException("Source dir not found " + srcFile); + } + if (!srcFS.isFile(srcFile)) { + throw new FileNotFoundException( + "Source dir not a file " + srcFile); + } + FileSystem destFS = FileSystem.get(destFile.toUri(), conf); + if (destFS.exists(destFile)) { + throw new IOException("Dest file already exists " + destFile); + } + FileUtil.copy(srcFS, srcFile, destFS, destFile, false, true, conf); + } public static String stringify(Throwable t) { StringWriter sw = new StringWriter(); @@ -927,6 +954,38 @@ public final class SliderUtils { } /** + * Merge string maps excluding prefixes + * @param first first map + * @param second second map + * @param prefixes prefixes to ignore + * @return 'first' merged with the second + */ + public static Map<String, String> mergeMapsIgnorePrefixes( + Map<String, String> first, Map<String, String> second, + String... prefixes) { + Preconditions.checkArgument(first != null, "Null 'first' value"); + Preconditions.checkArgument(second != null, "Null 'second' value"); + Preconditions.checkArgument(prefixes != null, "Null 'prefixes' value"); + for (Map.Entry<String, String> entry : second.entrySet()) { + String key = entry.getKey(); + boolean hasPrefix = false; + for (String prefix : prefixes) { + if (key.startsWith(prefix)) { + hasPrefix = true; + break; + } + } + if (hasPrefix) { + continue; + } + if (!first.containsKey(key)) { + first.put(key, entry.getValue()); + } + } + return first; + } + + /** * Convert a map to a multi-line string for printing * @param map map to stringify * @return a string representation of the map @@ -2340,8 +2399,28 @@ public final class SliderUtils { */ public static String getApplicationDefinitionPath(ConfTreeOperations conf) throws BadConfigException { + return getApplicationDefinitionPath(conf, null); + } + + /** + * return the HDFS path where the application package has been uploaded + * manually or by using slider client (install package command) + * + * @param conf configuration + * @param roleGroup name of component + * @return + */ + public static String getApplicationDefinitionPath(ConfTreeOperations conf, + String roleGroup) + throws BadConfigException { String appDefPath = conf.getGlobalOptions().getMandatoryOption( AgentKeys.APP_DEF); + if (roleGroup != null) { + MapOperations component = conf.getComponent(roleGroup); + if (component != null) { + appDefPath = component.getOption(AgentKeys.APP_DEF, appDefPath); + } + } return appDefPath; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/17096ffd/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java index 7756055..4afdf7c 100644 --- a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java +++ b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java @@ -25,7 +25,9 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.slider.api.InternalKeys; import org.apache.slider.api.OptionKeys; +import org.apache.slider.api.ResourceKeys; import org.apache.slider.api.StatusKeys; +import org.apache.slider.common.SliderKeys; import org.apache.slider.common.SliderXmlConfKeys; import org.apache.slider.common.tools.CoreFileSystem; import org.apache.slider.common.tools.SliderUtils; @@ -42,11 +44,17 @@ import org.apache.slider.core.persist.LockAcquireFailedException; import org.apache.slider.core.persist.LockHeldAction; import org.apache.slider.core.zk.ZKPathBuilder; import org.apache.slider.core.zk.ZookeeperUtils; +import org.apache.slider.providers.agent.AgentKeys; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeSet; import static org.apache.slider.api.InternalKeys.INTERNAL_ADDONS_DIR_PATH; import static org.apache.slider.api.InternalKeys.INTERNAL_APPDEF_DIR_PATH; @@ -61,6 +69,11 @@ import static org.apache.slider.api.OptionKeys.INTERNAL_SNAPSHOT_CONF_PATH; import static org.apache.slider.api.OptionKeys.ZOOKEEPER_HOSTS; import static org.apache.slider.api.OptionKeys.ZOOKEEPER_PATH; import static org.apache.slider.api.OptionKeys.ZOOKEEPER_QUORUM; +import static org.apache.slider.common.SliderKeys.COMPONENT_AM; +import static org.apache.slider.common.SliderKeys.COMPONENT_SEPARATOR; +import static org.apache.slider.common.SliderKeys.COMPONENT_TYPE; +import static org.apache.slider.common.SliderKeys.EXTERNAL_COMPONENT; +import static org.apache.slider.common.tools.SliderUtils.isClusternameValid; /** * Build up the instance of a cluster. @@ -72,6 +85,8 @@ public class InstanceBuilder { private final CoreFileSystem coreFS; private final InstancePaths instancePaths; private AggregateConf instanceDescription; + private Map<Path, Path> externalAppDefs = new HashMap<>(); + private TreeSet<Integer> priorities = new TreeSet<>(); private static final Logger log = LoggerFactory.getLogger(InstanceBuilder.class); @@ -244,6 +259,136 @@ public class InstanceBuilder { } + private void getExternalComponents(ConfTreeOperations ops, + Set<String> externalComponents) throws BadConfigException { + if (ops.getGlobalOptions().get(COMPONENT_TYPE) != null) { + throw new BadConfigException(COMPONENT_TYPE + " must be " + + "specified per-component, not in global"); + } + + for (Entry<String, Map<String, String>> entry : ops.getComponents() + .entrySet()) { + if (COMPONENT_AM.equals(entry.getKey())) { + continue; + } + Map<String, String> options = entry.getValue(); + if (options.containsKey(COMPONENT_TYPE) && + EXTERNAL_COMPONENT.equals(options.get(COMPONENT_TYPE))) { + externalComponents.add(entry.getKey()); + } + } + } + + private static String[] PREFIXES_TO_SKIP = {"zookeeper.", + "env.MALLOC_ARENA_MAX", "site.fs.", "site.dfs."}; + + private void mergeExternalComponent(ConfTreeOperations ops, + ConfTreeOperations externalOps, String externalComponent, + Integer priority) { + for (String subComponent : externalOps.getComponentNames()) { + if (COMPONENT_AM.equals(subComponent)) { + continue; + } + log.debug("Merging options for {} into {}", subComponent, + externalComponent + COMPONENT_SEPARATOR + subComponent); + MapOperations subComponentOps = ops.getOrAddComponent(externalComponent + + COMPONENT_SEPARATOR + subComponent); + SliderUtils.mergeMapsIgnorePrefixes(subComponentOps, + externalOps.getComponent(subComponent), PREFIXES_TO_SKIP); + if (priority != null) { + subComponentOps.put(ResourceKeys.COMPONENT_PRIORITY, + Integer.toString(priority)); + priorities.add(priority); + priority++; + } + } + } + + private int getNextPriority() { + if (priorities.isEmpty()) { + return 1; + } else { + return priorities.last() + 1; + } + } + + public void resolve() + throws BadConfigException, IOException, BadClusterStateException { + ConfTreeOperations appConf = instanceDescription.getAppConfOperations(); + ConfTreeOperations resources = instanceDescription.getResourceOperations(); + + for (Entry<String, Map<String, String>> entry : resources.getComponents() + .entrySet()) { + if (COMPONENT_AM.equals(entry.getKey())) { + continue; + } + if (entry.getKey().contains(COMPONENT_SEPARATOR)) { + throw new BadConfigException("Components must not contain " + + COMPONENT_SEPARATOR + ": " + entry.getKey()); + } + if (entry.getValue().containsKey(ResourceKeys.COMPONENT_PRIORITY)) { + priorities.add(Integer.parseInt(entry.getValue().get( + ResourceKeys.COMPONENT_PRIORITY))); + } + } + + Set<String> externalComponents = new HashSet<>(); + getExternalComponents(appConf, externalComponents); + if (!externalComponents.isEmpty()) { + log.info("Found external components {}", externalComponents); + } + + for (String component : externalComponents) { + if (!isClusternameValid(component)) { + throw new BadConfigException(component + " is not a valid external " + + "component"); + } + Path componentClusterDir = coreFS.buildClusterDirPath(component); + try { + coreFS.verifyPathExists(componentClusterDir); + } catch (IOException e) { + throw new BadConfigException("external component " + component + + " doesn't exist"); + } + AggregateConf componentConf = new AggregateConf(); + ConfPersister persister = new ConfPersister(coreFS, componentClusterDir); + try { + persister.load(componentConf); + } catch (Exception e) { + throw new BadConfigException("Couldn't read configuration for " + + "external component " + component); + } + String externalAppDef = componentConf.getAppConfOperations() + .getGlobalOptions().get(AgentKeys.APP_DEF); + if (SliderUtils.isSet(externalAppDef)) { + Path newAppDef = new Path(coreFS.buildAppDefDirPath(clustername), + component + "_" + SliderKeys.DEFAULT_APP_PKG); + componentConf.getAppConfOperations().set(AgentKeys.APP_DEF, newAppDef); + externalAppDefs.put(newAppDef, new Path(externalAppDef)); + } + for (String rcomp : componentConf.getResourceOperations() + .getComponentNames()) { + if (COMPONENT_AM.equals(rcomp)) { + continue; + } + log.debug("Adding component {} to appConf for {}", rcomp, component); + componentConf.getAppConfOperations().getOrAddComponent(rcomp); + } + SliderUtils.mergeMaps( + componentConf.getAppConfOperations().getGlobalOptions().options, + appConf.getComponent(component).options); + componentConf.getAppConfOperations().getGlobalOptions() + .remove(COMPONENT_TYPE); + componentConf.resolve(); + + mergeExternalComponent(appConf, componentConf.getAppConfOperations(), + component, null); + mergeExternalComponent(resources, componentConf.getResourceOperations(), + component, getNextPriority()); + } + } + + /** * Persist this * @param appconfdir conf dir @@ -266,6 +411,9 @@ public class InstanceBuilder { action = new ConfDirSnapshotAction(appconfdir); } persister.save(instanceDescription, action); + for (Entry<Path, Path> appDef : externalAppDefs.entrySet()) { + SliderUtils.copy(conf, appDef.getValue(), appDef.getKey()); + } } /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/17096ffd/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java index 9013edb..4a0ae41 100644 --- a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java +++ b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java @@ -257,6 +257,17 @@ public class ConfTreeOperations { Map<String, String> map, String prefix, boolean overwrite) { + boolean needsMerge = false; + for (Map.Entry<String, String> entry : map.entrySet()) { + String key = entry.getKey(); + if (key.startsWith(prefix)) { + needsMerge = true; + break; + } + } + if (!needsMerge) { + return; + } MapOperations comp = getOrAddComponent(component); comp.mergeMapPrefixedKeys(map,prefix, overwrite); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/17096ffd/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java index 61b2655..92766f5 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java @@ -169,7 +169,8 @@ public abstract class AbstractProviderService */ @Override public void initializeApplicationConfiguration( - AggregateConf instanceDefinition, SliderFileSystem fileSystem) + AggregateConf instanceDefinition, SliderFileSystem fileSystem, + String roleGroup) throws IOException, SliderException { } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/17096ffd/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java index f754eee..3f24665 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java @@ -118,11 +118,13 @@ public interface ProviderService extends ProviderCore, * * @param instanceDefinition * @param fileSystem + * @param roleGroup * @throws IOException * @throws SliderException */ void initializeApplicationConfiguration(AggregateConf instanceDefinition, - SliderFileSystem fileSystem) throws IOException, SliderException; + SliderFileSystem fileSystem, String roleGroup) throws IOException, + SliderException; /** * This is a validation of the application configuration on the AM. http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/17096ffd/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java index f3dcd1d..6eae75e 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java @@ -74,6 +74,8 @@ import java.util.Set; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; +import static org.apache.slider.providers.agent.AgentUtils.getMetainfoComponentName; + /** This class implements the client-side aspects of the agent deployer */ public class AgentClientProvider extends AbstractClientProvider implements AgentKeys, SliderKeys { @@ -168,13 +170,21 @@ public class AgentClientProvider extends AbstractClientProvider names.remove(SliderKeys.COMPONENT_AM); Map<Integer, String> priorityMap = new HashMap<Integer, String>(); - Metainfo metaInfo = getMetainfo(fs, appDef); - for (String name : names) { + try { + // Validate the app definition + appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition + .getAppConfOperations(), name); + } catch (BadConfigException bce) { + throw new BadConfigException("Application definition must be provided. " + bce.getMessage()); + } + Metainfo metaInfo = getMetainfo(fs, appDef); + MapOperations component = resources.getMandatoryComponent(name); if (metaInfo != null) { - Component componentDef = metaInfo.getApplicationComponent(name); + Component componentDef = metaInfo.getApplicationComponent( + getMetainfoComponentName(name)); if (componentDef == null) { throw new BadConfigException( "Component %s is not a member of application.", name); @@ -200,16 +210,12 @@ public class AgentClientProvider extends AbstractClientProvider existing); } priorityMap.put(priority, name); - } - // fileSystem may be null for tests - if (metaInfo != null) { - for (String name : names) { - Component componentDef = metaInfo.getApplicationComponent(name); - if (componentDef == null) { - throw new BadConfigException( - "Component %s is not a member of application.", name); - } + // fileSystem may be null for tests + if (metaInfo != null) { + Component componentDef = metaInfo.getApplicationComponent( + getMetainfoComponentName(name)); + // already checked it wasn't null // ensure that intance count is 0 for client components if ("CLIENT".equals(componentDef.getCategory())) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/17096ffd/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java index b027939..8514401 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java @@ -97,6 +97,8 @@ public interface AgentKeys { String CERT_FILE_LOCALIZATION_PATH = INFRA_RUN_SECURITY_DIR + "ca.crt"; String KEY_CONTAINER_LAUNCH_DELAY = "container.launch.delay.sec"; String TEST_RELAX_VERIFICATION = "test.relax.validation"; + + String DEFAULT_METAINFO_MAP_KEY = "DEFAULT"; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/17096ffd/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java index f20757a..66fac99 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java @@ -71,6 +71,7 @@ import org.apache.slider.providers.ProviderRole; import org.apache.slider.providers.ProviderUtils; import org.apache.slider.providers.agent.application.metadata.AbstractComponent; import org.apache.slider.providers.agent.application.metadata.Application; +import org.apache.slider.providers.agent.application.metadata.CommandOrder; import org.apache.slider.providers.agent.application.metadata.CommandScript; import org.apache.slider.providers.agent.application.metadata.Component; import org.apache.slider.providers.agent.application.metadata.ComponentCommand; @@ -124,7 +125,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Map.Entry; import java.util.Scanner; import java.util.Set; import java.util.TreeMap; @@ -132,8 +132,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import java.util.regex.Pattern; +import static org.apache.slider.providers.agent.AgentUtils.getMetainfoMapKey; import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_AGENTS; /** @@ -168,9 +168,8 @@ public class AgentProviderService extends AbstractProviderService implements private int heartbeatMonitorInterval = 0; private AgentClientProvider clientProvider; private AtomicInteger taskId = new AtomicInteger(0); - private volatile Metainfo metaInfo = null; - private Map<String, DefaultConfig> defaultConfigs = null; - private ComponentCommandOrder commandOrder = null; + private volatile Map<String, MetainfoHolder> metaInfoMap = new HashMap<>(); + private ComponentCommandOrder commandOrder = new ComponentCommandOrder(); private HeartbeatMonitor monitor; private Boolean canAnyMasterPublish = null; private AgentLaunchParameter agentLaunchParameter = null; @@ -205,6 +204,17 @@ public class AgentProviderService extends AbstractProviderService implements private final Map<String, Set<String>> containerExportsMap = new HashMap<String, Set<String>>(); + private static class MetainfoHolder { + Metainfo metaInfo; + private Map<String, DefaultConfig> defaultConfigs = null; + + public MetainfoHolder(Metainfo metaInfo, + Map<String, DefaultConfig> defaultConfigs) { + this.metaInfo = metaInfo; + this.defaultConfigs = defaultConfigs; + } + } + /** * Create an instance of AgentProviderService */ @@ -249,10 +259,11 @@ public class AgentProviderService extends AbstractProviderService implements Set<String> names = resources.getComponentNames(); names.remove(SliderKeys.COMPONENT_AM); for (String name : names) { - Component componentDef = getMetaInfo().getApplicationComponent(name); + Component componentDef = getApplicationComponent(name); if (componentDef == null) { - throw new BadConfigException( - "Component %s is not a member of application.", name); + // component member is validated elsewhere, so we don't need to throw + // an exception here + continue; } MapOperations componentConfig = resources.getMandatoryComponent(name); @@ -274,31 +285,62 @@ public class AgentProviderService extends AbstractProviderService implements // Reads the metainfo.xml in the application package and loads it private void buildMetainfo(AggregateConf instanceDefinition, - SliderFileSystem fileSystem) throws IOException, SliderException { - String appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition - .getAppConfOperations()); + SliderFileSystem fileSystem, + String roleGroup) + throws IOException, SliderException { + String mapKey = getMetainfoMapKey(roleGroup); + String appDef = SliderUtils.getApplicationDefinitionPath( + instanceDefinition.getAppConfOperations(), roleGroup); + MapOperations component = null; + if (roleGroup != null) { + component = instanceDefinition.getAppConfOperations().getComponent(roleGroup); + } - if (metaInfo == null) { + MetainfoHolder metaInfoHolder = metaInfoMap.get(mapKey); + if (metaInfoHolder == null) { synchronized (syncLock) { - if (metaInfo == null) { + metaInfoHolder = metaInfoMap.get(mapKey); + if (metaInfoHolder == null) { readAndSetHeartbeatMonitoringInterval(instanceDefinition); initializeAgentDebugCommands(instanceDefinition); - metaInfo = getApplicationMetainfo(fileSystem, appDef, false); + Metainfo metaInfo = getApplicationMetainfo(fileSystem, appDef, false); log.info("Master package metainfo: {}", metaInfo.toString()); if (metaInfo == null || metaInfo.getApplication() == null) { log.error("metainfo.xml is unavailable or malformed at {}.", appDef); throw new SliderException( "metainfo.xml is required in app package."); } - commandOrder = new ComponentCommandOrder(metaInfo.getApplication().getCommandOrders()); - defaultConfigs = initializeDefaultConfigs(fileSystem, appDef, metaInfo); + List<CommandOrder> commandOrders = metaInfo.getApplication() + .getCommandOrders(); + if (!DEFAULT_METAINFO_MAP_KEY.equals(mapKey)) { + for (Component comp : metaInfo.getApplication().getComponents()) { + comp.setName(mapKey + comp.getName()); + log.info("Modifying external metainfo component name to {}", + comp.getName()); + } + String commandPrefix = mapKey.substring(0, + mapKey.indexOf(COMPONENT_SEPARATOR)+1); + for (CommandOrder co : commandOrders) { + log.info("Adding prefix {} to command order {}", + commandPrefix, co); + co.setCommand(commandPrefix + co.getCommand()); + co.setRequires(commandPrefix + co.getRequires()); + } + } + commandOrder.mergeCommandOrders(commandOrders); + Map<String, DefaultConfig> defaultConfigs = + initializeDefaultConfigs(fileSystem, appDef, metaInfo); + metaInfoMap.put(mapKey, new MetainfoHolder(metaInfo, defaultConfigs)); monitor = new HeartbeatMonitor(this, getHeartbeatMonitorInterval()); monitor.start(); // build a map from component to metainfo String addonAppDefString = instanceDefinition.getAppConfOperations() .getGlobalOptions().getOption(AgentKeys.ADDONS, null); + if (component != null) { + addonAppDefString = component.getOption(AgentKeys.ADDONS, addonAppDefString); + } log.debug("All addon appdefs: {}", addonAppDefString); if (addonAppDefString != null) { Scanner scanner = new Scanner(addonAppDefString).useDelimiter(","); @@ -306,6 +348,9 @@ public class AgentProviderService extends AbstractProviderService implements String addonAppDef = scanner.next(); String addonAppDefPath = instanceDefinition .getAppConfOperations().getGlobalOptions().get(addonAppDef); + if (component != null) { + addonAppDefPath = component.getOption(addonAppDef, addonAppDefPath); + } log.debug("Addon package {} is stored at: {}", addonAppDef + addonAppDefPath); Metainfo addonMetaInfo = getApplicationMetainfo(fileSystem, @@ -324,9 +369,10 @@ public class AgentProviderService extends AbstractProviderService implements @Override public void initializeApplicationConfiguration( - AggregateConf instanceDefinition, SliderFileSystem fileSystem) + AggregateConf instanceDefinition, SliderFileSystem fileSystem, + String roleGroup) throws IOException, SliderException { - buildMetainfo(instanceDefinition, fileSystem); + buildMetainfo(instanceDefinition, fileSystem, roleGroup); } @Override @@ -345,9 +391,9 @@ public class AgentProviderService extends AbstractProviderService implements String roleName = providerRole.name; String roleGroup = providerRole.group; String appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition - .getAppConfOperations()); + .getAppConfOperations(), roleGroup); - initializeApplicationConfiguration(instanceDefinition, fileSystem); + initializeApplicationConfiguration(instanceDefinition, fileSystem, roleGroup); log.info("Build launch context for Agent"); log.debug(instanceDefinition.toString()); @@ -564,7 +610,7 @@ public class AgentProviderService extends AbstractProviderService implements // initialize the component instance state getComponentStatuses().put(label, new ComponentInstanceState( - roleName, + roleGroup, container.getId(), getClusterInfoPropertyValue(OptionKeys.APPLICATION_NAME), pkgStatuses)); @@ -740,11 +786,12 @@ public class AgentProviderService extends AbstractProviderService implements .extractRole(container)); if (role != null) { String roleName = role.name; - String label = getContainerLabel(container, roleName, role.group); + String roleGroup = role.group; + String label = getContainerLabel(container, roleName, roleGroup); log.info("Rebuilding in-memory: container {} in role {} in cluster {}", container.getId(), roleName, applicationId); getComponentStatuses().put(label, - new ComponentInstanceState(roleName, container.getId(), + new ComponentInstanceState(roleGroup, container.getId(), applicationId)); } else { log.warn("Role not found for container {} in cluster {}", @@ -865,7 +912,7 @@ public class AgentProviderService extends AbstractProviderService implements StateAccessForProviders accessor = getAmState(); CommandScript cmdScript = getScriptPathForMasterPackage(roleGroup); - List<ComponentCommand> commands = getMetaInfo().getApplicationComponent(roleGroup).getCommands(); + List<ComponentCommand> commands = getApplicationComponent(roleGroup).getCommands(); if (!isDockerContainer(roleGroup) && !isYarnDockerContainer(roleGroup) && (cmdScript == null || cmdScript.getScript() == null) @@ -1143,7 +1190,7 @@ public class AgentProviderService extends AbstractProviderService implements } private boolean isDockerContainer(String roleGroup) { - String type = getMetaInfo().getApplicationComponent(roleGroup).getType(); + String type = getApplicationComponent(roleGroup).getType(); if (SliderUtils.isSet(type)) { return type.toLowerCase().equals(SliderUtils.DOCKER) || type.toLowerCase().equals(SliderUtils.DOCKER_YARN); } @@ -1151,7 +1198,7 @@ public class AgentProviderService extends AbstractProviderService implements } private boolean isYarnDockerContainer(String roleGroup) { - String type = getMetaInfo().getApplicationComponent(roleGroup).getType(); + String type = getApplicationComponent(roleGroup).getType(); if (SliderUtils.isSet(type)) { return type.toLowerCase().equals(SliderUtils.DOCKER_YARN); } @@ -1345,8 +1392,12 @@ public class AgentProviderService extends AbstractProviderService implements } @VisibleForTesting - protected Metainfo getMetaInfo() { - return this.metaInfo; + protected Metainfo getMetaInfo(String roleGroup) { + MetainfoHolder mh = this.metaInfoMap.get(getMetainfoMapKey(roleGroup)); + if (mh == null) { + return null; + } + return mh.metaInfo; } @VisibleForTesting @@ -1416,8 +1467,8 @@ public class AgentProviderService extends AbstractProviderService implements return defaultConfigMap; } - protected Map<String, DefaultConfig> getDefaultConfigs() { - return defaultConfigs; + protected Map<String, DefaultConfig> getDefaultConfigs(String roleGroup) { + return metaInfoMap.get(getMetainfoMapKey(roleGroup)).defaultConfigs; } private int getHeartbeatMonitorInterval() { @@ -1583,9 +1634,9 @@ public class AgentProviderService extends AbstractProviderService implements log.info("Status report: {}", status.toString()); if (status.getConfigs() != null) { - Application application = getMetaInfo().getApplication(); + Application application = getMetaInfo(componentGroup).getApplication(); - if (canAnyMasterPublishConfig() == false || canPublishConfig(componentGroup)) { + if (canAnyMasterPublishConfig(componentGroup) == false || canPublishConfig(componentGroup)) { // If no Master can explicitly publish then publish if its a master // Otherwise, wait till the master that can publish is ready @@ -1727,7 +1778,7 @@ public class AgentProviderService extends AbstractProviderService implements String hostNamePattern = "${THIS_HOST}"; Map<String, String> toPublish = new HashMap<String, String>(); - Application application = getMetaInfo().getApplication(); + Application application = getMetaInfo(componentGroup).getApplication(); for (Component component : application.getComponents()) { if (component.getName().equals(componentGroup)) { if (component.getComponentExports().size() > 0) { @@ -1778,8 +1829,8 @@ public class AgentProviderService extends AbstractProviderService implements String portVarFormat = "${site.%s}"; String hostNamePattern = "${" + compGroup + "_HOST}"; - List<ExportGroup> appExportGroups = getMetaInfo().getApplication().getExportGroups(); - Component component = getMetaInfo().getApplicationComponent(compGroup); + List<ExportGroup> appExportGroups = getMetaInfo(compGroup).getApplication().getExportGroups(); + Component component = getApplicationComponent(compGroup); if (component != null && SliderUtils.isSet(component.getCompExports()) && SliderUtils.isNotEmpty(appExportGroups)) { @@ -1881,7 +1932,11 @@ public class AgentProviderService extends AbstractProviderService implements * @return the component entry or null for no match */ protected Component getApplicationComponent(String roleGroup) { - return getMetaInfo().getApplicationComponent(roleGroup); + Metainfo metainfo = getMetaInfo(roleGroup); + if (metainfo == null) { + return null; + } + return metainfo.getApplicationComponent(roleGroup); } /** @@ -1950,9 +2005,9 @@ public class AgentProviderService extends AbstractProviderService implements * * @return true if the condition holds */ - protected boolean canAnyMasterPublishConfig() { + protected boolean canAnyMasterPublishConfig(String roleGroup) { if (canAnyMasterPublish == null) { - Application application = getMetaInfo().getApplication(); + Application application = getMetaInfo(roleGroup).getApplication(); if (application == null) { log.error("Malformed app definition: Expect application as root element in the metainfo.xml"); } else { @@ -2027,7 +2082,7 @@ public class AgentProviderService extends AbstractProviderService implements cmd.setPkg(pkg); Map<String, String> hostLevelParams = new TreeMap<String, String>(); hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getOption(JAVA_HOME, getJDKDir())); - hostLevelParams.put(PACKAGE_LIST, getPackageList()); + hostLevelParams.put(PACKAGE_LIST, getPackageList(roleGroup)); hostLevelParams.put(CONTAINER_ID, containerId); cmd.setHostLevelParams(hostLevelParams); @@ -2076,7 +2131,7 @@ public class AgentProviderService extends AbstractProviderService implements cmd.setComponentName(roleName); cmd.setRole(roleName); Map<String, String> hostLevelParams = new TreeMap<String, String>(); - hostLevelParams.put(PACKAGE_LIST, getPackageList()); + hostLevelParams.put(PACKAGE_LIST, getPackageList(roleGroup)); hostLevelParams.put(CONTAINER_ID, containerId); cmd.setHostLevelParams(hostLevelParams); @@ -2096,7 +2151,7 @@ public class AgentProviderService extends AbstractProviderService implements configurations.get("global").put("exec_cmd", effectiveCommand.getExec()); cmd.setHostname(getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME)); - cmd.addContainerDetails(roleGroup, getMetaInfo()); + cmd.addContainerDetails(roleGroup, getMetaInfo(roleGroup)); Map<String, String> dockerConfig = new HashMap<String, String>(); if(isYarnDockerContainer(roleGroup)){ @@ -2179,8 +2234,8 @@ public class AgentProviderService extends AbstractProviderService implements } } - private String getPackageList() { - return getPackageListFromApplication(getMetaInfo().getApplication()); + private String getPackageList(String roleGroup) { + return getPackageListFromApplication(getMetaInfo(roleGroup).getApplication()); } private void prepareExecutionCommand(ExecutionCommand cmd) { @@ -2345,7 +2400,7 @@ public class AgentProviderService extends AbstractProviderService implements private String getConfigFromMetaInfoWithAppConfigOverriding(String roleGroup, String configName){ ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); - String containerName = getMetaInfo().getApplicationComponent(roleGroup) + String containerName = getApplicationComponent(roleGroup) .getDockerContainers().get(0).getName(); String composedConfigName = null; String appConfigValue = null; @@ -2486,7 +2541,7 @@ public class AgentProviderService extends AbstractProviderService implements cmd.setConfigurations(configurations); // configurations.get("global").put("exec_cmd", startCommand.getExec()); - cmd.addContainerDetails(roleGroup, getMetaInfo()); + cmd.addContainerDetails(roleGroup, getMetaInfo(roleGroup)); log.info("Docker- command: {}", cmd.toString()); @@ -2496,7 +2551,7 @@ public class AgentProviderService extends AbstractProviderService implements private String getConfigFromMetaInfo(String roleGroup, String configName) { String result = null; - List<DockerContainer> containers = getMetaInfo().getApplicationComponent( + List<DockerContainer> containers = getApplicationComponent( roleGroup).getDockerContainers();// to support multi container per // component later log.debug("Docker- containers metainfo: {}", containers.toString()); @@ -2798,10 +2853,11 @@ public class AgentProviderService extends AbstractProviderService implements for (String configType : configs) { addNamedConfiguration(configType, appConf.getGlobalOptions().options, - configurations, tokens, containerId, roleName); + configurations, tokens, containerId, roleName, + roleGroup); if (appConf.getComponent(roleGroup) != null) { addNamedConfiguration(configType, appConf.getComponent(roleGroup).options, - configurations, tokens, containerId, roleName); + configurations, tokens, containerId, roleName, roleGroup); } } @@ -2844,15 +2900,32 @@ public class AgentProviderService extends AbstractProviderService implements tokens.put("${NN_HOST}", URI.create(nnuri).getHost()); tokens.put("${ZK_HOST}", appConf.get(OptionKeys.ZOOKEEPER_HOSTS)); tokens.put("${DEFAULT_ZK_PATH}", appConf.get(OptionKeys.ZOOKEEPER_PATH)); + String mapKey = getMetainfoMapKey(componentGroup); + String dataDirSuffix = ""; + if (!DEFAULT_METAINFO_MAP_KEY.equals(mapKey)) { + dataDirSuffix = "_" + mapKey.substring(0, mapKey.length()-1); + } else { + mapKey = ""; + } + mapKey = mapKey.toLowerCase(); tokens.put("${DEFAULT_DATA_DIR}", getAmState() .getInternalsSnapshot() .getGlobalOptions() - .getMandatoryOption(InternalKeys.INTERNAL_DATA_DIR_PATH)); + .getMandatoryOption(InternalKeys.INTERNAL_DATA_DIR_PATH) + dataDirSuffix); tokens.put("${JAVA_HOME}", appConf.get(AgentKeys.JAVA_HOME)); tokens.put("${COMPONENT_NAME}", componentName); + tokens.put("${COMPONENT_NAME.lc}", componentName.toLowerCase()); + tokens.put("${COMPONENT_PREFIX}", mapKey); + tokens.put("${COMPONENT_PREFIX.lc}", mapKey.toLowerCase()); if (!componentName.equals(componentGroup) && componentName.startsWith(componentGroup)) { tokens.put("${COMPONENT_ID}", componentName.substring(componentGroup.length())); } + tokens.put("${CLUSTER_NAME}", getClusterName()); + tokens.put("${CLUSTER_NAME.lc}", getClusterName().toLowerCase()); + tokens.put("${APP_NAME}", getClusterName()); + tokens.put("${APP_NAME.lc}", getClusterName().toLowerCase()); + tokens.put("${APP_COMPONENT_NAME}", componentName); + tokens.put("${APP_COMPONENT_NAME.lc}", componentName.toLowerCase()); return tokens; } @@ -2877,12 +2950,12 @@ public class AgentProviderService extends AbstractProviderService implements List<String> configList = new ArrayList<String>(); configList.add(GLOBAL_CONFIG_TAG); - List<ConfigFile> configFiles = getMetaInfo().getApplication().getConfigFiles(); + List<ConfigFile> configFiles = getMetaInfo(roleGroup).getApplication().getConfigFiles(); for (ConfigFile configFile : configFiles) { log.info("Expecting config type {}.", configFile.getDictionaryName()); configList.add(configFile.getDictionaryName()); } - for (Component component : getMetaInfo().getApplication().getComponents()) { + for (Component component : getMetaInfo(roleGroup).getApplication().getComponents()) { if (!component.getName().equals(roleGroup)) { continue; } @@ -2907,7 +2980,7 @@ public class AgentProviderService extends AbstractProviderService implements private void addNamedConfiguration(String configName, Map<String, String> sourceConfig, Map<String, Map<String, String>> configurations, Map<String, String> tokens, String containerId, - String roleName) { + String roleName, String roleGroup) { Map<String, String> config = new HashMap<String, String>(); if (configName.equals(GLOBAL_CONFIG_TAG)) { addDefaultGlobalConfig(config, containerId, roleName); @@ -2936,9 +3009,9 @@ public class AgentProviderService extends AbstractProviderService implements } //apply defaults only if the key is not present and value is not empty - if (getDefaultConfigs().containsKey(configName)) { + if (getDefaultConfigs(roleGroup).containsKey(configName)) { log.info("Adding default configs for type {}.", configName); - for (PropertyInfo defaultConfigProp : getDefaultConfigs().get(configName).getPropertyInfos()) { + for (PropertyInfo defaultConfigProp : getDefaultConfigs(roleGroup).get(configName).getPropertyInfos()) { if (!config.containsKey(defaultConfigProp.getName())) { if (!defaultConfigProp.getName().isEmpty() && defaultConfigProp.getValue() != null && http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/17096ffd/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java index cfcfc5d..ed5108c 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java @@ -35,6 +35,9 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import static org.apache.slider.common.SliderKeys.COMPONENT_SEPARATOR; +import static org.apache.slider.providers.agent.AgentKeys.DEFAULT_METAINFO_MAP_KEY; + /** * */ @@ -131,4 +134,25 @@ public class AgentUtils { return new DefaultConfigParser().parse(configStream); } + + static String getMetainfoMapKey(String roleGroup) { + if (roleGroup == null) { + return DEFAULT_METAINFO_MAP_KEY; + } + int lastIndex = roleGroup.lastIndexOf(COMPONENT_SEPARATOR); + if (lastIndex == -1) { + return DEFAULT_METAINFO_MAP_KEY; + } else { + return roleGroup.substring(0, lastIndex+1); + } + } + + static String getMetainfoComponentName(String roleGroup) { + int lastIndex = roleGroup.lastIndexOf(COMPONENT_SEPARATOR); + if (lastIndex == -1) { + return roleGroup; + } else { + return roleGroup.substring(lastIndex+1); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/17096ffd/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java index 91f1259..e2c879e 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java @@ -40,7 +40,13 @@ public class ComponentCommandOrder { Map<Command, Map<String, List<ComponentState>>> dependencies = new HashMap<Command, Map<String, List<ComponentState>>>(); + public ComponentCommandOrder() {} + public ComponentCommandOrder(List<CommandOrder> commandOrders) { + mergeCommandOrders(commandOrders); + } + + void mergeCommandOrders(List<CommandOrder> commandOrders) { if (commandOrders != null && commandOrders.size() > 0) { for (CommandOrder commandOrder : commandOrders) { ComponentCommand componentCmd = getComponentCommand(commandOrder.getCommand()); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/17096ffd/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index b767059..040f12b 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -882,7 +882,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService Configuration providerConf = providerService.loadProviderConfigurationInformation(confDir); - providerService.initializeApplicationConfiguration(instanceDefinition, fs); + providerService.initializeApplicationConfiguration(instanceDefinition, + fs, null); providerService.validateApplicationConfiguration(instanceDefinition, confDir, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/17096ffd/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java index f1e105a..20ef068 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java @@ -73,7 +73,7 @@ public class AgentResource extends AbstractSliderResource { } @POST - @Path("/{agent_name: [a-zA-Z][a-zA-Z_0-9]*}/register") + @Path("/{agent_name: [a-zA-Z][a-zA-Z0-9_-]*}/register") @Consumes({MediaType.APPLICATION_JSON}) @Produces({MediaType.APPLICATION_JSON}) public RegistrationResponse register(Register registration, @@ -87,7 +87,7 @@ public class AgentResource extends AbstractSliderResource { } @POST - @Path("/{agent_name: [a-zA-Z][a-zA-Z_0-9]*}/heartbeat") + @Path("/{agent_name: [a-zA-Z][a-zA-Z0-9_-]*}/heartbeat") @Consumes(MediaType.APPLICATION_JSON) @Produces({MediaType.APPLICATION_JSON}) public HeartBeatResponse heartbeat(HeartBeat message, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/17096ffd/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component.json new file mode 100644 index 0000000..853310d --- /dev/null +++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component.json @@ -0,0 +1,12 @@ +{ + "schema": "http://example.org/specification/v2.0.0";, + "metadata": { + }, + "global": { + }, + "components": { + "test_sleep": { + "site.global.component_type": "external" + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/17096ffd/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources_external_component.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources_external_component.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources_external_component.json new file mode 100644 index 0000000..bccea68 --- /dev/null +++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources_external_component.json @@ -0,0 +1,22 @@ +{ + "schema" : "http://example.org/specification/v2.0.0";, + "metadata" : { + }, + "global" : { + }, + "components": { + "slider-appmaster": { + "yarn.memory": "384" + }, + "SLEEP_100": { + "yarn.role.priority": "1", + "yarn.component.instances": "0", + "yarn.memory": "128" + }, + "SLEEP_LONG": { + "yarn.role.priority": "2", + "yarn.component.instances": "1", + "yarn.memory": "128" + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/17096ffd/slider-core/src/test/groovy/org/apache/slider/client/TestReplaceTokens.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestReplaceTokens.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestReplaceTokens.groovy index c4c755d..a35bcc0 100644 --- a/slider-core/src/test/groovy/org/apache/slider/client/TestReplaceTokens.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/client/TestReplaceTokens.groovy @@ -51,8 +51,9 @@ class TestReplaceTokens extends Assert { new JsonSerDeser<ConfTree>(ConfTree) def confTree = confTreeJsonSerDeser.fromResource(PACKAGE + app_configuration) SliderClient.replaceTokens(confTree, "testUser", "testCluster") - assert confTree.global.get("site.fs.defaultFS") == "hdfs://testCluster:8020" - assert confTree.global.get("site.fs.default.name") == "hdfs://testCluster:8020" + // cluster name is resolved later now + assert confTree.global.get("site.fs.defaultFS") == "hdfs://\${CLUSTER_NAME}:8020" + assert confTree.global.get("site.fs.default.name") == "hdfs://\${CLUSTER_NAME}:8020" assert confTree.global.get("site.hbase.user_name") == "testUser" assert confTree.global.get("site.hbase.another.user") == "testUser" http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/17096ffd/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy index 659af0e..1ea2277 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy @@ -175,7 +175,8 @@ class MockProviderService implements ProviderService { @Override void initializeApplicationConfiguration(AggregateConf instanceDefinition, - SliderFileSystem fileSystem) throws IOException, SliderException { + SliderFileSystem fileSystem, String roleGroup) + throws IOException, SliderException { } @Override http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/17096ffd/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java index 9fbb3d0..07d21d7 100644 --- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java +++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java @@ -471,7 +471,7 @@ public class TestAgentProviderService { metainfo.setApplication(application); doReturn(metainfo).when(mockAps).getApplicationMetainfo( any(SliderFileSystem.class), anyString()); - doReturn(metainfo).when(mockAps).getMetaInfo(); + doReturn(metainfo).when(mockAps).getMetaInfo(anyString()); @@ -664,7 +664,7 @@ public class TestAgentProviderService { hbaseMaster.setName("HBASE_MASTER"); application.addComponent(hbaseMaster); metainfo.setApplication(application); - doReturn(metainfo).when(mockAps).getMetaInfo(); + doReturn(metainfo).when(mockAps).getMetaInfo(anyString()); doReturn(metainfo).when(mockAps).getApplicationMetainfo( any(SliderFileSystem.class), anyString(), anyBoolean()); @@ -679,7 +679,7 @@ public class TestAgentProviderService { reg.setActualState(State.STARTED); mockAps.initializeApplicationConfiguration(instanceDefinition, - null); + null, null); RegistrationResponse resp = mockAps.handleRegistration(reg); Assert.assertEquals(0, resp.getResponseId()); @@ -735,7 +735,7 @@ public class TestAgentProviderService { metainfo.setApplication(application); doReturn(metainfo).when(mockAps).getApplicationMetainfo( any(SliderFileSystem.class), anyString(), anyBoolean()); - doReturn(metainfo).when(mockAps).getMetaInfo(); + doReturn(metainfo).when(mockAps).getMetaInfo(anyString()); doNothing().when(mockAps).addRoleRelatedTokens(anyMap()); Register reg = new Register(); @@ -748,7 +748,7 @@ public class TestAgentProviderService { // Simulating agent in INSTALLED state reg.setActualState(State.INSTALLED); - mockAps.initializeApplicationConfiguration(instanceDefinition, null); + mockAps.initializeApplicationConfiguration(instanceDefinition, null, "HBASE_MASTER"); RegistrationResponse resp = mockAps.handleRegistration(reg); Assert.assertEquals(0, resp.getResponseId()); @@ -853,7 +853,7 @@ public class TestAgentProviderService { AgentProviderService aps = createAgentProviderService(new Configuration()); AgentProviderService mockAps = Mockito.spy(aps); doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection()); - doReturn(metainfo).when(mockAps).getMetaInfo(); + doReturn(metainfo).when(mockAps).getMetaInfo(anyString()); Map<String, String> ports = new HashMap<String, String>(); ports.put("global.listen_port", "10010"); @@ -892,7 +892,7 @@ public class TestAgentProviderService { AgentProviderService aps = createAgentProviderService(new Configuration()); AgentProviderService mockAps = Mockito.spy(aps); doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection()); - doReturn(metainfo).when(mockAps).getMetaInfo(); + doReturn(metainfo).when(mockAps).getMetaInfo(anyString()); StateAccessForProviders access = createNiceMock(StateAccessForProviders.class); doReturn(access).when(mockAps).getAmState(); PublishedExportsSet pubExpSet = new PublishedExportsSet(); @@ -1005,7 +1005,7 @@ public class TestAgentProviderService { new MockContainerId(1), "cid"); AgentProviderService mockAps = Mockito.spy(aps); doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection()); - doReturn(metainfo).when(mockAps).getMetaInfo(); + doReturn(metainfo).when(mockAps).getMetaInfo(anyString()); doReturn(roleClusterNodeMap).when(mockAps).getRoleClusterNodeMapping(); StateAccessForProviders access = createNiceMock(StateAccessForProviders.class); doReturn(access).when(mockAps).getAmState(); @@ -1149,7 +1149,7 @@ public class TestAgentProviderService { AgentProviderService aps = createAgentProviderService(new Configuration()); AgentProviderService mockAps = Mockito.spy(aps); - doReturn(metainfo).when(mockAps).getMetaInfo(); + doReturn(metainfo).when(mockAps).getMetaInfo(anyString()); CommandScript script = mockAps.getScriptPathForMasterPackage("HBASE_MASTER"); Assert.assertEquals(script.getScript(), "scripts/hbase_master.py"); @@ -1178,22 +1178,24 @@ public class TestAgentProviderService { AgentProviderService aps1 = createAgentProviderService(new Configuration()); AgentProviderService mockAps = Mockito.spy(aps1); - doReturn(metainfo).when(mockAps).getMetaInfo(); + doReturn(metainfo).when(mockAps).getMetaInfo(anyString()); AgentProviderService mockAps2 = Mockito.spy(aps1); - doReturn(metainfo2).when(mockAps2).getMetaInfo(); + doReturn(metainfo2).when(mockAps2).getMetaInfo(anyString()); Assert.assertTrue(mockAps.isMaster(role_hm)); Assert.assertFalse(mockAps.isMaster(role_hrs)); Assert.assertTrue(mockAps.canPublishConfig(role_hm)); Assert.assertFalse(mockAps.canPublishConfig(role_hrs)); - Assert.assertTrue(mockAps.canAnyMasterPublishConfig()); + Assert.assertTrue(mockAps.canAnyMasterPublishConfig(role_hm)); + Assert.assertTrue(mockAps.canAnyMasterPublishConfig(role_hrs)); Assert.assertTrue(mockAps2.isMaster(role_hm)); Assert.assertFalse(mockAps2.isMaster(role_hrs)); Assert.assertTrue(mockAps2.canPublishConfig(role_hm)); Assert.assertFalse(mockAps2.canPublishConfig(role_hrs)); - Assert.assertTrue(mockAps2.canAnyMasterPublishConfig()); + Assert.assertTrue(mockAps2.canAnyMasterPublishConfig(role_hm)); + Assert.assertTrue(mockAps2.canAnyMasterPublishConfig(role_hrs)); } @Test @@ -1628,8 +1630,8 @@ public class TestAgentProviderService { expect(access.isApplicationLive()).andReturn(true).anyTimes(); doReturn("HOST1").when(mockAps).getClusterInfoPropertyValue(anyString()); - doReturn(metainfo).when(mockAps).getMetaInfo(); - doReturn(new HashMap<String, DefaultConfig>()).when(mockAps).getDefaultConfigs(); + doReturn(metainfo).when(mockAps).getMetaInfo(anyString()); + doReturn(new HashMap<String, DefaultConfig>()).when(mockAps).getDefaultConfigs(anyString()); Map<String, Map<String, ClusterNode>> roleClusterNodeMap = new HashMap<String, Map<String, ClusterNode>>(); Map<String, ClusterNode> container = new HashMap<String, ClusterNode>(); @@ -1695,7 +1697,7 @@ public class TestAgentProviderService { expect(access.isApplicationLive()).andReturn(true).anyTimes(); doReturn("HOST1").when(mockAps).getClusterInfoPropertyValue(anyString()); - doReturn(defaultConfigMap).when(mockAps).getDefaultConfigs(); + doReturn(defaultConfigMap).when(mockAps).getDefaultConfigs(anyString()); List<String> configurations = new ArrayList<String>(); configurations.add("hbase-site"); configurations.add("global"); @@ -1783,7 +1785,7 @@ public class TestAgentProviderService { expect(access.isApplicationLive()).andReturn(true).anyTimes(); doReturn("HOST1").when(mockAps).getClusterInfoPropertyValue(anyString()); - doReturn(defaultConfigMap).when(mockAps).getDefaultConfigs(); + doReturn(defaultConfigMap).when(mockAps).getDefaultConfigs(anyString()); List<String> configurations = new ArrayList<String>(); configurations.add("hbase-site"); configurations.add("global"); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/17096ffd/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy index 5de2b8e..1cb6f0f 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy @@ -38,4 +38,6 @@ interface ResourcePaths { String SLEEP_META = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/metainfo.json" String SLEEP_APPCONFIG = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/appConfig.json" + String EXTERNAL_RESOURCES = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/resources_external_component.json" + String EXTERNAL_APPCONFIG = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/appConfig_external_component.json" } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/17096ffd/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/ExternalComponentIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/ExternalComponentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/ExternalComponentIT.groovy new file mode 100644 index 0000000..b5e0270 --- /dev/null +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/ExternalComponentIT.groovy @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.funtest.lifecycle + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import org.apache.slider.api.ClusterDescription +import org.apache.slider.api.ResourceKeys +import org.apache.slider.common.SliderExitCodes +import org.apache.slider.common.SliderKeys +import org.apache.slider.common.params.Arguments +import org.apache.slider.common.params.SliderActions +import org.apache.slider.funtest.ResourcePaths +import org.apache.slider.funtest.framework.AgentCommandTestBase +import org.apache.slider.funtest.framework.FuntestProperties +import org.junit.After +import org.junit.Before +import org.junit.Test + +@CompileStatic +@Slf4j +public class ExternalComponentIT extends AgentCommandTestBase + implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { + + static String NAME = "test-external-component" + static String EXT_NAME = "test_sleep" + + static String BUILD_APPCONFIG = ResourcePaths.SLEEP_APPCONFIG + static String BUILD_RESOURCES = ResourcePaths.EXTERNAL_RESOURCES + static String BUILD_METAINFO = ResourcePaths.SLEEP_META + static String TEST_APPCONFIG = ResourcePaths.EXTERNAL_APPCONFIG + static String TEST_RESOURCES = ResourcePaths.EXTERNAL_RESOURCES + static String TEST_METAINFO = ResourcePaths.SLEEP_META + public static final String SLEEP_100 = "SLEEP_100" + public static final String SLEEP_LONG = "SLEEP_LONG" + public static final String EXT_SLEEP_100 = EXT_NAME + + SliderKeys.COMPONENT_SEPARATOR + SLEEP_100 + public static final String EXT_SLEEP_LONG = EXT_NAME + + SliderKeys.COMPONENT_SEPARATOR + SLEEP_LONG + + @Before + public void prepareCluster() { + setupCluster(NAME) + setupCluster(EXT_NAME) + } + + @After + public void destroyCluster() { + cleanup(NAME) + cleanup(EXT_NAME) + } + + @Test + public void testExternalComponent() throws Throwable { + assumeAgentTestsEnabled() + describe NAME + + slider(0, [ACTION_BUILD, EXT_NAME, ARG_METAINFO, BUILD_METAINFO, + ARG_TEMPLATE, BUILD_APPCONFIG, ARG_RESOURCES, BUILD_RESOURCES, + ARG_RES_COMP_OPT, SLEEP_100, ResourceKeys.COMPONENT_INSTANCES, "0", + ARG_RES_COMP_OPT, SLEEP_LONG, ResourceKeys.COMPONENT_INSTANCES, "1" + ]) + + slider(0, [ACTION_CREATE, NAME, ARG_METAINFO, TEST_METAINFO, + ARG_TEMPLATE, TEST_APPCONFIG, ARG_RESOURCES, TEST_RESOURCES]) + + ensureApplicationIsUp(NAME) + status(0, NAME) + + ClusterDescription cd = execStatus(NAME) + + assert 5 == cd.statistics.size() + assert cd.statistics.keySet().containsAll([SliderKeys.COMPONENT_AM, + SLEEP_100, SLEEP_LONG, + EXT_SLEEP_100, EXT_SLEEP_LONG]) + + expectLiveContainerCountReached(NAME, SLEEP_LONG, 1, + CONTAINER_LAUNCH_TIMEOUT) + expectLiveContainerCountReached(NAME, EXT_SLEEP_LONG, 1, + CONTAINER_LAUNCH_TIMEOUT) + expectLiveContainerCountReached(NAME, SLEEP_100, 0, + CONTAINER_LAUNCH_TIMEOUT) + expectLiveContainerCountReached(NAME, EXT_SLEEP_100, 0, + CONTAINER_LAUNCH_TIMEOUT) + + cleanup(NAME) + + // test overriding the number of instances of external components + describe NAME + "-2" + + slider(0, [ACTION_CREATE, NAME, ARG_METAINFO, TEST_METAINFO, + ARG_TEMPLATE, TEST_APPCONFIG, ARG_RESOURCES, TEST_RESOURCES, + ARG_RES_COMP_OPT, SLEEP_LONG, ResourceKeys.COMPONENT_INSTANCES, "0", + ARG_RES_COMP_OPT, EXT_SLEEP_LONG, ResourceKeys.COMPONENT_INSTANCES, "2", + ]) + + ensureApplicationIsUp(NAME) + status(0, NAME) + + cd = execStatus(NAME) + + assert 5 == cd.statistics.size() + assert cd.statistics.keySet().containsAll([SliderKeys.COMPONENT_AM, + SLEEP_100, SLEEP_LONG, + EXT_SLEEP_100, EXT_SLEEP_LONG]) + + expectLiveContainerCountReached(NAME, SLEEP_LONG, 0, + CONTAINER_LAUNCH_TIMEOUT) + expectLiveContainerCountReached(NAME, EXT_SLEEP_LONG, 2, + CONTAINER_LAUNCH_TIMEOUT) + expectLiveContainerCountReached(NAME, SLEEP_100, 0, + CONTAINER_LAUNCH_TIMEOUT) + expectLiveContainerCountReached(NAME, EXT_SLEEP_100, 0, + CONTAINER_LAUNCH_TIMEOUT) + + } +}
