Revert "SLIDER-875 initial uber app support" This reverts commit 17096ffd0562279aaf082004e2f774711b37c56f.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c0f4e922 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c0f4e922 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c0f4e922 Branch: refs/heads/feature/SLIDER-1107_AM_config_generation Commit: c0f4e922086fb0cda0751054ddbac6e23e3c9f8c Parents: 17096ff Author: Billie Rinaldi <billie.rina...@gmail.com> Authored: Thu Jun 23 08:41:27 2016 -0700 Committer: Billie Rinaldi <billie.rina...@gmail.com> Committed: Thu Jun 23 08:41:27 2016 -0700 ---------------------------------------------------------------------- .../org/apache/slider/client/SliderClient.java | 4 +- .../org/apache/slider/common/SliderKeys.java | 7 - .../apache/slider/common/tools/SliderUtils.java | 79 -------- .../slider/core/build/InstanceBuilder.java | 148 --------------- .../slider/core/conf/ConfTreeOperations.java | 11 -- .../providers/AbstractProviderService.java | 3 +- .../slider/providers/ProviderService.java | 4 +- .../providers/agent/AgentClientProvider.java | 30 ++- .../slider/providers/agent/AgentKeys.java | 2 - .../providers/agent/AgentProviderService.java | 181 ++++++------------- .../slider/providers/agent/AgentUtils.java | 24 --- .../providers/agent/ComponentCommandOrder.java | 6 - .../server/appmaster/SliderAppMaster.java | 3 +- .../appmaster/web/rest/agent/AgentResource.java | 4 +- .../sleep_cmd/appConfig_external_component.json | 12 -- .../sleep_cmd/resources_external_component.json | 22 --- .../slider/client/TestReplaceTokens.groovy | 5 +- .../model/mock/MockProviderService.groovy | 3 +- .../agent/TestAgentProviderService.java | 36 ++-- .../apache/slider/funtest/ResourcePaths.groovy | 2 - .../funtest/misc/ExternalComponentIT.groovy | 133 -------------- 21 files changed, 93 insertions(+), 626 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c0f4e922/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 372ec48..dd90e46 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -1597,7 +1597,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe resources.mergeComponents(buildInfo.getResourceCompOptionMap()); builder.init(providerName, instanceDefinition); - builder.resolve(); builder.propagateFilename(); builder.propagatePrincipals(); builder.setImageDetailsIfAvailable(buildInfo.getImage(), @@ -1787,7 +1786,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe private static String replaceTokens(String s, String userName, String clusterName) throws IOException { return s.replaceAll(Pattern.quote("${USER}"), userName) - .replaceAll(Pattern.quote("${USER_NAME}"), userName); + .replaceAll(Pattern.quote("${USER_NAME}"), userName) + .replaceAll(Pattern.quote("${CLUSTER_NAME}"), clusterName); } public FsPermission getClusterDirectoryPermissions(Configuration conf) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c0f4e922/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java index 3921324..05c7048 100644 --- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java +++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java @@ -70,13 +70,6 @@ public interface SliderKeys extends SliderXmlConfKeys { String APP_TYPE = "org-apache-slider"; /** - * Key for component type. This MUST NOT be set in app_config/global {@value} - */ - String COMPONENT_TYPE = "site.global.component_type"; - String EXTERNAL_COMPONENT = "external"; - String COMPONENT_SEPARATOR = "-"; - - /** * Key for application version. This must be set in app_config/global {@value} */ String APP_VERSION = "site.global.app_version"; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c0f4e922/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index c48e109..746e468 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -69,7 +69,6 @@ import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.core.launch.ClasspathConstructor; import org.apache.slider.core.main.LauncherExitCodes; import org.apache.slider.providers.agent.AgentKeys; -import org.apache.slider.providers.agent.application.metadata.Component; import org.apache.slider.server.services.utility.PatternValidator; import org.apache.slider.server.services.workflow.ForkedProcessService; import org.apache.zookeeper.server.util.KerberosUtil; @@ -476,32 +475,6 @@ public final class SliderUtils { return srcFileCount; } - /** - * Copy a file to a new FS -both paths must be qualified. - * @param conf conf file - * @param srcFile src file - * @param destFile dest file - */ - public static void copy(Configuration conf, - Path srcFile, - Path destFile) throws - IOException, - BadClusterStateException { - FileSystem srcFS = FileSystem.get(srcFile.toUri(), conf); - //list all paths in the src. - if (!srcFS.exists(srcFile)) { - throw new FileNotFoundException("Source dir not found " + srcFile); - } - if (!srcFS.isFile(srcFile)) { - throw new FileNotFoundException( - "Source dir not a file " + srcFile); - } - FileSystem destFS = FileSystem.get(destFile.toUri(), conf); - if (destFS.exists(destFile)) { - throw new IOException("Dest file already exists " + destFile); - } - FileUtil.copy(srcFS, srcFile, destFS, destFile, false, true, conf); - } public static String stringify(Throwable t) { StringWriter sw = new StringWriter(); @@ -954,38 +927,6 @@ public final class SliderUtils { } /** - * Merge string maps excluding prefixes - * @param first first map - * @param second second map - * @param prefixes prefixes to ignore - * @return 'first' merged with the second - */ - public static Map<String, String> mergeMapsIgnorePrefixes( - Map<String, String> first, Map<String, String> second, - String... prefixes) { - Preconditions.checkArgument(first != null, "Null 'first' value"); - Preconditions.checkArgument(second != null, "Null 'second' value"); - Preconditions.checkArgument(prefixes != null, "Null 'prefixes' value"); - for (Map.Entry<String, String> entry : second.entrySet()) { - String key = entry.getKey(); - boolean hasPrefix = false; - for (String prefix : prefixes) { - if (key.startsWith(prefix)) { - hasPrefix = true; - break; - } - } - if (hasPrefix) { - continue; - } - if (!first.containsKey(key)) { - first.put(key, entry.getValue()); - } - } - return first; - } - - /** * Convert a map to a multi-line string for printing * @param map map to stringify * @return a string representation of the map @@ -2399,28 +2340,8 @@ public final class SliderUtils { */ public static String getApplicationDefinitionPath(ConfTreeOperations conf) throws BadConfigException { - return getApplicationDefinitionPath(conf, null); - } - - /** - * return the HDFS path where the application package has been uploaded - * manually or by using slider client (install package command) - * - * @param conf configuration - * @param roleGroup name of component - * @return - */ - public static String getApplicationDefinitionPath(ConfTreeOperations conf, - String roleGroup) - throws BadConfigException { String appDefPath = conf.getGlobalOptions().getMandatoryOption( AgentKeys.APP_DEF); - if (roleGroup != null) { - MapOperations component = conf.getComponent(roleGroup); - if (component != null) { - appDefPath = component.getOption(AgentKeys.APP_DEF, appDefPath); - } - } return appDefPath; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c0f4e922/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java index 4afdf7c..7756055 100644 --- a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java +++ b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java @@ -25,9 +25,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.slider.api.InternalKeys; import org.apache.slider.api.OptionKeys; -import org.apache.slider.api.ResourceKeys; import org.apache.slider.api.StatusKeys; -import org.apache.slider.common.SliderKeys; import org.apache.slider.common.SliderXmlConfKeys; import org.apache.slider.common.tools.CoreFileSystem; import org.apache.slider.common.tools.SliderUtils; @@ -44,17 +42,11 @@ import org.apache.slider.core.persist.LockAcquireFailedException; import org.apache.slider.core.persist.LockHeldAction; import org.apache.slider.core.zk.ZKPathBuilder; import org.apache.slider.core.zk.ZookeeperUtils; -import org.apache.slider.providers.agent.AgentKeys; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.HashMap; -import java.util.HashSet; import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.TreeSet; import static org.apache.slider.api.InternalKeys.INTERNAL_ADDONS_DIR_PATH; import static org.apache.slider.api.InternalKeys.INTERNAL_APPDEF_DIR_PATH; @@ -69,11 +61,6 @@ import static org.apache.slider.api.OptionKeys.INTERNAL_SNAPSHOT_CONF_PATH; import static org.apache.slider.api.OptionKeys.ZOOKEEPER_HOSTS; import static org.apache.slider.api.OptionKeys.ZOOKEEPER_PATH; import static org.apache.slider.api.OptionKeys.ZOOKEEPER_QUORUM; -import static org.apache.slider.common.SliderKeys.COMPONENT_AM; -import static org.apache.slider.common.SliderKeys.COMPONENT_SEPARATOR; -import static org.apache.slider.common.SliderKeys.COMPONENT_TYPE; -import static org.apache.slider.common.SliderKeys.EXTERNAL_COMPONENT; -import static org.apache.slider.common.tools.SliderUtils.isClusternameValid; /** * Build up the instance of a cluster. @@ -85,8 +72,6 @@ public class InstanceBuilder { private final CoreFileSystem coreFS; private final InstancePaths instancePaths; private AggregateConf instanceDescription; - private Map<Path, Path> externalAppDefs = new HashMap<>(); - private TreeSet<Integer> priorities = new TreeSet<>(); private static final Logger log = LoggerFactory.getLogger(InstanceBuilder.class); @@ -259,136 +244,6 @@ public class InstanceBuilder { } - private void getExternalComponents(ConfTreeOperations ops, - Set<String> externalComponents) throws BadConfigException { - if (ops.getGlobalOptions().get(COMPONENT_TYPE) != null) { - throw new BadConfigException(COMPONENT_TYPE + " must be " + - "specified per-component, not in global"); - } - - for (Entry<String, Map<String, String>> entry : ops.getComponents() - .entrySet()) { - if (COMPONENT_AM.equals(entry.getKey())) { - continue; - } - Map<String, String> options = entry.getValue(); - if (options.containsKey(COMPONENT_TYPE) && - EXTERNAL_COMPONENT.equals(options.get(COMPONENT_TYPE))) { - externalComponents.add(entry.getKey()); - } - } - } - - private static String[] PREFIXES_TO_SKIP = {"zookeeper.", - "env.MALLOC_ARENA_MAX", "site.fs.", "site.dfs."}; - - private void mergeExternalComponent(ConfTreeOperations ops, - ConfTreeOperations externalOps, String externalComponent, - Integer priority) { - for (String subComponent : externalOps.getComponentNames()) { - if (COMPONENT_AM.equals(subComponent)) { - continue; - } - log.debug("Merging options for {} into {}", subComponent, - externalComponent + COMPONENT_SEPARATOR + subComponent); - MapOperations subComponentOps = ops.getOrAddComponent(externalComponent + - COMPONENT_SEPARATOR + subComponent); - SliderUtils.mergeMapsIgnorePrefixes(subComponentOps, - externalOps.getComponent(subComponent), PREFIXES_TO_SKIP); - if (priority != null) { - subComponentOps.put(ResourceKeys.COMPONENT_PRIORITY, - Integer.toString(priority)); - priorities.add(priority); - priority++; - } - } - } - - private int getNextPriority() { - if (priorities.isEmpty()) { - return 1; - } else { - return priorities.last() + 1; - } - } - - public void resolve() - throws BadConfigException, IOException, BadClusterStateException { - ConfTreeOperations appConf = instanceDescription.getAppConfOperations(); - ConfTreeOperations resources = instanceDescription.getResourceOperations(); - - for (Entry<String, Map<String, String>> entry : resources.getComponents() - .entrySet()) { - if (COMPONENT_AM.equals(entry.getKey())) { - continue; - } - if (entry.getKey().contains(COMPONENT_SEPARATOR)) { - throw new BadConfigException("Components must not contain " + - COMPONENT_SEPARATOR + ": " + entry.getKey()); - } - if (entry.getValue().containsKey(ResourceKeys.COMPONENT_PRIORITY)) { - priorities.add(Integer.parseInt(entry.getValue().get( - ResourceKeys.COMPONENT_PRIORITY))); - } - } - - Set<String> externalComponents = new HashSet<>(); - getExternalComponents(appConf, externalComponents); - if (!externalComponents.isEmpty()) { - log.info("Found external components {}", externalComponents); - } - - for (String component : externalComponents) { - if (!isClusternameValid(component)) { - throw new BadConfigException(component + " is not a valid external " + - "component"); - } - Path componentClusterDir = coreFS.buildClusterDirPath(component); - try { - coreFS.verifyPathExists(componentClusterDir); - } catch (IOException e) { - throw new BadConfigException("external component " + component + - " doesn't exist"); - } - AggregateConf componentConf = new AggregateConf(); - ConfPersister persister = new ConfPersister(coreFS, componentClusterDir); - try { - persister.load(componentConf); - } catch (Exception e) { - throw new BadConfigException("Couldn't read configuration for " + - "external component " + component); - } - String externalAppDef = componentConf.getAppConfOperations() - .getGlobalOptions().get(AgentKeys.APP_DEF); - if (SliderUtils.isSet(externalAppDef)) { - Path newAppDef = new Path(coreFS.buildAppDefDirPath(clustername), - component + "_" + SliderKeys.DEFAULT_APP_PKG); - componentConf.getAppConfOperations().set(AgentKeys.APP_DEF, newAppDef); - externalAppDefs.put(newAppDef, new Path(externalAppDef)); - } - for (String rcomp : componentConf.getResourceOperations() - .getComponentNames()) { - if (COMPONENT_AM.equals(rcomp)) { - continue; - } - log.debug("Adding component {} to appConf for {}", rcomp, component); - componentConf.getAppConfOperations().getOrAddComponent(rcomp); - } - SliderUtils.mergeMaps( - componentConf.getAppConfOperations().getGlobalOptions().options, - appConf.getComponent(component).options); - componentConf.getAppConfOperations().getGlobalOptions() - .remove(COMPONENT_TYPE); - componentConf.resolve(); - - mergeExternalComponent(appConf, componentConf.getAppConfOperations(), - component, null); - mergeExternalComponent(resources, componentConf.getResourceOperations(), - component, getNextPriority()); - } - } - - /** * Persist this * @param appconfdir conf dir @@ -411,9 +266,6 @@ public class InstanceBuilder { action = new ConfDirSnapshotAction(appconfdir); } persister.save(instanceDescription, action); - for (Entry<Path, Path> appDef : externalAppDefs.entrySet()) { - SliderUtils.copy(conf, appDef.getValue(), appDef.getKey()); - } } /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c0f4e922/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java index 4a0ae41..9013edb 100644 --- a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java +++ b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java @@ -257,17 +257,6 @@ public class ConfTreeOperations { Map<String, String> map, String prefix, boolean overwrite) { - boolean needsMerge = false; - for (Map.Entry<String, String> entry : map.entrySet()) { - String key = entry.getKey(); - if (key.startsWith(prefix)) { - needsMerge = true; - break; - } - } - if (!needsMerge) { - return; - } MapOperations comp = getOrAddComponent(component); comp.mergeMapPrefixedKeys(map,prefix, overwrite); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c0f4e922/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java index 92766f5..61b2655 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java @@ -169,8 +169,7 @@ public abstract class AbstractProviderService */ @Override public void initializeApplicationConfiguration( - AggregateConf instanceDefinition, SliderFileSystem fileSystem, - String roleGroup) + AggregateConf instanceDefinition, SliderFileSystem fileSystem) throws IOException, SliderException { } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c0f4e922/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java index 3f24665..f754eee 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java @@ -118,13 +118,11 @@ public interface ProviderService extends ProviderCore, * * @param instanceDefinition * @param fileSystem - * @param roleGroup * @throws IOException * @throws SliderException */ void initializeApplicationConfiguration(AggregateConf instanceDefinition, - SliderFileSystem fileSystem, String roleGroup) throws IOException, - SliderException; + SliderFileSystem fileSystem) throws IOException, SliderException; /** * This is a validation of the application configuration on the AM. http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c0f4e922/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java index 6eae75e..f3dcd1d 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java @@ -74,8 +74,6 @@ import java.util.Set; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; -import static org.apache.slider.providers.agent.AgentUtils.getMetainfoComponentName; - /** This class implements the client-side aspects of the agent deployer */ public class AgentClientProvider extends AbstractClientProvider implements AgentKeys, SliderKeys { @@ -170,21 +168,13 @@ public class AgentClientProvider extends AbstractClientProvider names.remove(SliderKeys.COMPONENT_AM); Map<Integer, String> priorityMap = new HashMap<Integer, String>(); - for (String name : names) { - try { - // Validate the app definition - appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition - .getAppConfOperations(), name); - } catch (BadConfigException bce) { - throw new BadConfigException("Application definition must be provided. " + bce.getMessage()); - } - Metainfo metaInfo = getMetainfo(fs, appDef); + Metainfo metaInfo = getMetainfo(fs, appDef); + for (String name : names) { MapOperations component = resources.getMandatoryComponent(name); if (metaInfo != null) { - Component componentDef = metaInfo.getApplicationComponent( - getMetainfoComponentName(name)); + Component componentDef = metaInfo.getApplicationComponent(name); if (componentDef == null) { throw new BadConfigException( "Component %s is not a member of application.", name); @@ -210,12 +200,16 @@ public class AgentClientProvider extends AbstractClientProvider existing); } priorityMap.put(priority, name); + } - // fileSystem may be null for tests - if (metaInfo != null) { - Component componentDef = metaInfo.getApplicationComponent( - getMetainfoComponentName(name)); - // already checked it wasn't null + // fileSystem may be null for tests + if (metaInfo != null) { + for (String name : names) { + Component componentDef = metaInfo.getApplicationComponent(name); + if (componentDef == null) { + throw new BadConfigException( + "Component %s is not a member of application.", name); + } // ensure that intance count is 0 for client components if ("CLIENT".equals(componentDef.getCategory())) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c0f4e922/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java index 8514401..b027939 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java @@ -97,8 +97,6 @@ public interface AgentKeys { String CERT_FILE_LOCALIZATION_PATH = INFRA_RUN_SECURITY_DIR + "ca.crt"; String KEY_CONTAINER_LAUNCH_DELAY = "container.launch.delay.sec"; String TEST_RELAX_VERIFICATION = "test.relax.validation"; - - String DEFAULT_METAINFO_MAP_KEY = "DEFAULT"; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c0f4e922/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java index 66fac99..f20757a 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java @@ -71,7 +71,6 @@ import org.apache.slider.providers.ProviderRole; import org.apache.slider.providers.ProviderUtils; import org.apache.slider.providers.agent.application.metadata.AbstractComponent; import org.apache.slider.providers.agent.application.metadata.Application; -import org.apache.slider.providers.agent.application.metadata.CommandOrder; import org.apache.slider.providers.agent.application.metadata.CommandScript; import org.apache.slider.providers.agent.application.metadata.Component; import org.apache.slider.providers.agent.application.metadata.ComponentCommand; @@ -125,6 +124,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Map.Entry; import java.util.Scanner; import java.util.Set; import java.util.TreeMap; @@ -132,8 +132,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.regex.Pattern; -import static org.apache.slider.providers.agent.AgentUtils.getMetainfoMapKey; import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_AGENTS; /** @@ -168,8 +168,9 @@ public class AgentProviderService extends AbstractProviderService implements private int heartbeatMonitorInterval = 0; private AgentClientProvider clientProvider; private AtomicInteger taskId = new AtomicInteger(0); - private volatile Map<String, MetainfoHolder> metaInfoMap = new HashMap<>(); - private ComponentCommandOrder commandOrder = new ComponentCommandOrder(); + private volatile Metainfo metaInfo = null; + private Map<String, DefaultConfig> defaultConfigs = null; + private ComponentCommandOrder commandOrder = null; private HeartbeatMonitor monitor; private Boolean canAnyMasterPublish = null; private AgentLaunchParameter agentLaunchParameter = null; @@ -204,17 +205,6 @@ public class AgentProviderService extends AbstractProviderService implements private final Map<String, Set<String>> containerExportsMap = new HashMap<String, Set<String>>(); - private static class MetainfoHolder { - Metainfo metaInfo; - private Map<String, DefaultConfig> defaultConfigs = null; - - public MetainfoHolder(Metainfo metaInfo, - Map<String, DefaultConfig> defaultConfigs) { - this.metaInfo = metaInfo; - this.defaultConfigs = defaultConfigs; - } - } - /** * Create an instance of AgentProviderService */ @@ -259,11 +249,10 @@ public class AgentProviderService extends AbstractProviderService implements Set<String> names = resources.getComponentNames(); names.remove(SliderKeys.COMPONENT_AM); for (String name : names) { - Component componentDef = getApplicationComponent(name); + Component componentDef = getMetaInfo().getApplicationComponent(name); if (componentDef == null) { - // component member is validated elsewhere, so we don't need to throw - // an exception here - continue; + throw new BadConfigException( + "Component %s is not a member of application.", name); } MapOperations componentConfig = resources.getMandatoryComponent(name); @@ -285,62 +274,31 @@ public class AgentProviderService extends AbstractProviderService implements // Reads the metainfo.xml in the application package and loads it private void buildMetainfo(AggregateConf instanceDefinition, - SliderFileSystem fileSystem, - String roleGroup) - throws IOException, SliderException { - String mapKey = getMetainfoMapKey(roleGroup); - String appDef = SliderUtils.getApplicationDefinitionPath( - instanceDefinition.getAppConfOperations(), roleGroup); - MapOperations component = null; - if (roleGroup != null) { - component = instanceDefinition.getAppConfOperations().getComponent(roleGroup); - } + SliderFileSystem fileSystem) throws IOException, SliderException { + String appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition + .getAppConfOperations()); - MetainfoHolder metaInfoHolder = metaInfoMap.get(mapKey); - if (metaInfoHolder == null) { + if (metaInfo == null) { synchronized (syncLock) { - metaInfoHolder = metaInfoMap.get(mapKey); - if (metaInfoHolder == null) { + if (metaInfo == null) { readAndSetHeartbeatMonitoringInterval(instanceDefinition); initializeAgentDebugCommands(instanceDefinition); - Metainfo metaInfo = getApplicationMetainfo(fileSystem, appDef, false); + metaInfo = getApplicationMetainfo(fileSystem, appDef, false); log.info("Master package metainfo: {}", metaInfo.toString()); if (metaInfo == null || metaInfo.getApplication() == null) { log.error("metainfo.xml is unavailable or malformed at {}.", appDef); throw new SliderException( "metainfo.xml is required in app package."); } - List<CommandOrder> commandOrders = metaInfo.getApplication() - .getCommandOrders(); - if (!DEFAULT_METAINFO_MAP_KEY.equals(mapKey)) { - for (Component comp : metaInfo.getApplication().getComponents()) { - comp.setName(mapKey + comp.getName()); - log.info("Modifying external metainfo component name to {}", - comp.getName()); - } - String commandPrefix = mapKey.substring(0, - mapKey.indexOf(COMPONENT_SEPARATOR)+1); - for (CommandOrder co : commandOrders) { - log.info("Adding prefix {} to command order {}", - commandPrefix, co); - co.setCommand(commandPrefix + co.getCommand()); - co.setRequires(commandPrefix + co.getRequires()); - } - } - commandOrder.mergeCommandOrders(commandOrders); - Map<String, DefaultConfig> defaultConfigs = - initializeDefaultConfigs(fileSystem, appDef, metaInfo); - metaInfoMap.put(mapKey, new MetainfoHolder(metaInfo, defaultConfigs)); + commandOrder = new ComponentCommandOrder(metaInfo.getApplication().getCommandOrders()); + defaultConfigs = initializeDefaultConfigs(fileSystem, appDef, metaInfo); monitor = new HeartbeatMonitor(this, getHeartbeatMonitorInterval()); monitor.start(); // build a map from component to metainfo String addonAppDefString = instanceDefinition.getAppConfOperations() .getGlobalOptions().getOption(AgentKeys.ADDONS, null); - if (component != null) { - addonAppDefString = component.getOption(AgentKeys.ADDONS, addonAppDefString); - } log.debug("All addon appdefs: {}", addonAppDefString); if (addonAppDefString != null) { Scanner scanner = new Scanner(addonAppDefString).useDelimiter(","); @@ -348,9 +306,6 @@ public class AgentProviderService extends AbstractProviderService implements String addonAppDef = scanner.next(); String addonAppDefPath = instanceDefinition .getAppConfOperations().getGlobalOptions().get(addonAppDef); - if (component != null) { - addonAppDefPath = component.getOption(addonAppDef, addonAppDefPath); - } log.debug("Addon package {} is stored at: {}", addonAppDef + addonAppDefPath); Metainfo addonMetaInfo = getApplicationMetainfo(fileSystem, @@ -369,10 +324,9 @@ public class AgentProviderService extends AbstractProviderService implements @Override public void initializeApplicationConfiguration( - AggregateConf instanceDefinition, SliderFileSystem fileSystem, - String roleGroup) + AggregateConf instanceDefinition, SliderFileSystem fileSystem) throws IOException, SliderException { - buildMetainfo(instanceDefinition, fileSystem, roleGroup); + buildMetainfo(instanceDefinition, fileSystem); } @Override @@ -391,9 +345,9 @@ public class AgentProviderService extends AbstractProviderService implements String roleName = providerRole.name; String roleGroup = providerRole.group; String appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition - .getAppConfOperations(), roleGroup); + .getAppConfOperations()); - initializeApplicationConfiguration(instanceDefinition, fileSystem, roleGroup); + initializeApplicationConfiguration(instanceDefinition, fileSystem); log.info("Build launch context for Agent"); log.debug(instanceDefinition.toString()); @@ -610,7 +564,7 @@ public class AgentProviderService extends AbstractProviderService implements // initialize the component instance state getComponentStatuses().put(label, new ComponentInstanceState( - roleGroup, + roleName, container.getId(), getClusterInfoPropertyValue(OptionKeys.APPLICATION_NAME), pkgStatuses)); @@ -786,12 +740,11 @@ public class AgentProviderService extends AbstractProviderService implements .extractRole(container)); if (role != null) { String roleName = role.name; - String roleGroup = role.group; - String label = getContainerLabel(container, roleName, roleGroup); + String label = getContainerLabel(container, roleName, role.group); log.info("Rebuilding in-memory: container {} in role {} in cluster {}", container.getId(), roleName, applicationId); getComponentStatuses().put(label, - new ComponentInstanceState(roleGroup, container.getId(), + new ComponentInstanceState(roleName, container.getId(), applicationId)); } else { log.warn("Role not found for container {} in cluster {}", @@ -912,7 +865,7 @@ public class AgentProviderService extends AbstractProviderService implements StateAccessForProviders accessor = getAmState(); CommandScript cmdScript = getScriptPathForMasterPackage(roleGroup); - List<ComponentCommand> commands = getApplicationComponent(roleGroup).getCommands(); + List<ComponentCommand> commands = getMetaInfo().getApplicationComponent(roleGroup).getCommands(); if (!isDockerContainer(roleGroup) && !isYarnDockerContainer(roleGroup) && (cmdScript == null || cmdScript.getScript() == null) @@ -1190,7 +1143,7 @@ public class AgentProviderService extends AbstractProviderService implements } private boolean isDockerContainer(String roleGroup) { - String type = getApplicationComponent(roleGroup).getType(); + String type = getMetaInfo().getApplicationComponent(roleGroup).getType(); if (SliderUtils.isSet(type)) { return type.toLowerCase().equals(SliderUtils.DOCKER) || type.toLowerCase().equals(SliderUtils.DOCKER_YARN); } @@ -1198,7 +1151,7 @@ public class AgentProviderService extends AbstractProviderService implements } private boolean isYarnDockerContainer(String roleGroup) { - String type = getApplicationComponent(roleGroup).getType(); + String type = getMetaInfo().getApplicationComponent(roleGroup).getType(); if (SliderUtils.isSet(type)) { return type.toLowerCase().equals(SliderUtils.DOCKER_YARN); } @@ -1392,12 +1345,8 @@ public class AgentProviderService extends AbstractProviderService implements } @VisibleForTesting - protected Metainfo getMetaInfo(String roleGroup) { - MetainfoHolder mh = this.metaInfoMap.get(getMetainfoMapKey(roleGroup)); - if (mh == null) { - return null; - } - return mh.metaInfo; + protected Metainfo getMetaInfo() { + return this.metaInfo; } @VisibleForTesting @@ -1467,8 +1416,8 @@ public class AgentProviderService extends AbstractProviderService implements return defaultConfigMap; } - protected Map<String, DefaultConfig> getDefaultConfigs(String roleGroup) { - return metaInfoMap.get(getMetainfoMapKey(roleGroup)).defaultConfigs; + protected Map<String, DefaultConfig> getDefaultConfigs() { + return defaultConfigs; } private int getHeartbeatMonitorInterval() { @@ -1634,9 +1583,9 @@ public class AgentProviderService extends AbstractProviderService implements log.info("Status report: {}", status.toString()); if (status.getConfigs() != null) { - Application application = getMetaInfo(componentGroup).getApplication(); + Application application = getMetaInfo().getApplication(); - if (canAnyMasterPublishConfig(componentGroup) == false || canPublishConfig(componentGroup)) { + if (canAnyMasterPublishConfig() == false || canPublishConfig(componentGroup)) { // If no Master can explicitly publish then publish if its a master // Otherwise, wait till the master that can publish is ready @@ -1778,7 +1727,7 @@ public class AgentProviderService extends AbstractProviderService implements String hostNamePattern = "${THIS_HOST}"; Map<String, String> toPublish = new HashMap<String, String>(); - Application application = getMetaInfo(componentGroup).getApplication(); + Application application = getMetaInfo().getApplication(); for (Component component : application.getComponents()) { if (component.getName().equals(componentGroup)) { if (component.getComponentExports().size() > 0) { @@ -1829,8 +1778,8 @@ public class AgentProviderService extends AbstractProviderService implements String portVarFormat = "${site.%s}"; String hostNamePattern = "${" + compGroup + "_HOST}"; - List<ExportGroup> appExportGroups = getMetaInfo(compGroup).getApplication().getExportGroups(); - Component component = getApplicationComponent(compGroup); + List<ExportGroup> appExportGroups = getMetaInfo().getApplication().getExportGroups(); + Component component = getMetaInfo().getApplicationComponent(compGroup); if (component != null && SliderUtils.isSet(component.getCompExports()) && SliderUtils.isNotEmpty(appExportGroups)) { @@ -1932,11 +1881,7 @@ public class AgentProviderService extends AbstractProviderService implements * @return the component entry or null for no match */ protected Component getApplicationComponent(String roleGroup) { - Metainfo metainfo = getMetaInfo(roleGroup); - if (metainfo == null) { - return null; - } - return metainfo.getApplicationComponent(roleGroup); + return getMetaInfo().getApplicationComponent(roleGroup); } /** @@ -2005,9 +1950,9 @@ public class AgentProviderService extends AbstractProviderService implements * * @return true if the condition holds */ - protected boolean canAnyMasterPublishConfig(String roleGroup) { + protected boolean canAnyMasterPublishConfig() { if (canAnyMasterPublish == null) { - Application application = getMetaInfo(roleGroup).getApplication(); + Application application = getMetaInfo().getApplication(); if (application == null) { log.error("Malformed app definition: Expect application as root element in the metainfo.xml"); } else { @@ -2082,7 +2027,7 @@ public class AgentProviderService extends AbstractProviderService implements cmd.setPkg(pkg); Map<String, String> hostLevelParams = new TreeMap<String, String>(); hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getOption(JAVA_HOME, getJDKDir())); - hostLevelParams.put(PACKAGE_LIST, getPackageList(roleGroup)); + hostLevelParams.put(PACKAGE_LIST, getPackageList()); hostLevelParams.put(CONTAINER_ID, containerId); cmd.setHostLevelParams(hostLevelParams); @@ -2131,7 +2076,7 @@ public class AgentProviderService extends AbstractProviderService implements cmd.setComponentName(roleName); cmd.setRole(roleName); Map<String, String> hostLevelParams = new TreeMap<String, String>(); - hostLevelParams.put(PACKAGE_LIST, getPackageList(roleGroup)); + hostLevelParams.put(PACKAGE_LIST, getPackageList()); hostLevelParams.put(CONTAINER_ID, containerId); cmd.setHostLevelParams(hostLevelParams); @@ -2151,7 +2096,7 @@ public class AgentProviderService extends AbstractProviderService implements configurations.get("global").put("exec_cmd", effectiveCommand.getExec()); cmd.setHostname(getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME)); - cmd.addContainerDetails(roleGroup, getMetaInfo(roleGroup)); + cmd.addContainerDetails(roleGroup, getMetaInfo()); Map<String, String> dockerConfig = new HashMap<String, String>(); if(isYarnDockerContainer(roleGroup)){ @@ -2234,8 +2179,8 @@ public class AgentProviderService extends AbstractProviderService implements } } - private String getPackageList(String roleGroup) { - return getPackageListFromApplication(getMetaInfo(roleGroup).getApplication()); + private String getPackageList() { + return getPackageListFromApplication(getMetaInfo().getApplication()); } private void prepareExecutionCommand(ExecutionCommand cmd) { @@ -2400,7 +2345,7 @@ public class AgentProviderService extends AbstractProviderService implements private String getConfigFromMetaInfoWithAppConfigOverriding(String roleGroup, String configName){ ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); - String containerName = getApplicationComponent(roleGroup) + String containerName = getMetaInfo().getApplicationComponent(roleGroup) .getDockerContainers().get(0).getName(); String composedConfigName = null; String appConfigValue = null; @@ -2541,7 +2486,7 @@ public class AgentProviderService extends AbstractProviderService implements cmd.setConfigurations(configurations); // configurations.get("global").put("exec_cmd", startCommand.getExec()); - cmd.addContainerDetails(roleGroup, getMetaInfo(roleGroup)); + cmd.addContainerDetails(roleGroup, getMetaInfo()); log.info("Docker- command: {}", cmd.toString()); @@ -2551,7 +2496,7 @@ public class AgentProviderService extends AbstractProviderService implements private String getConfigFromMetaInfo(String roleGroup, String configName) { String result = null; - List<DockerContainer> containers = getApplicationComponent( + List<DockerContainer> containers = getMetaInfo().getApplicationComponent( roleGroup).getDockerContainers();// to support multi container per // component later log.debug("Docker- containers metainfo: {}", containers.toString()); @@ -2853,11 +2798,10 @@ public class AgentProviderService extends AbstractProviderService implements for (String configType : configs) { addNamedConfiguration(configType, appConf.getGlobalOptions().options, - configurations, tokens, containerId, roleName, - roleGroup); + configurations, tokens, containerId, roleName); if (appConf.getComponent(roleGroup) != null) { addNamedConfiguration(configType, appConf.getComponent(roleGroup).options, - configurations, tokens, containerId, roleName, roleGroup); + configurations, tokens, containerId, roleName); } } @@ -2900,32 +2844,15 @@ public class AgentProviderService extends AbstractProviderService implements tokens.put("${NN_HOST}", URI.create(nnuri).getHost()); tokens.put("${ZK_HOST}", appConf.get(OptionKeys.ZOOKEEPER_HOSTS)); tokens.put("${DEFAULT_ZK_PATH}", appConf.get(OptionKeys.ZOOKEEPER_PATH)); - String mapKey = getMetainfoMapKey(componentGroup); - String dataDirSuffix = ""; - if (!DEFAULT_METAINFO_MAP_KEY.equals(mapKey)) { - dataDirSuffix = "_" + mapKey.substring(0, mapKey.length()-1); - } else { - mapKey = ""; - } - mapKey = mapKey.toLowerCase(); tokens.put("${DEFAULT_DATA_DIR}", getAmState() .getInternalsSnapshot() .getGlobalOptions() - .getMandatoryOption(InternalKeys.INTERNAL_DATA_DIR_PATH) + dataDirSuffix); + .getMandatoryOption(InternalKeys.INTERNAL_DATA_DIR_PATH)); tokens.put("${JAVA_HOME}", appConf.get(AgentKeys.JAVA_HOME)); tokens.put("${COMPONENT_NAME}", componentName); - tokens.put("${COMPONENT_NAME.lc}", componentName.toLowerCase()); - tokens.put("${COMPONENT_PREFIX}", mapKey); - tokens.put("${COMPONENT_PREFIX.lc}", mapKey.toLowerCase()); if (!componentName.equals(componentGroup) && componentName.startsWith(componentGroup)) { tokens.put("${COMPONENT_ID}", componentName.substring(componentGroup.length())); } - tokens.put("${CLUSTER_NAME}", getClusterName()); - tokens.put("${CLUSTER_NAME.lc}", getClusterName().toLowerCase()); - tokens.put("${APP_NAME}", getClusterName()); - tokens.put("${APP_NAME.lc}", getClusterName().toLowerCase()); - tokens.put("${APP_COMPONENT_NAME}", componentName); - tokens.put("${APP_COMPONENT_NAME.lc}", componentName.toLowerCase()); return tokens; } @@ -2950,12 +2877,12 @@ public class AgentProviderService extends AbstractProviderService implements List<String> configList = new ArrayList<String>(); configList.add(GLOBAL_CONFIG_TAG); - List<ConfigFile> configFiles = getMetaInfo(roleGroup).getApplication().getConfigFiles(); + List<ConfigFile> configFiles = getMetaInfo().getApplication().getConfigFiles(); for (ConfigFile configFile : configFiles) { log.info("Expecting config type {}.", configFile.getDictionaryName()); configList.add(configFile.getDictionaryName()); } - for (Component component : getMetaInfo(roleGroup).getApplication().getComponents()) { + for (Component component : getMetaInfo().getApplication().getComponents()) { if (!component.getName().equals(roleGroup)) { continue; } @@ -2980,7 +2907,7 @@ public class AgentProviderService extends AbstractProviderService implements private void addNamedConfiguration(String configName, Map<String, String> sourceConfig, Map<String, Map<String, String>> configurations, Map<String, String> tokens, String containerId, - String roleName, String roleGroup) { + String roleName) { Map<String, String> config = new HashMap<String, String>(); if (configName.equals(GLOBAL_CONFIG_TAG)) { addDefaultGlobalConfig(config, containerId, roleName); @@ -3009,9 +2936,9 @@ public class AgentProviderService extends AbstractProviderService implements } //apply defaults only if the key is not present and value is not empty - if (getDefaultConfigs(roleGroup).containsKey(configName)) { + if (getDefaultConfigs().containsKey(configName)) { log.info("Adding default configs for type {}.", configName); - for (PropertyInfo defaultConfigProp : getDefaultConfigs(roleGroup).get(configName).getPropertyInfos()) { + for (PropertyInfo defaultConfigProp : getDefaultConfigs().get(configName).getPropertyInfos()) { if (!config.containsKey(defaultConfigProp.getName())) { if (!defaultConfigProp.getName().isEmpty() && defaultConfigProp.getValue() != null && http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c0f4e922/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java index ed5108c..cfcfc5d 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java @@ -35,9 +35,6 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; -import static org.apache.slider.common.SliderKeys.COMPONENT_SEPARATOR; -import static org.apache.slider.providers.agent.AgentKeys.DEFAULT_METAINFO_MAP_KEY; - /** * */ @@ -134,25 +131,4 @@ public class AgentUtils { return new DefaultConfigParser().parse(configStream); } - - static String getMetainfoMapKey(String roleGroup) { - if (roleGroup == null) { - return DEFAULT_METAINFO_MAP_KEY; - } - int lastIndex = roleGroup.lastIndexOf(COMPONENT_SEPARATOR); - if (lastIndex == -1) { - return DEFAULT_METAINFO_MAP_KEY; - } else { - return roleGroup.substring(0, lastIndex+1); - } - } - - static String getMetainfoComponentName(String roleGroup) { - int lastIndex = roleGroup.lastIndexOf(COMPONENT_SEPARATOR); - if (lastIndex == -1) { - return roleGroup; - } else { - return roleGroup.substring(lastIndex+1); - } - } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c0f4e922/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java index e2c879e..91f1259 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java @@ -40,13 +40,7 @@ public class ComponentCommandOrder { Map<Command, Map<String, List<ComponentState>>> dependencies = new HashMap<Command, Map<String, List<ComponentState>>>(); - public ComponentCommandOrder() {} - public ComponentCommandOrder(List<CommandOrder> commandOrders) { - mergeCommandOrders(commandOrders); - } - - void mergeCommandOrders(List<CommandOrder> commandOrders) { if (commandOrders != null && commandOrders.size() > 0) { for (CommandOrder commandOrder : commandOrders) { ComponentCommand componentCmd = getComponentCommand(commandOrder.getCommand()); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c0f4e922/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index 040f12b..b767059 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -882,8 +882,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService Configuration providerConf = providerService.loadProviderConfigurationInformation(confDir); - providerService.initializeApplicationConfiguration(instanceDefinition, - fs, null); + providerService.initializeApplicationConfiguration(instanceDefinition, fs); providerService.validateApplicationConfiguration(instanceDefinition, confDir, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c0f4e922/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java index 20ef068..f1e105a 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java @@ -73,7 +73,7 @@ public class AgentResource extends AbstractSliderResource { } @POST - @Path("/{agent_name: [a-zA-Z][a-zA-Z0-9_-]*}/register") + @Path("/{agent_name: [a-zA-Z][a-zA-Z_0-9]*}/register") @Consumes({MediaType.APPLICATION_JSON}) @Produces({MediaType.APPLICATION_JSON}) public RegistrationResponse register(Register registration, @@ -87,7 +87,7 @@ public class AgentResource extends AbstractSliderResource { } @POST - @Path("/{agent_name: [a-zA-Z][a-zA-Z0-9_-]*}/heartbeat") + @Path("/{agent_name: [a-zA-Z][a-zA-Z_0-9]*}/heartbeat") @Consumes(MediaType.APPLICATION_JSON) @Produces({MediaType.APPLICATION_JSON}) public HeartBeatResponse heartbeat(HeartBeat message, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c0f4e922/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component.json deleted file mode 100644 index 853310d..0000000 --- a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "schema": "http://example.org/specification/v2.0.0";, - "metadata": { - }, - "global": { - }, - "components": { - "test_sleep": { - "site.global.component_type": "external" - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c0f4e922/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources_external_component.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources_external_component.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources_external_component.json deleted file mode 100644 index bccea68..0000000 --- a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources_external_component.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "schema" : "http://example.org/specification/v2.0.0";, - "metadata" : { - }, - "global" : { - }, - "components": { - "slider-appmaster": { - "yarn.memory": "384" - }, - "SLEEP_100": { - "yarn.role.priority": "1", - "yarn.component.instances": "0", - "yarn.memory": "128" - }, - "SLEEP_LONG": { - "yarn.role.priority": "2", - "yarn.component.instances": "1", - "yarn.memory": "128" - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c0f4e922/slider-core/src/test/groovy/org/apache/slider/client/TestReplaceTokens.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestReplaceTokens.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestReplaceTokens.groovy index a35bcc0..c4c755d 100644 --- a/slider-core/src/test/groovy/org/apache/slider/client/TestReplaceTokens.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/client/TestReplaceTokens.groovy @@ -51,9 +51,8 @@ class TestReplaceTokens extends Assert { new JsonSerDeser<ConfTree>(ConfTree) def confTree = confTreeJsonSerDeser.fromResource(PACKAGE + app_configuration) SliderClient.replaceTokens(confTree, "testUser", "testCluster") - // cluster name is resolved later now - assert confTree.global.get("site.fs.defaultFS") == "hdfs://\${CLUSTER_NAME}:8020" - assert confTree.global.get("site.fs.default.name") == "hdfs://\${CLUSTER_NAME}:8020" + assert confTree.global.get("site.fs.defaultFS") == "hdfs://testCluster:8020" + assert confTree.global.get("site.fs.default.name") == "hdfs://testCluster:8020" assert confTree.global.get("site.hbase.user_name") == "testUser" assert confTree.global.get("site.hbase.another.user") == "testUser" http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c0f4e922/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy index 1ea2277..659af0e 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy @@ -175,8 +175,7 @@ class MockProviderService implements ProviderService { @Override void initializeApplicationConfiguration(AggregateConf instanceDefinition, - SliderFileSystem fileSystem, String roleGroup) - throws IOException, SliderException { + SliderFileSystem fileSystem) throws IOException, SliderException { } @Override http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c0f4e922/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java index 07d21d7..9fbb3d0 100644 --- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java +++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java @@ -471,7 +471,7 @@ public class TestAgentProviderService { metainfo.setApplication(application); doReturn(metainfo).when(mockAps).getApplicationMetainfo( any(SliderFileSystem.class), anyString()); - doReturn(metainfo).when(mockAps).getMetaInfo(anyString()); + doReturn(metainfo).when(mockAps).getMetaInfo(); @@ -664,7 +664,7 @@ public class TestAgentProviderService { hbaseMaster.setName("HBASE_MASTER"); application.addComponent(hbaseMaster); metainfo.setApplication(application); - doReturn(metainfo).when(mockAps).getMetaInfo(anyString()); + doReturn(metainfo).when(mockAps).getMetaInfo(); doReturn(metainfo).when(mockAps).getApplicationMetainfo( any(SliderFileSystem.class), anyString(), anyBoolean()); @@ -679,7 +679,7 @@ public class TestAgentProviderService { reg.setActualState(State.STARTED); mockAps.initializeApplicationConfiguration(instanceDefinition, - null, null); + null); RegistrationResponse resp = mockAps.handleRegistration(reg); Assert.assertEquals(0, resp.getResponseId()); @@ -735,7 +735,7 @@ public class TestAgentProviderService { metainfo.setApplication(application); doReturn(metainfo).when(mockAps).getApplicationMetainfo( any(SliderFileSystem.class), anyString(), anyBoolean()); - doReturn(metainfo).when(mockAps).getMetaInfo(anyString()); + doReturn(metainfo).when(mockAps).getMetaInfo(); doNothing().when(mockAps).addRoleRelatedTokens(anyMap()); Register reg = new Register(); @@ -748,7 +748,7 @@ public class TestAgentProviderService { // Simulating agent in INSTALLED state reg.setActualState(State.INSTALLED); - mockAps.initializeApplicationConfiguration(instanceDefinition, null, "HBASE_MASTER"); + mockAps.initializeApplicationConfiguration(instanceDefinition, null); RegistrationResponse resp = mockAps.handleRegistration(reg); Assert.assertEquals(0, resp.getResponseId()); @@ -853,7 +853,7 @@ public class TestAgentProviderService { AgentProviderService aps = createAgentProviderService(new Configuration()); AgentProviderService mockAps = Mockito.spy(aps); doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection()); - doReturn(metainfo).when(mockAps).getMetaInfo(anyString()); + doReturn(metainfo).when(mockAps).getMetaInfo(); Map<String, String> ports = new HashMap<String, String>(); ports.put("global.listen_port", "10010"); @@ -892,7 +892,7 @@ public class TestAgentProviderService { AgentProviderService aps = createAgentProviderService(new Configuration()); AgentProviderService mockAps = Mockito.spy(aps); doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection()); - doReturn(metainfo).when(mockAps).getMetaInfo(anyString()); + doReturn(metainfo).when(mockAps).getMetaInfo(); StateAccessForProviders access = createNiceMock(StateAccessForProviders.class); doReturn(access).when(mockAps).getAmState(); PublishedExportsSet pubExpSet = new PublishedExportsSet(); @@ -1005,7 +1005,7 @@ public class TestAgentProviderService { new MockContainerId(1), "cid"); AgentProviderService mockAps = Mockito.spy(aps); doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection()); - doReturn(metainfo).when(mockAps).getMetaInfo(anyString()); + doReturn(metainfo).when(mockAps).getMetaInfo(); doReturn(roleClusterNodeMap).when(mockAps).getRoleClusterNodeMapping(); StateAccessForProviders access = createNiceMock(StateAccessForProviders.class); doReturn(access).when(mockAps).getAmState(); @@ -1149,7 +1149,7 @@ public class TestAgentProviderService { AgentProviderService aps = createAgentProviderService(new Configuration()); AgentProviderService mockAps = Mockito.spy(aps); - doReturn(metainfo).when(mockAps).getMetaInfo(anyString()); + doReturn(metainfo).when(mockAps).getMetaInfo(); CommandScript script = mockAps.getScriptPathForMasterPackage("HBASE_MASTER"); Assert.assertEquals(script.getScript(), "scripts/hbase_master.py"); @@ -1178,24 +1178,22 @@ public class TestAgentProviderService { AgentProviderService aps1 = createAgentProviderService(new Configuration()); AgentProviderService mockAps = Mockito.spy(aps1); - doReturn(metainfo).when(mockAps).getMetaInfo(anyString()); + doReturn(metainfo).when(mockAps).getMetaInfo(); AgentProviderService mockAps2 = Mockito.spy(aps1); - doReturn(metainfo2).when(mockAps2).getMetaInfo(anyString()); + doReturn(metainfo2).when(mockAps2).getMetaInfo(); Assert.assertTrue(mockAps.isMaster(role_hm)); Assert.assertFalse(mockAps.isMaster(role_hrs)); Assert.assertTrue(mockAps.canPublishConfig(role_hm)); Assert.assertFalse(mockAps.canPublishConfig(role_hrs)); - Assert.assertTrue(mockAps.canAnyMasterPublishConfig(role_hm)); - Assert.assertTrue(mockAps.canAnyMasterPublishConfig(role_hrs)); + Assert.assertTrue(mockAps.canAnyMasterPublishConfig()); Assert.assertTrue(mockAps2.isMaster(role_hm)); Assert.assertFalse(mockAps2.isMaster(role_hrs)); Assert.assertTrue(mockAps2.canPublishConfig(role_hm)); Assert.assertFalse(mockAps2.canPublishConfig(role_hrs)); - Assert.assertTrue(mockAps2.canAnyMasterPublishConfig(role_hm)); - Assert.assertTrue(mockAps2.canAnyMasterPublishConfig(role_hrs)); + Assert.assertTrue(mockAps2.canAnyMasterPublishConfig()); } @Test @@ -1630,8 +1628,8 @@ public class TestAgentProviderService { expect(access.isApplicationLive()).andReturn(true).anyTimes(); doReturn("HOST1").when(mockAps).getClusterInfoPropertyValue(anyString()); - doReturn(metainfo).when(mockAps).getMetaInfo(anyString()); - doReturn(new HashMap<String, DefaultConfig>()).when(mockAps).getDefaultConfigs(anyString()); + doReturn(metainfo).when(mockAps).getMetaInfo(); + doReturn(new HashMap<String, DefaultConfig>()).when(mockAps).getDefaultConfigs(); Map<String, Map<String, ClusterNode>> roleClusterNodeMap = new HashMap<String, Map<String, ClusterNode>>(); Map<String, ClusterNode> container = new HashMap<String, ClusterNode>(); @@ -1697,7 +1695,7 @@ public class TestAgentProviderService { expect(access.isApplicationLive()).andReturn(true).anyTimes(); doReturn("HOST1").when(mockAps).getClusterInfoPropertyValue(anyString()); - doReturn(defaultConfigMap).when(mockAps).getDefaultConfigs(anyString()); + doReturn(defaultConfigMap).when(mockAps).getDefaultConfigs(); List<String> configurations = new ArrayList<String>(); configurations.add("hbase-site"); configurations.add("global"); @@ -1785,7 +1783,7 @@ public class TestAgentProviderService { expect(access.isApplicationLive()).andReturn(true).anyTimes(); doReturn("HOST1").when(mockAps).getClusterInfoPropertyValue(anyString()); - doReturn(defaultConfigMap).when(mockAps).getDefaultConfigs(anyString()); + doReturn(defaultConfigMap).when(mockAps).getDefaultConfigs(); List<String> configurations = new ArrayList<String>(); configurations.add("hbase-site"); configurations.add("global"); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c0f4e922/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy index 1cb6f0f..5de2b8e 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy @@ -38,6 +38,4 @@ interface ResourcePaths { String SLEEP_META = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/metainfo.json" String SLEEP_APPCONFIG = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/appConfig.json" - String EXTERNAL_RESOURCES = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/resources_external_component.json" - String EXTERNAL_APPCONFIG = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/appConfig_external_component.json" } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c0f4e922/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/ExternalComponentIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/ExternalComponentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/ExternalComponentIT.groovy deleted file mode 100644 index b5e0270..0000000 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/ExternalComponentIT.groovy +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.funtest.lifecycle - -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import org.apache.slider.api.ClusterDescription -import org.apache.slider.api.ResourceKeys -import org.apache.slider.common.SliderExitCodes -import org.apache.slider.common.SliderKeys -import org.apache.slider.common.params.Arguments -import org.apache.slider.common.params.SliderActions -import org.apache.slider.funtest.ResourcePaths -import org.apache.slider.funtest.framework.AgentCommandTestBase -import org.apache.slider.funtest.framework.FuntestProperties -import org.junit.After -import org.junit.Before -import org.junit.Test - -@CompileStatic -@Slf4j -public class ExternalComponentIT extends AgentCommandTestBase - implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { - - static String NAME = "test-external-component" - static String EXT_NAME = "test_sleep" - - static String BUILD_APPCONFIG = ResourcePaths.SLEEP_APPCONFIG - static String BUILD_RESOURCES = ResourcePaths.EXTERNAL_RESOURCES - static String BUILD_METAINFO = ResourcePaths.SLEEP_META - static String TEST_APPCONFIG = ResourcePaths.EXTERNAL_APPCONFIG - static String TEST_RESOURCES = ResourcePaths.EXTERNAL_RESOURCES - static String TEST_METAINFO = ResourcePaths.SLEEP_META - public static final String SLEEP_100 = "SLEEP_100" - public static final String SLEEP_LONG = "SLEEP_LONG" - public static final String EXT_SLEEP_100 = EXT_NAME + - SliderKeys.COMPONENT_SEPARATOR + SLEEP_100 - public static final String EXT_SLEEP_LONG = EXT_NAME + - SliderKeys.COMPONENT_SEPARATOR + SLEEP_LONG - - @Before - public void prepareCluster() { - setupCluster(NAME) - setupCluster(EXT_NAME) - } - - @After - public void destroyCluster() { - cleanup(NAME) - cleanup(EXT_NAME) - } - - @Test - public void testExternalComponent() throws Throwable { - assumeAgentTestsEnabled() - describe NAME - - slider(0, [ACTION_BUILD, EXT_NAME, ARG_METAINFO, BUILD_METAINFO, - ARG_TEMPLATE, BUILD_APPCONFIG, ARG_RESOURCES, BUILD_RESOURCES, - ARG_RES_COMP_OPT, SLEEP_100, ResourceKeys.COMPONENT_INSTANCES, "0", - ARG_RES_COMP_OPT, SLEEP_LONG, ResourceKeys.COMPONENT_INSTANCES, "1" - ]) - - slider(0, [ACTION_CREATE, NAME, ARG_METAINFO, TEST_METAINFO, - ARG_TEMPLATE, TEST_APPCONFIG, ARG_RESOURCES, TEST_RESOURCES]) - - ensureApplicationIsUp(NAME) - status(0, NAME) - - ClusterDescription cd = execStatus(NAME) - - assert 5 == cd.statistics.size() - assert cd.statistics.keySet().containsAll([SliderKeys.COMPONENT_AM, - SLEEP_100, SLEEP_LONG, - EXT_SLEEP_100, EXT_SLEEP_LONG]) - - expectLiveContainerCountReached(NAME, SLEEP_LONG, 1, - CONTAINER_LAUNCH_TIMEOUT) - expectLiveContainerCountReached(NAME, EXT_SLEEP_LONG, 1, - CONTAINER_LAUNCH_TIMEOUT) - expectLiveContainerCountReached(NAME, SLEEP_100, 0, - CONTAINER_LAUNCH_TIMEOUT) - expectLiveContainerCountReached(NAME, EXT_SLEEP_100, 0, - CONTAINER_LAUNCH_TIMEOUT) - - cleanup(NAME) - - // test overriding the number of instances of external components - describe NAME + "-2" - - slider(0, [ACTION_CREATE, NAME, ARG_METAINFO, TEST_METAINFO, - ARG_TEMPLATE, TEST_APPCONFIG, ARG_RESOURCES, TEST_RESOURCES, - ARG_RES_COMP_OPT, SLEEP_LONG, ResourceKeys.COMPONENT_INSTANCES, "0", - ARG_RES_COMP_OPT, EXT_SLEEP_LONG, ResourceKeys.COMPONENT_INSTANCES, "2", - ]) - - ensureApplicationIsUp(NAME) - status(0, NAME) - - cd = execStatus(NAME) - - assert 5 == cd.statistics.size() - assert cd.statistics.keySet().containsAll([SliderKeys.COMPONENT_AM, - SLEEP_100, SLEEP_LONG, - EXT_SLEEP_100, EXT_SLEEP_LONG]) - - expectLiveContainerCountReached(NAME, SLEEP_LONG, 0, - CONTAINER_LAUNCH_TIMEOUT) - expectLiveContainerCountReached(NAME, EXT_SLEEP_LONG, 2, - CONTAINER_LAUNCH_TIMEOUT) - expectLiveContainerCountReached(NAME, SLEEP_100, 0, - CONTAINER_LAUNCH_TIMEOUT) - expectLiveContainerCountReached(NAME, EXT_SLEEP_100, 0, - CONTAINER_LAUNCH_TIMEOUT) - - } -}
