Repository: incubator-slider Updated Branches: refs/heads/feature/SLIDER-875_uber_app a4dc574c9 -> 2893d34bf
SLIDER-875 address gour's comments and fix accumulo funtest failures Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2893d34b Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2893d34b Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2893d34b Branch: refs/heads/feature/SLIDER-875_uber_app Commit: 2893d34bf09fd605971c80af1445a76c4a4d0bbd Parents: a4dc574 Author: Billie Rinaldi <billie.rina...@gmail.com> Authored: Tue Aug 9 19:54:50 2016 -0700 Committer: Billie Rinaldi <billie.rina...@gmail.com> Committed: Tue Aug 9 19:54:50 2016 -0700 ---------------------------------------------------------------------- .../funtest/accumulo/AccumuloBasicIT.groovy | 3 ++ .../org/apache/slider/client/SliderClient.java | 9 ++++-- .../org/apache/slider/common/SliderKeys.java | 10 +++++-- .../apache/slider/common/tools/SliderUtils.java | 6 ++-- .../slider/core/build/InstanceBuilder.java | 29 ++++++++++---------- .../providers/agent/AgentClientProvider.java | 7 ++--- .../slider/providers/agent/AgentKeys.java | 2 +- .../sleep_cmd/appConfig_external_component.json | 2 +- .../appConfig_external_component_nested.json | 2 +- .../TestBuildExternalComponent.groovy | 6 ++-- 10 files changed, 42 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2893d34b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy index 56014d7..81d290a 100644 --- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy +++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy @@ -38,6 +38,8 @@ import org.apache.slider.funtest.framework.SliderShell import org.junit.Before import org.junit.Test +import java.util.regex.Pattern + import static org.apache.hadoop.registry.client.binding.RegistryUtils.currentUser import static org.apache.hadoop.registry.client.binding.RegistryUtils.servicePath @@ -90,6 +92,7 @@ class AccumuloBasicIT extends AccumuloAgentCommandTestBase { def keys = tree.credentials.get(jks) assert keys!=null, "jks specified in $PROVIDER_PROPERTY wasn't requested " + "in credentials" + jks = jks.replaceAll(Pattern.quote('${CLUSTER_NAME}'), clusterName) Path jksPath = ProviderUtils.unnestUri(new URI(jks)) if (clusterFS.exists(jksPath)) { clusterFS.delete(jksPath, false) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2893d34b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 372ec48..78e9b89 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -721,7 +721,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved( clustername, clusterDirectory); try { - checkForCredentials(getConfig(), instanceDefinition.getAppConf()); + checkForCredentials(getConfig(), instanceDefinition.getAppConf(), + clustername); } catch (IOException e) { sliderFileSystem.getFileSystem().delete(clusterDirectory, true); throw e; @@ -903,7 +904,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } protected static void checkForCredentials(Configuration conf, - ConfTree tree) throws IOException { + ConfTree tree, String clusterName) throws IOException { if (tree.credentials == null || tree.credentials.isEmpty()) { log.info("No credentials requested"); return; @@ -912,7 +913,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe BufferedReader br = null; try { for (Entry<String, List<String>> cred : tree.credentials.entrySet()) { - String provider = cred.getKey(); + String provider = cred.getKey() + .replaceAll(Pattern.quote("${CLUSTER_NAME}"), clusterName) + .replaceAll(Pattern.quote("${CLUSTER}"), clusterName); List<String> aliases = cred.getValue(); if (aliases == null || aliases.isEmpty()) { continue; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2893d34b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java index 3921324..2c9be09 100644 --- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java +++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java @@ -72,9 +72,15 @@ public interface SliderKeys extends SliderXmlConfKeys { /** * Key for component type. This MUST NOT be set in app_config/global {@value} */ - String COMPONENT_TYPE = "site.global.component_type"; - String EXTERNAL_COMPONENT = "external"; + String COMPONENT_TYPE_KEY = "site.global.component_type"; + /** + * A component type for an external app that has been predefined using the + * slider build command + */ + String COMPONENT_TYPE_EXTERNAL_APP = "external_app"; String COMPONENT_SEPARATOR = "-"; + String[] COMPONENT_KEYS_TO_SKIP = {"zookeeper.", "env.MALLOC_ARENA_MAX", + "site.fs.", "site.dfs."}; /** * Key for application version. This must be set in app_config/global {@value} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2893d34b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index b798d4d..aad8b45 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -492,11 +492,11 @@ public final class SliderUtils { FileSystem srcFS = FileSystem.get(srcFile.toUri(), conf); //list all paths in the src. if (!srcFS.exists(srcFile)) { - throw new FileNotFoundException("Source dir not found " + srcFile); + throw new FileNotFoundException("Source file not found " + srcFile); } if (!srcFS.isFile(srcFile)) { throw new FileNotFoundException( - "Source dir not a file " + srcFile); + "Source file not a file " + srcFile); } FileSystem destFS = FileSystem.get(destFile.toUri(), conf); if (destFS.exists(destFile)) { @@ -962,7 +962,7 @@ public final class SliderUtils { * @param prefixes prefixes to ignore * @return 'first' merged with the second */ - public static Map<String, String> mergeMapsIgnorePrefixes( + public static Map<String, String> mergeMapsIgnoreDuplicateKeysAndPrefixes( Map<String, String> first, Map<String, String> second, String... prefixes) { Preconditions.checkArgument(first != null, "Null 'first' value"); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2893d34b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java index 6a7975e..8155214 100644 --- a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java +++ b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java @@ -72,8 +72,8 @@ import static org.apache.slider.api.OptionKeys.ZOOKEEPER_QUORUM; import static org.apache.slider.api.RoleKeys.ROLE_PREFIX; import static org.apache.slider.common.SliderKeys.COMPONENT_AM; import static org.apache.slider.common.SliderKeys.COMPONENT_SEPARATOR; -import static org.apache.slider.common.SliderKeys.COMPONENT_TYPE; -import static org.apache.slider.common.SliderKeys.EXTERNAL_COMPONENT; +import static org.apache.slider.common.SliderKeys.COMPONENT_TYPE_EXTERNAL_APP; +import static org.apache.slider.common.SliderKeys.COMPONENT_TYPE_KEY; import static org.apache.slider.common.tools.SliderUtils.isClusternameValid; /** @@ -260,10 +260,11 @@ public class InstanceBuilder { } - private void getExternalComponents(ConfTreeOperations ops, - Set<String> externalComponents) throws BadConfigException { - if (ops.getGlobalOptions().get(COMPONENT_TYPE) != null) { - throw new BadConfigException(COMPONENT_TYPE + " must be " + + private Set<String> getExternalComponents(ConfTreeOperations ops) + throws BadConfigException { + Set<String> externalComponents = new HashSet<>(); + if (ops.getGlobalOptions().containsKey(COMPONENT_TYPE_KEY)) { + throw new BadConfigException(COMPONENT_TYPE_KEY + " must be " + "specified per-component, not in global"); } @@ -273,15 +274,13 @@ public class InstanceBuilder { continue; } Map<String, String> options = entry.getValue(); - if (EXTERNAL_COMPONENT.equals(options.get(COMPONENT_TYPE))) { + if (COMPONENT_TYPE_EXTERNAL_APP.equals(options.get(COMPONENT_TYPE_KEY))) { externalComponents.add(entry.getKey()); } } + return externalComponents; } - private static String[] PREFIXES_TO_SKIP = {"zookeeper.", - "env.MALLOC_ARENA_MAX", "site.fs.", "site.dfs."}; - private void mergeExternalComponent(ConfTreeOperations ops, ConfTreeOperations externalOps, String externalComponent, Integer priority) throws BadConfigException { @@ -297,11 +296,12 @@ public class InstanceBuilder { if (priority == null) { SliderUtils.mergeMaps(subComponentOps, ops.getComponent(externalComponent).options); - subComponentOps.remove(COMPONENT_TYPE); + subComponentOps.remove(COMPONENT_TYPE_KEY); } - SliderUtils.mergeMapsIgnorePrefixes(subComponentOps, - externalOps.getComponent(subComponent), PREFIXES_TO_SKIP); + SliderUtils.mergeMapsIgnoreDuplicateKeysAndPrefixes(subComponentOps, + externalOps.getComponent(subComponent), + SliderKeys.COMPONENT_KEYS_TO_SKIP); // add prefix to existing prefix String existingPrefix = subComponentOps.get(ROLE_PREFIX); @@ -348,8 +348,7 @@ public class InstanceBuilder { } } - Set<String> externalComponents = new HashSet<>(); - getExternalComponents(appConf, externalComponents); + Set<String> externalComponents = getExternalComponents(appConf); if (!externalComponents.isEmpty()) { log.info("Found external components {}", externalComponents); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2893d34b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java index 8c0a2e4..1567fc9 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java @@ -74,8 +74,6 @@ import java.util.Set; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; -import static org.apache.slider.providers.agent.AgentUtils.getMetainfoComponentName; - /** This class implements the client-side aspects of the agent deployer */ public class AgentClientProvider extends AbstractClientProvider implements AgentKeys, SliderKeys { @@ -184,7 +182,7 @@ public class AgentClientProvider extends AbstractClientProvider if (metaInfo != null) { Component componentDef = metaInfo.getApplicationComponent( - getMetainfoComponentName(name, + AgentUtils.getMetainfoComponentName(name, instanceDefinition.getAppConfOperations())); if (componentDef == null) { throw new BadConfigException( @@ -215,9 +213,8 @@ public class AgentClientProvider extends AbstractClientProvider // fileSystem may be null for tests if (metaInfo != null) { Component componentDef = metaInfo.getApplicationComponent( - getMetainfoComponentName(name, + AgentUtils.getMetainfoComponentName(name, instanceDefinition.getAppConfOperations())); - // already checked it wasn't null // ensure that intance count is 0 for client components if ("CLIENT".equals(componentDef.getCategory())) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2893d34b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java index 565d73d..8341af4 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java @@ -99,7 +99,7 @@ public interface AgentKeys { String KEY_CONTAINER_LAUNCH_DELAY = "container.launch.delay.sec"; String TEST_RELAX_VERIFICATION = "test.relax.validation"; - String DEFAULT_METAINFO_MAP_KEY = "DEFAULT"; + String DEFAULT_METAINFO_MAP_KEY = "DEFAULT_KEY"; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2893d34b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component.json index 853310d..b026b9d 100644 --- a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component.json +++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component.json @@ -6,7 +6,7 @@ }, "components": { "test_sleep": { - "site.global.component_type": "external" + "site.global.component_type": "external_app" } } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2893d34b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component_nested.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component_nested.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component_nested.json index f67f83a..a9972cf 100644 --- a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component_nested.json +++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component_nested.json @@ -6,7 +6,7 @@ }, "components": { "test-external-component": { - "site.global.component_type": "external" + "site.global.component_type": "external_app" } } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2893d34b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildExternalComponent.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildExternalComponent.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildExternalComponent.groovy index 8dd693e..13f5f3b 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildExternalComponent.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildExternalComponent.groovy @@ -67,7 +67,7 @@ class TestBuildExternalComponent extends AgentMiniClusterTestBase { SliderActions.ACTION_BUILD, parent1, [(echo): 1], - [ARG_COMP_OPT, clustername, COMPONENT_TYPE, EXTERNAL_COMPONENT, + [ARG_COMP_OPT, clustername, COMPONENT_TYPE_KEY, COMPONENT_TYPE_EXTERNAL_APP, ARG_RES_COMP_OPT, echo, ResourceKeys.COMPONENT_PRIORITY, "3"], true, false, @@ -97,8 +97,8 @@ class TestBuildExternalComponent extends AgentMiniClusterTestBase { SliderActions.ACTION_BUILD, parent2, [(echo): 1], - [ARG_COMP_OPT, clustername, COMPONENT_TYPE, EXTERNAL_COMPONENT, - ARG_COMP_OPT, parent1, COMPONENT_TYPE, EXTERNAL_COMPONENT, + [ARG_COMP_OPT, clustername, COMPONENT_TYPE_KEY, COMPONENT_TYPE_EXTERNAL_APP, + ARG_COMP_OPT, parent1, COMPONENT_TYPE_KEY, COMPONENT_TYPE_EXTERNAL_APP, ARG_RES_COMP_OPT, echo, ResourceKeys.COMPONENT_PRIORITY, "4"], true, false,