Repository: incubator-slider Updated Branches: refs/heads/develop e37572697 -> 3b32b8f7d
SLIDER-105. Agent provider service/client should reject component instance count when it exceeds the specified maximum or below the minimum SLIDER-274. Flex'ing with unknown role name results in BadConfigException Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3b32b8f7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3b32b8f7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3b32b8f7 Branch: refs/heads/develop Commit: 3b32b8f7d333bfc506b6539c90d16f000005a55c Parents: e375726 Author: Sumit Mohanty <[email protected]> Authored: Wed Oct 8 14:43:29 2014 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Wed Oct 8 14:43:38 2014 -0700 ---------------------------------------------------------------------- app-packages/hbase-win/metainfo.xml | 2 - app-packages/hbase/metainfo.xml | 4 - app-packages/hbase/pom.xml | 1 + app-packages/storm/metainfo.xml | 3 + .../org/apache/slider/client/SliderClient.java | 20 +++-- .../providers/AbstractClientProvider.java | 4 +- .../providers/agent/AgentClientProvider.java | 79 +++++++++++++++----- .../slider/providers/agent/AgentKeys.java | 2 +- .../providers/agent/AgentProviderService.java | 43 ++++++++--- .../agent/application/metadata/Component.java | 28 +++++++ .../agent/application/metadata/Metainfo.java | 19 +++++ .../slideram/SliderAMClientProvider.java | 4 +- .../server/appmaster/SliderAppMaster.java | 4 +- .../providers/agent/TestBuildBasicAgent.groovy | 65 ++++++++++++++-- .../agent/TestAgentClientProvider.java | 2 +- .../publisher/TestAgentProviderService.java | 1 + slider-core/src/test/python/metainfo.xml | 21 ++++++ .../providers/agent/tests/bad/resources-3.json | 13 ++++ .../providers/agent/tests/good/resources.json | 4 +- .../accumulo/AccumuloClientProvider.java | 4 +- .../accumulo/AccumuloProviderService.java | 2 +- .../providers/hbase/HBaseClientProvider.java | 4 +- .../providers/hbase/HBaseProviderService.java | 4 +- 23 files changed, 265 insertions(+), 68 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/app-packages/hbase-win/metainfo.xml ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/metainfo.xml b/app-packages/hbase-win/metainfo.xml index 17b4404..f0c5d4f 100644 --- a/app-packages/hbase-win/metainfo.xml +++ b/app-packages/hbase-win/metainfo.xml @@ -59,7 +59,6 @@ <name>HBASE_MASTER</name> <category>MASTER</category> <minInstanceCount>1</minInstanceCount> - <maxInstanceCount>2</maxInstanceCount> <appExports>QuickLinks-org.apache.slider.jmx,QuickLinks-org.apache.slider.monitor</appExports> <componentExports> <componentExport> @@ -81,7 +80,6 @@ <component> <name>HBASE_REGIONSERVER</name> <category>SLAVE</category> - <minInstanceCount>1</minInstanceCount> <commandScript> <script>scripts/hbase_regionserver.py</script> <scriptType>PYTHON</scriptType> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/app-packages/hbase/metainfo.xml ---------------------------------------------------------------------- diff --git a/app-packages/hbase/metainfo.xml b/app-packages/hbase/metainfo.xml index 921ee48..d5e07a7 100644 --- a/app-packages/hbase/metainfo.xml +++ b/app-packages/hbase/metainfo.xml @@ -81,7 +81,6 @@ <name>HBASE_MASTER</name> <category>MASTER</category> <minInstanceCount>1</minInstanceCount> - <maxInstanceCount>2</maxInstanceCount> <appExports>QuickLinks-org.apache.slider.jmx,QuickLinks-org.apache.slider.monitor,QuickLinks-org.apache.slider.metrics,QuickLinks-org.apache.slider.metrics.ui</appExports> <componentExports> <componentExport> @@ -113,7 +112,6 @@ <component> <name>HBASE_REST</name> <category>MASTER</category> - <minInstanceCount>0</minInstanceCount> <appExports>QuickLinks-org.apache.slider.hbase.rest</appExports> <commandScript> <script>scripts/hbase_rest.py</script> @@ -124,7 +122,6 @@ <component> <name>HBASE_THRIFT</name> <category>MASTER</category> - <minInstanceCount>0</minInstanceCount> <appExports>QuickLinks-org.apache.slider.hbase.thrift</appExports> <commandScript> <script>scripts/hbase_thrift.py</script> @@ -146,7 +143,6 @@ <component> <name>HBASE_CLIENT</name> <category>CLIENT</category> - <minInstanceCount>0</minInstanceCount> <commandScript> <script>scripts/hbase_client.py</script> <scriptType>PYTHON</scriptType> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/app-packages/hbase/pom.xml ---------------------------------------------------------------------- diff --git a/app-packages/hbase/pom.xml b/app-packages/hbase/pom.xml index be1b858..b85923b 100644 --- a/app-packages/hbase/pom.xml +++ b/app-packages/hbase/pom.xml @@ -32,6 +32,7 @@ <properties> <work.dir>package-tmp</work.dir> <app.package.name>${project.artifactId}-${project.version}</app.package.name> + <bld.version>0.51.0-incubating-SNAPSHOT</bld.version> </properties> <profiles> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/app-packages/storm/metainfo.xml ---------------------------------------------------------------------- diff --git a/app-packages/storm/metainfo.xml b/app-packages/storm/metainfo.xml index 1579174..e07ae92 100644 --- a/app-packages/storm/metainfo.xml +++ b/app-packages/storm/metainfo.xml @@ -78,6 +78,7 @@ <category>MASTER</category> <autoStartOnFailure>true</autoStartOnFailure> <appExports>QuickLinks-nimbus.host_port,QuickLinks-org.apache.slider.metrics.ui,QuickLinks-org.apache.slider.metrics</appExports> + <maxInstanceCount>1</maxInstanceCount> <commandScript> <script>scripts/nimbus.py</script> <scriptType>PYTHON</scriptType> @@ -107,6 +108,7 @@ <category>MASTER</category> <publishConfig>true</publishConfig> <appExports>QuickLinks-org.apache.slider.monitor,QuickLinks-org.apache.slider.jmx</appExports> + <maxInstanceCount>1</maxInstanceCount> <autoStartOnFailure>true</autoStartOnFailure> <commandScript> <script>scripts/ui_server.py</script> @@ -119,6 +121,7 @@ <name>DRPC_SERVER</name> <category>MASTER</category> <autoStartOnFailure>true</autoStartOnFailure> + <maxInstanceCount>1</maxInstanceCount> <commandScript> <script>scripts/drpc_server.py</script> <scriptType>PYTHON</scriptType> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 8d2bcfd..f15bc10 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -841,8 +841,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // providers to validate what there is AggregateConf instanceDescription = builder.getInstanceDescription(); - validateInstanceDefinition(sliderAM, instanceDescription); - validateInstanceDefinition(provider, instanceDescription); + validateInstanceDefinition(sliderAM, instanceDescription, sliderFileSystem); + validateInstanceDefinition(provider, instanceDescription, sliderFileSystem); try { persistInstanceDefinition(overwrite, appconfdir, builder); } catch (LockAcquireFailedException e) { @@ -1973,13 +1973,17 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe resources); } SliderAMClientProvider sliderAM = new SliderAMClientProvider(getConfig()); + AbstractClientProvider provider = createClientProvider( + instanceDefinition.getInternalOperations().getGlobalOptions().getMandatoryOption( + InternalKeys.INTERNAL_PROVIDER_NAME)); // slider provider to validate what there is - validateInstanceDefinition(sliderAM, instanceDefinition); - + validateInstanceDefinition(sliderAM, instanceDefinition, sliderFileSystem); + validateInstanceDefinition(provider, instanceDefinition, sliderFileSystem); + int exitCode = EXIT_FALSE; // save the specification try { - InstanceIO.updateInstanceDefinition(sliderFileSystem, clusterDirectory,instanceDefinition); + InstanceIO.updateInstanceDefinition(sliderFileSystem, clusterDirectory, instanceDefinition); } catch (LockAcquireFailedException e) { // lock failure log.debug("Failed to lock dir {}", clusterDirectory, e); @@ -2008,12 +2012,12 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe * @throws SliderException if invalid. */ protected void validateInstanceDefinition(AbstractClientProvider provider, - AggregateConf instanceDefinition) throws SliderException { + AggregateConf instanceDefinition, SliderFileSystem fs) throws SliderException { try { - provider.validateInstanceDefinition(instanceDefinition); + provider.validateInstanceDefinition(instanceDefinition, fs); } catch (SliderException e) { //problem, reject it - log.info("Error {} validating application instance definition ", e); + log.info("Error {} validating application instance definition ", e.getMessage()); log.debug("Error validating application instance definition ", e); log.info(instanceDefinition.toString()); throw e; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java index 7c2c7f4..f89f842 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java +++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java @@ -66,7 +66,7 @@ public abstract class AbstractClientProvider extends Configured { * @param instanceDefinition instance definition * @throws SliderException if the configuration is not valid */ - public void validateInstanceDefinition(AggregateConf instanceDefinition) throws + public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws SliderException { List<ProviderRole> roles = getRoles(); @@ -206,7 +206,7 @@ public abstract class AbstractClientProvider extends Configured { Path generatedConfDirPath, boolean secure) throws SliderException, IOException { - validateInstanceDefinition(instanceDefinition); + validateInstanceDefinition(instanceDefinition, sliderFileSystem); } /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java index 4c2f213..d7ae245 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java @@ -36,6 +36,7 @@ import org.apache.slider.providers.AbstractClientProvider; import org.apache.slider.providers.ProviderRole; import org.apache.slider.providers.ProviderUtils; import org.apache.slider.providers.agent.application.metadata.Application; +import org.apache.slider.providers.agent.application.metadata.Component; import org.apache.slider.providers.agent.application.metadata.Metainfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -109,25 +110,55 @@ public class AgentClientProvider extends AbstractClientProvider } @Override - public void validateInstanceDefinition(AggregateConf instanceDefinition) throws + public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws SliderException { - super.validateInstanceDefinition(instanceDefinition); + super.validateInstanceDefinition(instanceDefinition, fs); log.debug("Validating conf {}", instanceDefinition); ConfTreeOperations resources = instanceDefinition.getResourceOperations(); - ConfTreeOperations appConf = - instanceDefinition.getAppConfOperations(); providerUtils.validateNodeCount(instanceDefinition, ROLE_NODE, 0, -1); + try { + // Validate the app definition + instanceDefinition.getAppConfOperations(). + getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF); + } catch (BadConfigException bce) { + throw new BadConfigException("Application definition must be provided. " + bce.getMessage()); + } + String appDef = instanceDefinition.getAppConfOperations(). + getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF); + log.info("Validating app definition {}", appDef); + String extension = appDef.substring(appDef.lastIndexOf(".") + 1, appDef.length()); + if (!"zip".equalsIgnoreCase(extension)) { + throw new BadConfigException("App definition must be packaged as a .zip file. File provided is " + appDef); + } + Set<String> names = resources.getComponentNames(); names.remove(SliderKeys.COMPONENT_AM); Map<Integer, String> priorityMap = new HashMap<Integer, String>(); + + Metainfo metaInfo = null; + if (fs != null) { + try { + metaInfo = AgentUtils.getApplicationMetainfo(fs, appDef); + } catch (IOException ioe) { + // Ignore missing metainfo file for now + log.info("Missing metainfo.xml {}", ioe.getMessage()); + } + } + for (String name : names) { MapOperations component = resources.getMandatoryComponent(name); - // Validate count against the metainfo.xml + if (metaInfo != null) { + Component componentDef = metaInfo.getApplicationComponent(name); + if (componentDef == null) { + throw new BadConfigException( + "Component %s is not a member of application.", name); + } + } int priority = component.getMandatoryOptionInt(ResourceKeys.COMPONENT_PRIORITY); @@ -150,22 +181,34 @@ public class AgentClientProvider extends AbstractClientProvider priorityMap.put(priority, name); } - try { - // Validate the app definition - instanceDefinition.getAppConfOperations(). - getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF); - } catch (BadConfigException bce) { - throw new BadConfigException("Application definition must be provided. " + bce.getMessage()); - } - String appDef = instanceDefinition.getAppConfOperations(). - getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF); - log.info("Validating app definition {}", appDef); - String extension = appDef.substring(appDef.lastIndexOf(".") + 1, appDef.length()); - if (!"zip".equalsIgnoreCase(extension)) { - throw new BadConfigException("App definition must be packaged as a .zip file. File provided is " + appDef); + // fileSystem may be null for tests + if (metaInfo != null) { + for (String name : names) { + Component componentDef = metaInfo.getApplicationComponent(name); + if (componentDef == null) { + throw new BadConfigException( + "Component %s is not a member of application.", name); + } + + MapOperations componentConfig = resources.getMandatoryComponent(name); + int count = + componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES); + int definedMinCount = componentDef.getMinInstanceCountInt(); + int definedMaxCount = componentDef.getMaxInstanceCountInt(); + if (count < definedMinCount || count > definedMaxCount) { + throw new BadConfigException("Component %s, %s value %d out of range. " + + "Expected minimum is %d and maximum is %d", + name, + ResourceKeys.COMPONENT_INSTANCES, + count, + definedMinCount, + definedMaxCount); + } + } } } + @Override public void prepareAMAndConfigForLaunch(SliderFileSystem fileSystem, Configuration serviceConf, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java index b30c18c..2612865 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java @@ -24,7 +24,7 @@ package org.apache.slider.providers.agent; public interface AgentKeys { String PROVIDER_AGENT = "agent"; - String ROLE_NODE = "node"; + String ROLE_NODE = "echo"; /** * {@value} */ http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java index 88c8709..a200e19 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java @@ -34,6 +34,7 @@ import org.apache.slider.api.ClusterDescriptionKeys; import org.apache.slider.api.ClusterNode; import org.apache.slider.api.InternalKeys; import org.apache.slider.api.OptionKeys; +import org.apache.slider.api.ResourceKeys; import org.apache.slider.api.StatusKeys; import org.apache.slider.common.SliderKeys; import org.apache.slider.common.tools.SliderFileSystem; @@ -189,7 +190,35 @@ public class AgentProviderService extends AbstractProviderService implements public void validateInstanceDefinition(AggregateConf instanceDefinition) throws SliderException { - clientProvider.validateInstanceDefinition(instanceDefinition); + clientProvider.validateInstanceDefinition(instanceDefinition, null); + + ConfTreeOperations resources = + instanceDefinition.getResourceOperations(); + + Set<String> names = resources.getComponentNames(); + names.remove(SliderKeys.COMPONENT_AM); + for (String name : names) { + Component componentDef = getMetainfo().getApplicationComponent(name); + if (componentDef == null) { + throw new BadConfigException( + "Component %s is not a member of application.", name); + } + + MapOperations componentConfig = resources.getMandatoryComponent(name); + int count = + componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES); + int definedMinCount = componentDef.getMinInstanceCountInt(); + int definedMaxCount = componentDef.getMaxInstanceCountInt(); + if (count < definedMinCount || count > definedMaxCount) { + throw new BadConfigException("Component %s, %s value %d out of range. " + + "Expected minimum is %d and maximum is %d", + name, + ResourceKeys.COMPONENT_INSTANCES, + count, + definedMinCount, + definedMaxCount); + } + } } // Reads the metainfo.xml in the application package and loads it @@ -1073,17 +1102,7 @@ public class AgentProviderService extends AbstractProviderService implements * @return */ protected Component getApplicationComponent(String roleName) { - Application application = getMetainfo().getApplication(); - if (application == null) { - log.error("Malformed app definition: Expect application as the top level element for metainfo.xml"); - } else { - for (Component component : application.getComponents()) { - if (component.getName().equals(roleName)) { - return component; - } - } - } - return null; + return getMetainfo().getApplicationComponent(roleName); } /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java index 1117ff1..9f3dd0f 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java @@ -16,6 +16,10 @@ */ package org.apache.slider.providers.agent.application.metadata; +import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.core.exceptions.BadConfigException; +import org.apache.slider.core.exceptions.SliderException; + import java.util.ArrayList; import java.util.List; @@ -82,6 +86,30 @@ public class Component { return minInstanceCount; } + public int getMinInstanceCountInt() throws BadConfigException { + if (SliderUtils.isUnset(minInstanceCount)) { + return 0; + } + + try { + return Integer.parseInt(minInstanceCount); + } catch (NumberFormatException nfe) { + throw new BadConfigException(nfe, "Invalid value for minInstanceCount for %s", name); + } + } + + public int getMaxInstanceCountInt() throws BadConfigException { + if (SliderUtils.isUnset(maxInstanceCount)) { + return Integer.MAX_VALUE; + } + + try { + return Integer.parseInt(maxInstanceCount); + } catch (NumberFormatException nfe) { + throw new BadConfigException(nfe, "Invalid value for maxInstanceCount for %s", name); + } + } + public void setMinInstanceCount(String minInstanceCount) { this.minInstanceCount = minInstanceCount; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java index 2455e8e..b34cba1 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java @@ -16,10 +16,16 @@ */ package org.apache.slider.providers.agent.application.metadata; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * Application metainfo uber class */ public class Metainfo { + protected static final Logger log = + LoggerFactory.getLogger(Metainfo.class); + String schemaVersion; Application application; @@ -41,4 +47,17 @@ public class Metainfo { public void setApplication(Application application) { this.application = application; } + + public Component getApplicationComponent(String roleName) { + if (application == null) { + log.error("Malformed app definition: Expect application as the top level element for metainfo.xml"); + } else { + for (Component component : application.getComponents()) { + if (component.getName().equals(roleName)) { + return component; + } + } + } + return null; + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java index 748e5f1..aa1312c 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java +++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java @@ -139,10 +139,10 @@ public class SliderAMClientProvider extends AbstractClientProvider * @param instanceDefinition instance definition * @throws SliderException if the configuration is not valid */ - public void validateInstanceDefinition(AggregateConf instanceDefinition) throws + public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws SliderException { - super.validateInstanceDefinition(instanceDefinition); + super.validateInstanceDefinition(instanceDefinition, fs); // make sure there is no negative entry in the instance count Map<String, Map<String, String>> instanceMap = http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index 50fc265..31a9a28 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -811,9 +811,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService //Give the provider restricted access to the state, registry providerService.bind(stateForProviders, registry, actionQueues, - liveContainers); + liveContainers); sliderAMProvider.bind(stateForProviders, registry, actionQueues, - liveContainers); + liveContainers); // chaos monkey maybeStartMonkey(); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy index 2f6a15e..d0129c5 100644 --- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy @@ -79,7 +79,7 @@ class TestBuildBasicAgent extends AgentTestBase { true, false) buildAgentCluster("test_build_basic_agent_node_only", - [(ROLE_NODE): 5], + [(ROLE_NODE): 1], [ ARG_OPTION, CONTROLLER_URL, "http://localhost";, ARG_PACKAGE, ".", @@ -96,7 +96,7 @@ class TestBuildBasicAgent extends AgentTestBase { def rs = "hbase-rs" ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername, [ - (ROLE_NODE): 5, + (ROLE_NODE): 1, (master): 1, (rs): 5 ], @@ -138,7 +138,7 @@ class TestBuildBasicAgent extends AgentTestBase { def name2 = clustername + "-2" buildAgentCluster(name2, [ - (ROLE_NODE): 5, + (ROLE_NODE): 2, "role3": 1, "newnode": 5 ], @@ -154,6 +154,26 @@ class TestBuildBasicAgent extends AgentTestBase { } catch (BadConfigException expected) { } + try { + launcher = buildAgentCluster(clustername, + [ + (ROLE_NODE): 4, + ], + [ + ARG_OPTION, CONTROLLER_URL, "http://localhost";, + ARG_OPTION, PACKAGE_PATH, ".", + ARG_OPTION, APP_DEF, "file://" + getAppDef().absolutePath, + ARG_OPTION, AGENT_CONF, "file://" + getAgentConf().absolutePath, + ARG_COMP_OPT, ROLE_NODE, SCRIPT_PATH, "agent/scripts/agent.py", + ARG_RES_COMP_OPT, ROLE_NODE, ResourceKeys.COMPONENT_PRIORITY, "1", + ], + true, false, + false) + failWithBuildSucceeding(ROLE_NODE, "too many instances") + } catch (BadConfigException expected) { + assert expected.message.contains("Expected minimum is 1 and maximum is 2") + assert expected.message.contains("Component echo, yarn.component.instances value 4 out of range.") + } //duplicate priorities try { def name3 = clustername + "-3" @@ -212,13 +232,13 @@ class TestBuildBasicAgent extends AgentTestBase { def name5 = clustername + "-5" buildAgentCluster(name5, [ - "role": 1, + "hbase-rs": 1, ], [ ARG_OPTION, APP_DEF, "file://" + getAppDef().absolutePath, ARG_OPTION, AGENT_CONF, "file://" + getAgentConf().absolutePath, ARG_PACKAGE, ".", - ARG_RES_COMP_OPT, "role", ResourceKeys.COMPONENT_PRIORITY, "3", + ARG_RES_COMP_OPT, "hbase-rs", ResourceKeys.COMPONENT_PRIORITY, "3", ], true, false, false) @@ -240,7 +260,7 @@ class TestBuildBasicAgent extends AgentTestBase { def rs = "hbase-rs" ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername, [ - (ROLE_NODE): 5, + (ROLE_NODE): 2, (master): 1, (rs): 5 ], @@ -279,7 +299,7 @@ class TestBuildBasicAgent extends AgentTestBase { // change master priority and rs instances through update action ServiceLauncher<SliderClient> launcher2 = updateAgentCluster(clustername, [ - (ROLE_NODE): 5, + (ROLE_NODE): 2, (master): 1, (rs): 6 ], @@ -357,6 +377,37 @@ class TestBuildBasicAgent extends AgentTestBase { } @Test + public void testBadAgentArgs_Unknown_Component() throws Throwable { + String clustername = createMiniCluster( + "", + configuration, + 1, + 1, + 1, + true, + false) + + try { + def badArgs1 = "test_bad_agent_unk_comp" + buildAgentCluster(clustername, + [:], + [ + ARG_OPTION, CONTROLLER_URL, "http://localhost";, + ARG_PACKAGE, ".", + ARG_OPTION, APP_DEF, "file://" + appDef.absolutePath, + ARG_RESOURCES, TEST_FILES + "bad/resources-3.json", + ARG_TEMPLATE, TEST_FILES + "good/appconf.json" + ], + true, false, + false) + failWithBuildSucceeding(badArgs1, "bad component type node") + } catch (BadConfigException expected) { + log.info("Expected failure.", expected) + assert expected.message.contains("Component node is not a member of application") + } + } + + @Test public void testSubmitToSpecificQueue() throws Throwable { String clustername = createMiniCluster( "", http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java index 4cb35aa..0bea8fa 100644 --- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java +++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java @@ -67,7 +67,7 @@ public class TestAgentClientProvider { AggregateConf instanceDefinition = new AggregateConf(); try { - provider.validateInstanceDefinition(instanceDefinition); + provider.validateInstanceDefinition(instanceDefinition, null); Assert.assertFalse("Should fail with BadConfigException", true); } catch (BadConfigException e) { log.info(e.toString()); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java index eb368e3..a49a560 100644 --- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java +++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java @@ -17,6 +17,7 @@ package org.apache.slider.server.appmaster.web.rest.publisher; import org.apache.hadoop.yarn.api.records.Container; +import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.providers.agent.AgentProviderService; import org.apache.slider.server.appmaster.actions.QueueAccess; import org.apache.slider.server.appmaster.state.StateAccessForProviders; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-core/src/test/python/metainfo.xml ---------------------------------------------------------------------- diff --git a/slider-core/src/test/python/metainfo.xml b/slider-core/src/test/python/metainfo.xml index 09b314e..cf4afe1 100644 --- a/slider-core/src/test/python/metainfo.xml +++ b/slider-core/src/test/python/metainfo.xml @@ -27,6 +27,27 @@ <minHadoopVersion>2.1.0</minHadoopVersion> <components> <component> + <name>hbase-rs</name> + <category>MASTER</category> + <minInstanceCount>0</minInstanceCount> + <commandScript> + <script>echo.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + <component> + <name>hbase-master</name> + <category>MASTER</category> + <minInstanceCount>0</minInstanceCount> + <maxInstanceCount>2</maxInstanceCount> + <commandScript> + <script>echo.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + <component> <name>echo</name> <category>MASTER</category> <minInstanceCount>1</minInstanceCount> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/bad/resources-3.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/bad/resources-3.json b/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/bad/resources-3.json new file mode 100644 index 0000000..625b10e --- /dev/null +++ b/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/bad/resources-3.json @@ -0,0 +1,13 @@ +{ + "schema": "http://example.org/specification/v2.0.0";, + + "global": { + }, + "components": { + "node": { + "yarn.memory": "256", + "yarn.component.instances": "2", + "yarn.role.priority":"1" + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/good/resources.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/good/resources.json b/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/good/resources.json index f6111a0..83cdb99 100644 --- a/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/good/resources.json +++ b/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/good/resources.json @@ -4,9 +4,9 @@ "global": { }, "components": { - "node": { + "echo": { "yarn.memory": "256", - "yarn.component.instances": "5", + "yarn.component.instances": "2", "yarn.role.priority":"1" } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloClientProvider.java ---------------------------------------------------------------------- diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloClientProvider.java b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloClientProvider.java index 7f99573..8fe5170 100644 --- a/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloClientProvider.java +++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloClientProvider.java @@ -300,9 +300,9 @@ public class AccumuloClientProvider extends AbstractClientProvider implements } @Override - public void validateInstanceDefinition(AggregateConf instanceDefinition) throws + public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws SliderException { - super.validateInstanceDefinition(instanceDefinition); + super.validateInstanceDefinition(instanceDefinition, fs); ConfTreeOperations resources = instanceDefinition.getResourceOperations(); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java ---------------------------------------------------------------------- diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java index b8f4c00..c1bc9fe 100644 --- a/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java +++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java @@ -100,7 +100,7 @@ public class AccumuloProviderService extends AbstractProviderService implements @Override public void validateInstanceDefinition(AggregateConf instanceDefinition) throws SliderException { - clientProvider.validateInstanceDefinition(instanceDefinition); + clientProvider.validateInstanceDefinition(instanceDefinition, null); } @Override http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java index 9ad872f..81297fd 100644 --- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java +++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java @@ -224,9 +224,9 @@ public class HBaseClientProvider extends AbstractClientProvider implements * @param instanceDefinition instance definition */ @Override - public void validateInstanceDefinition(AggregateConf instanceDefinition) throws + public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws SliderException { - super.validateInstanceDefinition(instanceDefinition); + super.validateInstanceDefinition(instanceDefinition, fs); ConfTreeOperations resources = instanceDefinition.getResourceOperations(); Set<String> unknownRoles = resources.getComponentNames(); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java index f75a6c7..4228def 100644 --- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java +++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java @@ -106,9 +106,9 @@ public class HBaseProviderService extends AbstractProviderService * @param instanceDefinition the instance definition to validate */ @Override // Client and Server - public void validateInstanceDefinition(AggregateConf instanceDefinition) + public void validateInstanceDefinition(AggregateConf instanceDefinition) throws SliderException { - clientProvider.validateInstanceDefinition(instanceDefinition); + clientProvider.validateInstanceDefinition(instanceDefinition, null); } @Override
