SLIDER-470 slider appears to support negative component counts: flex checks and in AM (not tested that)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/cbf3a634 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/cbf3a634 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/cbf3a634 Branch: refs/heads/develop Commit: cbf3a634b82be14c714d05d59b10ee9959fe08f6 Parents: 483ab95 Author: Steve Loughran <[email protected]> Authored: Thu Oct 2 14:59:02 2014 -0700 Committer: Steve Loughran <[email protected]> Committed: Thu Oct 2 14:59:02 2014 -0700 ---------------------------------------------------------------------- .../org/apache/slider/client/SliderClient.java | 18 ++-- .../apache/slider/core/conf/AggregateConf.java | 20 +++++ .../apache/slider/providers/ProviderCore.java | 4 +- .../TestFreezeThawFlexStandaloneAM.groovy | 94 ++++++++++++++++++++ .../TestFreezeThawMasterlessAM.groovy | 92 ------------------- .../standalone/TestStandaloneAgentAM.groovy | 2 +- ...ndaloneClusterNegativeComponentCounts.groovy | 79 ---------------- .../slider/providers/agent/TestAgentEcho.groovy | 18 +++- 8 files changed, 140 insertions(+), 187 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/cbf3a634/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 05ef532..dc81f98 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -2053,29 +2053,24 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved( clustername, clusterDirectory); - SliderAMClientProvider sliderAM = new SliderAMClientProvider(getConfig()); - // provider to validate what there is - SliderAMClientProvider provider = sliderAM; - validateInstanceDefinition(provider, instanceDefinition); ConfTreeOperations resources = instanceDefinition.getResourceOperations(); for (Map.Entry<String, Integer> entry : roleInstances.entrySet()) { String role = entry.getKey(); int count = entry.getValue(); - if (count < 0) { - throw new BadCommandArgumentsException("Requested number of " + role - + " instances is out of range"); - } resources.getOrAddComponent(role).put(ResourceKeys.COMPONENT_INSTANCES, Integer.toString(count)); - log.debug("Flexed cluster specification ( {} -> {}) : \n{}", role, count, resources); } + SliderAMClientProvider sliderAM = new SliderAMClientProvider(getConfig()); + // slider provider to validate what there is + validateInstanceDefinition(sliderAM, instanceDefinition); + int exitCode = EXIT_FALSE; // save the specification try { @@ -2083,10 +2078,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } catch (LockAcquireFailedException e) { // lock failure log.debug("Failed to lock dir {}", clusterDirectory, e); - log.warn("Failed to save new resource definition to {} : {}", clusterDirectory, - e.toString()); - - + log.warn("Failed to save new resource definition to {} : {}", clusterDirectory, e); } // now see if it is actually running and tell it about the update if it is http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/cbf3a634/slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java b/slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java index 02669cd..e9e0dc9 100644 --- a/slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java +++ b/slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java @@ -23,6 +23,8 @@ import org.codehaus.jackson.annotate.JsonIgnore; import org.codehaus.jackson.annotate.JsonIgnoreProperties; import org.codehaus.jackson.map.annotate.JsonSerialize; +import java.io.IOException; + /** * Aggregate Configuration. * @@ -58,6 +60,24 @@ public final class AggregateConf { setInternal(internal); } + /** + * Take a snapshot of the configuration + * @param instanceDefinition source + * @throws IOException marshalling/copying problems + */ + public AggregateConf(AggregateConf instanceDefinition) throws IOException { + ConfTreeOperations resourcesSnapshot = + ConfTreeOperations.fromInstance(instanceDefinition.getResources()); + ConfTreeOperations appConfSnapshot = + ConfTreeOperations.fromInstance(instanceDefinition.getAppConf()); + ConfTreeOperations internalsSnapshot = + ConfTreeOperations.fromInstance(instanceDefinition.getInternal()); + //build a new aggregate from the snapshots + setResources(resourcesSnapshot.confTree); + setAppConf(appConfSnapshot.confTree); + setInternal(internalsSnapshot.confTree); + } + public void setResources(ConfTree resources) { this.resources = resources; resourceOperations = new ConfTreeOperations(resources); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/cbf3a634/slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java index 65d2138..9767430 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java +++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java @@ -20,6 +20,7 @@ package org.apache.slider.providers; import org.apache.hadoop.conf.Configuration; import org.apache.slider.core.conf.AggregateConf; +import org.apache.slider.core.conf.ConfTree; import org.apache.slider.core.exceptions.SliderException; import java.util.List; @@ -28,7 +29,7 @@ public interface ProviderCore { String getName(); List<ProviderRole> getRoles(); - + Configuration getConf(); /** @@ -38,4 +39,5 @@ public interface ProviderCore { */ void validateInstanceDefinition(AggregateConf instanceDefinition) throws SliderException; + } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/cbf3a634/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawFlexStandaloneAM.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawFlexStandaloneAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawFlexStandaloneAM.groovy new file mode 100644 index 0000000..7708502 --- /dev/null +++ b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawFlexStandaloneAM.groovy @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.agent.freezethaw + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import org.apache.hadoop.fs.FileSystem as HadoopFS +import org.apache.hadoop.fs.Path +import org.apache.hadoop.yarn.conf.YarnConfiguration +import org.apache.slider.agent.AgentMiniClusterTestBase +import org.apache.slider.client.SliderClient +import org.apache.slider.common.tools.SliderUtils +import org.apache.slider.core.main.ServiceLauncher +import org.junit.Test + +/** + * stop and start an AM + */ +@CompileStatic +@Slf4j + +class TestFreezeThawFlexStandaloneAM extends AgentMiniClusterTestBase { + + File getConfDirFile() { + return new File("target/testFreezeThawFlexStandaloneAM/conf") + } + + @Override + String getConfDir() { + return confDirFile.toURI().toString() + } + + @Test + public void testFreezeThawFlexStandaloneAM() throws Throwable { + YarnConfiguration conf = configuration + String clustername = createMiniCluster("", conf, 1, 1, 1, true, false) + + describe "create a standalone AM, stop it, start it" + //copy the confdir somewhere + Path resConfPath = new Path(resourceConfDirURI) + Path tempConfPath = new Path(confDir) + SliderUtils.copyDirectory(conf, resConfPath, tempConfPath, null) + + + ServiceLauncher<SliderClient> launcher = createStandaloneAM( + clustername, + true, + true) + SliderClient sliderClient = launcher.service + addToTeardown(sliderClient); + + assert 0 == clusterActionFreeze(sliderClient, clustername) + + + // here we do something devious: delete our copy of the configuration + // this makes sure the remote config gets picked up + HadoopFS localFS = HadoopFS.get(tempConfPath.toUri(), conf) + localFS.delete(tempConfPath,true) + + //now start the cluster + ServiceLauncher launcher2 = thawCluster(clustername, [], true); + SliderClient newCluster = launcher2.service + addToTeardown(newCluster); + + newCluster.getClusterDescription(clustername); + + // while running, flex it with no changes + newCluster.flex(clustername, [:]); + + //stop + assert 0 == clusterActionFreeze(sliderClient, clustername) + + //stop again + assert 0 == clusterActionFreeze(sliderClient, clustername) + + } + +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/cbf3a634/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawMasterlessAM.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawMasterlessAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawMasterlessAM.groovy deleted file mode 100644 index 5a1acef..0000000 --- a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawMasterlessAM.groovy +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.agent.freezethaw - -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import org.apache.hadoop.fs.FileSystem as HadoopFS -import org.apache.hadoop.fs.Path -import org.apache.hadoop.yarn.api.records.ApplicationReport -import org.apache.hadoop.yarn.conf.YarnConfiguration -import org.apache.slider.agent.AgentMiniClusterTestBase -import org.apache.slider.client.SliderClient -import org.apache.slider.common.tools.SliderUtils -import org.apache.slider.core.main.ServiceLauncher -import org.junit.Test - -/** - * stop and start an AM - */ -@CompileStatic -@Slf4j - -class TestFreezeThawMasterlessAM extends AgentMiniClusterTestBase { - - File getConfDirFile() { - return new File("target/TestFreezeThawMasterlessAM/conf") - } - - @Override - String getConfDir() { - return confDirFile.toURI().toString() - } - - @Test - public void testFreezeThawMasterlessAM() throws Throwable { - YarnConfiguration conf = configuration - String clustername = createMiniCluster("", conf, 1, 1, 1, true, false) - - describe "create a masterless AM, stop it, start it" - //copy the confdir somewhere - Path resConfPath = new Path(resourceConfDirURI) - Path tempConfPath = new Path(confDir) - SliderUtils.copyDirectory(conf, resConfPath, tempConfPath, null) - - - ServiceLauncher<SliderClient> launcher = createStandaloneAM( - clustername, - true, - true) - SliderClient sliderClient = launcher.service - addToTeardown(sliderClient); - - assert 0 == clusterActionFreeze(sliderClient, clustername) - - - // here we do something devious: delete our copy of the configuration - // this makes sure the remote config gets picked up - HadoopFS localFS = HadoopFS.get(tempConfPath.toUri(), conf) - localFS.delete(tempConfPath,true) - - //now start the cluster - ServiceLauncher launcher2 = thawCluster(clustername, [], true); - SliderClient newCluster = launcher2.service - addToTeardown(newCluster); - -// ApplicationReport report = waitForClusterLive(newCluster) - newCluster.getClusterDescription(clustername); - //stop - assert 0 == clusterActionFreeze(sliderClient, clustername) - - //stop again - assert 0 == clusterActionFreeze(sliderClient, clustername) - - } - -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/cbf3a634/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy index 4b66e1b..5e8ac9e 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy @@ -44,7 +44,7 @@ class TestStandaloneAgentAM extends AgentMiniClusterTestBase { @Test public void testStandaloneAgentAM() throws Throwable { - describe "create a masterless AM then get the service and look it up via the AM" + describe "create a masterless AM then perform actions on it" //launch fake master String clustername = createMiniCluster("", configuration, 1, true) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/cbf3a634/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneClusterNegativeComponentCounts.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneClusterNegativeComponentCounts.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneClusterNegativeComponentCounts.groovy deleted file mode 100644 index 8a23dc5..0000000 --- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneClusterNegativeComponentCounts.groovy +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.agent.standalone - -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import org.apache.hadoop.yarn.api.records.ApplicationReport -import org.apache.hadoop.yarn.api.records.FinalApplicationStatus -import org.apache.hadoop.yarn.api.records.YarnApplicationState -import org.apache.hadoop.yarn.conf.YarnConfiguration -import org.apache.slider.agent.AgentMiniClusterTestBase -import org.apache.slider.api.InternalKeys -import org.apache.slider.client.SliderClient -import org.apache.slider.common.SliderXmlConfKeys -import org.apache.slider.common.params.Arguments -import org.apache.slider.core.main.ServiceLauncher -import org.junit.Test - -@CompileStatic -@Slf4j -class TestStandaloneClusterNegativeComponentCounts - extends AgentMiniClusterTestBase { - - @Test - public void testStandaloneClusterNegativeComponentCounts() throws Throwable { - describe "Run a Standalone AM with cluster counts set to 0 ... expect failures" - // patch the configuration for AM restart - int threshold = 2; - YarnConfiguration conf = getRestartableConfiguration(threshold) - - String clustername = createMiniCluster("", conf, 1, true) - if (hdfsCluster) { - fail("Agent tests do not (currently) work with mini HDFS cluster") - } - ServiceLauncher<SliderClient> launcher = - createCluster(clustername, - ["negative":-4], - [ - ], - true, - false, - agentDefOptions) - SliderClient sliderClient = launcher.service - addToTeardown(sliderClient); - - ApplicationReport report - report = waitForClusterLive(sliderClient, 30000) - } - - /** - * Get a restartable configuration - * @param restarts - * @return - */ - public YarnConfiguration getRestartableConfiguration(int restarts) { - def conf = new YarnConfiguration(configuration) - conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, restarts) - conf.setInt(SliderXmlConfKeys.KEY_AM_RESTART_LIMIT, restarts) - conf - } - - -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/cbf3a634/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy index a7b5fe3..f67ee92 100644 --- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy @@ -23,6 +23,8 @@ import groovy.util.logging.Slf4j import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.slider.api.ResourceKeys import org.apache.slider.client.SliderClient +import org.apache.slider.common.SliderExitCodes +import org.apache.slider.core.exceptions.BadClusterStateException import org.apache.slider.core.main.ServiceLauncher import org.junit.Test @@ -85,9 +87,23 @@ class TestAgentEcho extends AgentTestBase { waitForRoleCount(sliderClient, roles, AGENT_CLUSTER_STARTUP_TIME) //sleep a bit - sleep(20000) + sleep(5000) //expect the role count to be the same waitForRoleCount(sliderClient, roles, 1000) + // flex size + // while running, flex it with no changes + sliderClient.flex(clustername, [(role): 2]); + sleep(1000) + waitForRoleCount(sliderClient, roles, 1000) + + // flex to an illegal value + try { + sliderClient.flex(clustername, [(role): -1]); + fail("expected an exception") + } catch (BadClusterStateException e) { + assertExceptionDetails(e, SliderExitCodes.EXIT_BAD_STATE, "") + } + } }
