SLIDER-469 Slider tool should have a option for flexing up or down by relative numbers (Sherry Guo via gourksaha)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/070a0af3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/070a0af3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/070a0af3 Branch: refs/heads/feature/SLIDER-906_docker_support Commit: 070a0af381499898f1f5e897c00c45162be39502 Parents: 72baea6 Author: Gour Saha <gourks...@apache.org> Authored: Fri Feb 19 17:32:37 2016 -0800 Committer: Gour Saha <gourks...@apache.org> Committed: Fri Feb 19 17:32:56 2016 -0800 ---------------------------------------------------------------------- .../org/apache/slider/client/SliderClient.java | 64 +++++++++++++++----- .../common/params/ComponentArgsDelegate.java | 2 +- .../providers/agent/TestAddonPackage.groovy | 11 ++-- .../providers/agent/TestAgentAAEcho.groovy | 6 +- .../slider/providers/agent/TestAgentEcho.groovy | 39 ++++++++++-- 5 files changed, 96 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/070a0af3/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 113b00e..d2c1cd3 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -2766,17 +2766,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } verifyBindingsDefined(); log.debug("actionFlex({})", name); - Map<String, Integer> roleInstances = new HashMap<>(); + Map<String, String> roleInstances = new HashMap<>(); for (Map.Entry<String, String> roleEntry : roleMap.entrySet()) { String key = roleEntry.getKey(); String val = roleEntry.getValue(); - try { - roleInstances.put(key, Integer.valueOf(val)); - } catch (NumberFormatException e) { - throw new BadCommandArgumentsException("Requested count of role %s" + - " is not a number: \"%s\"", - key, val); - } + roleInstances.put(key, val); } return flex(name, roleInstances); } @@ -3076,7 +3070,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe * @throws YarnException * @throws IOException */ - public int flex(String clustername, Map<String, Integer> roleInstances) + public int flex(String clustername, Map<String, String> roleInstances) throws YarnException, IOException { verifyBindingsDefined(); validateClusterName(clustername); @@ -3087,15 +3081,57 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe ConfTreeOperations resources = instanceDefinition.getResourceOperations(); - for (Map.Entry<String, Integer> entry : roleInstances.entrySet()) { + for (Map.Entry<String, String> entry : roleInstances.entrySet()) { String role = entry.getKey(); - int count = entry.getValue(); - resources.getOrAddComponent(role).put(COMPONENT_INSTANCES, - Integer.toString(count)); + String updateCountStr = entry.getValue(); + int currentCount = 0; + MapOperations component = resources.getOrAddComponent(role); + try { + // check if a relative count is specified + if (updateCountStr.startsWith("+") || updateCountStr.startsWith("-")) { + int updateCount = Integer.parseInt(updateCountStr); + // if component was specified before, get the current count + if (component.get(COMPONENT_INSTANCES) != null) { + currentCount = Integer.valueOf(component.get(COMPONENT_INSTANCES)); + if (currentCount + updateCount < 0) { + throw new BadCommandArgumentsException("The requested count " + + "of \"%s\" for role %s makes the total number of " + + "instances negative: \"%s\"", updateCount, role, + currentCount+updateCount); + } + else { + component.put(COMPONENT_INSTANCES, + Integer.toString(currentCount+updateCount)); + } + } + else { + if (updateCount < 0) { + throw new BadCommandArgumentsException("Invalid to request " + + "negative count of \"%s\" for role %s", updateCount, role); + } + else { + Map<String, String> map = new HashMap<>(); + resources.confTree.components.put(role, map); + component = new MapOperations(role, map); + component.put(COMPONENT_INSTANCES, Integer.toString(updateCount)); + } + } + } + else { + int count = Integer.parseInt(updateCountStr); + resources.getOrAddComponent(role).put(COMPONENT_INSTANCES, + Integer.toString(count)); + } + } + catch (NumberFormatException e) { + throw new BadCommandArgumentsException("Requested count of role %s" + + " is not a number: \"%s\"", + role, updateCountStr); + } log.debug("Flexed cluster specification ( {} -> {}) : \n{}", role, - count, + updateCountStr, resources); } SliderAMClientProvider sliderAM = new SliderAMClientProvider(getConfig()); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/070a0af3/slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java b/slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java index abda53f..5140059 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java @@ -32,7 +32,7 @@ public class ComponentArgsDelegate extends AbstractArgsDelegate { */ @Parameter(names = {ARG_COMPONENT, ARG_COMPONENT_SHORT, ARG_ROLE}, arity = 2, - description = "--component <name> <count>", + description = "--component <name> <count> e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3", splitter = DontSplitArguments.class) public List<String> componentTuples = new ArrayList<>(0); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/070a0af3/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAddonPackage.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAddonPackage.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAddonPackage.groovy index 1167b95..acfc355 100644 --- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAddonPackage.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAddonPackage.groovy @@ -36,6 +36,7 @@ import org.apache.slider.common.params.Arguments import org.apache.slider.common.params.ClientArgs import org.apache.slider.common.tools.SliderFileSystem import org.apache.slider.common.tools.SliderUtils +import org.apache.slider.core.exceptions.BadCommandArgumentsException import org.apache.slider.core.exceptions.SliderException import org.apache.slider.core.main.ServiceLauncher import org.apache.slider.providers.agent.AgentKeys @@ -137,16 +138,18 @@ class TestAddonPackage extends AgentTestBase { // flex size // while running, flex it with no changes - sliderClient.flex(clustername, [(role): 2]); + sliderClient.flex(clustername, [(role): "2"]); sleep(5000) waitForRoleCount(sliderClient, roles, 1000) // flex to an illegal value try { - sliderClient.flex(clustername, [(role): -1]); + sliderClient.flex(clustername, [(role): "-3"]); fail("expected an exception") - } catch (BadClusterStateException e) { - assertExceptionDetails(e, SliderExitCodes.EXIT_BAD_STATE, "negative") + } catch (BadCommandArgumentsException e) { + assertExceptionDetails(e, + BadCommandArgumentsException.class, + "total number of instances negative") } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/070a0af3/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy index 72cb550..c82f458 100644 --- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy @@ -131,6 +131,7 @@ class TestAgentAAEcho extends TestAgentEcho { Map<String, Integer> roles, String proxyAM) { def onlyOneEcho = [(rolename): 1] + def onlyOneEchoForFlex = [(rolename): "1"] def requested = roles[rolename] waitForRoleCount(sliderClient, onlyOneEcho, AGENT_CLUSTER_STARTUP_TIME) @@ -151,9 +152,10 @@ class TestAgentAAEcho extends TestAgentEcho { sleep(5000) requested = 50 + def requestedForFlex = "50" def expectedPending = requested - 1 - sliderClient.flex(clustername, [(rolename): requested]); + sliderClient.flex(clustername, [(rolename): requestedForFlex]); waitForRoleCount(sliderClient, onlyOneEcho, 1000) sleep(4000) def now = System.currentTimeMillis(); @@ -172,7 +174,7 @@ class TestAgentAAEcho extends TestAgentEcho { // while running, flex it to size = 1 sleep(1000) - sliderClient.flex(clustername, onlyOneEcho); + sliderClient.flex(clustername, onlyOneEchoForFlex); waitForRoleCount(sliderClient, onlyOneEcho, 1000) def echoInstances2 = sliderClient.listNodeUUIDsByRole(rolename) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/070a0af3/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy index 23a7bbb..e395140 100644 --- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy @@ -28,6 +28,7 @@ import org.apache.slider.common.SliderExitCodes import org.apache.slider.common.SliderXmlConfKeys import org.apache.slider.common.params.ActionNodesArgs import org.apache.slider.core.exceptions.BadClusterStateException +import org.apache.slider.core.exceptions.BadCommandArgumentsException import org.apache.slider.core.main.ServiceLauncher import org.apache.slider.core.persist.JsonSerDeser import org.junit.Before @@ -85,8 +86,9 @@ class TestAgentEcho extends AgentTestBase { validatePaths() def role = ECHO + int numInstances = 2 Map<String, Integer> roles = [ - (role): 2, + (role): numInstances, ]; ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername, roles, @@ -115,18 +117,45 @@ class TestAgentEcho extends AgentTestBase { // flex size // while running, flex it with no changes - sliderClient.flex(clustername, [(role): 2]); + sliderClient.flex(clustername, [(role): "2"]); sleep(1000) waitForRoleCount(sliderClient, roles, 1000) // flex to an illegal value try { - sliderClient.flex(clustername, [(role): -1]); + sliderClient.flex(clustername, [(role): "-o"]); fail("expected an exception") - } catch (BadClusterStateException e) { - assertExceptionDetails(e, SliderExitCodes.EXIT_BAD_STATE, "negative") + } catch (BadCommandArgumentsException e) { + assertExceptionDetails(e, + BadCommandArgumentsException.class, + "not a number") } + // flex up with a relative number + // -- add more instances + sliderClient.flex(clustername, [(role): "+1"]); + sleep(1000) + numInstances += 1 + roles = [ (role): numInstances ] + waitForRoleCount(sliderClient, roles, 1000) + + // flex down with relative number + // -- decrease number of instances + sliderClient.flex(clustername, [(role): "-2"]); + sleep(1000) + numInstances -= 2 + roles = [ (role): numInstances ] + waitForRoleCount(sliderClient, roles, 1000) + + // flex down again so the total number becomes negative + try { + sliderClient.flex(clustername, [(role): "-5"]); + fail("expected an exception") + } catch (BadCommandArgumentsException e) { + assertExceptionDetails(e, + BadCommandArgumentsException.class, + "total number of instances negative") + } runNodemapTests(sliderClient)