SLIDER-985: regression, TestBuildBasicAgent failing. (cause: increasing the limit on the #of agents allowed)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c776b1ad Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c776b1ad Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c776b1ad Branch: refs/heads/feature/SLIDER-82-pass-3.1 Commit: c776b1ad45dbd5403e2f6b2824254ba0f0935c5d Parents: 7278c39 Author: Steve Loughran <[email protected]> Authored: Mon Nov 16 14:01:35 2015 +0000 Committer: Steve Loughran <[email protected]> Committed: Mon Nov 16 14:01:50 2015 +0000 ---------------------------------------------------------------------- .../slider/core/persist/ConfPersister.java | 6 ++- .../slider/providers/agent/AgentTestBase.groovy | 16 ++---- .../providers/agent/TestAgentAAEcho.groovy | 34 +++++++++--- .../providers/agent/TestBuildBasicAgent.groovy | 57 +++++++++++--------- slider-core/src/test/python/metainfo.xml | 2 +- 5 files changed, 68 insertions(+), 47 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c776b1ad/slider-core/src/main/java/org/apache/slider/core/persist/ConfPersister.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/ConfPersister.java b/slider-core/src/main/java/org/apache/slider/core/persist/ConfPersister.java index 60717f6..9759205 100644 --- a/slider-core/src/main/java/org/apache/slider/core/persist/ConfPersister.java +++ b/slider-core/src/main/java/org/apache/slider/core/persist/ConfPersister.java @@ -101,7 +101,7 @@ public class ConfPersister { * Make the persistent directory * @throws IOException IO failure */ - private void mkPersistDir() throws IOException { + public void mkPersistDir() throws IOException { coreFS.getFileSystem().mkdirs(persistDir); } @@ -165,13 +165,15 @@ public class ConfPersister { * Acquire the writelock * @throws IOException IO * @throws LockAcquireFailedException - * @throws FileNotFoundException if the target dir does not exist + * @throws FileNotFoundException if the target dir does not exist. */ @VisibleForTesting boolean acquireReadLock() throws FileNotFoundException, IOException, LockAcquireFailedException { if (!coreFS.getFileSystem().exists(persistDir)) { + // the dir is not there, so the data is not there, so there + // is nothing to read throw new FileNotFoundException(persistDir.toString()); } long now = System.currentTimeMillis(); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c776b1ad/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy index 5bf1c1f..0e5cd00 100644 --- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy @@ -32,6 +32,7 @@ import org.apache.slider.test.YarnZKMiniClusterTestBase import org.junit.Before import org.junit.Rule import org.junit.rules.TemporaryFolder +import org.slf4j.Logger import static org.apache.slider.common.SliderXMLConfKeysForTesting.* import static org.apache.slider.providers.agent.AgentKeys.CONF_RESOURCE @@ -53,7 +54,7 @@ public abstract class AgentTestBase extends YarnZKMiniClusterTestBase { */ public static void assumeValidServerEnv() { try { - SliderUtils.validateSliderServerEnvironment(log, true) + SliderUtils.validateSliderServerEnvironment(log as Logger, true) } catch (Exception e) { skip(e.toString()) } @@ -135,12 +136,7 @@ public abstract class AgentTestBase extends YarnZKMiniClusterTestBase { boolean create, boolean blockUntilRunning) { - - YarnConfiguration conf = testConfiguration - - def clusterOps = [ - : - ] + def clusterOps = [:] return createOrBuildCluster( create ? SliderActions.ACTION_CREATE : SliderActions.ACTION_BUILD, @@ -166,11 +162,7 @@ public abstract class AgentTestBase extends YarnZKMiniClusterTestBase { List<String> extraArgs, boolean deleteExistingData) { - YarnConfiguration conf = testConfiguration - - def clusterOps = [ - : - ] + def clusterOps = [:] return createOrBuildCluster( SliderActions.ACTION_UPDATE, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c776b1ad/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy index 0b89f47..80ff5a8 100644 --- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy @@ -52,9 +52,7 @@ class TestAgentAAEcho extends TestAgentEcho { validatePaths() def echo = "echo" - Map<String, Integer> roles = [ - (echo): 2, - ]; + Map<String, Integer> roles = buildRoleMap(echo) ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername, roles, [ @@ -74,10 +72,33 @@ class TestAgentAAEcho extends TestAgentEcho { ], true, true, true) - SliderClient sliderClient = launcher.service + postLaunchActions(launcher.service, clustername, echo, roles) + + } + /** + * Build the role map to use when creating teh cluster + * @param roleName the name used for the echo role + * @return the map + */ + protected Map<String, Integer> buildRoleMap(String roleName) { + [ + (roleName): 2, + ]; + } - def onlyOneEcho = [(echo): 1] + /** + * Any actions to perform after starting the agent cluster + * @param sliderClient client for the cluster + * @param clustername cluster name + * @param roleName name of the echo role + * @parm original set of roles + */ + protected void postLaunchActions(SliderClient sliderClient, + String clustername, + String roleName, + Map<String, Integer> roles) { + def onlyOneEcho = [(roleName): 1] waitForRoleCount(sliderClient, onlyOneEcho, AGENT_CLUSTER_STARTUP_TIME) //sleep a bit sleep(5000) @@ -91,9 +112,8 @@ class TestAgentAAEcho extends TestAgentEcho { sliderClient.flex(clustername, onlyOneEcho); // while running, flex it with no changes - sliderClient.flex(clustername, [(echo): 3]); + sliderClient.flex(clustername, [(roleName): 3]); sleep(1000) waitForRoleCount(sliderClient, onlyOneEcho, 1000) - } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c776b1ad/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy index 264d260..60e9035 100644 --- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy @@ -94,7 +94,9 @@ class TestBuildBasicAgent extends AgentTestBase { 1, true, false) - buildAgentCluster("test_build_basic_agent_node_only", + + def cluster01 = clustername + "_01" + buildAgentCluster(cluster01, [(ROLE_NODE): 1], [ ARG_OPTION, CONTROLLER_URL, "http://localhost";, @@ -110,7 +112,9 @@ class TestBuildBasicAgent extends AgentTestBase { def master = "hbase-master" def rs = "hbase-rs" - ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername, + + def cluster02 = clustername + "_02" + ServiceLauncher<SliderClient> launcher = buildAgentCluster(cluster02, [ (ROLE_NODE): 1, (master): 1, @@ -149,10 +153,11 @@ class TestBuildBasicAgent extends AgentTestBase { def rscomponent = resource.getMandatoryComponent(rs) assert "5" == rscomponent.getMandatoryOption(ResourceKeys.COMPONENT_INSTANCES) - // now create an instance with no role priority for the newnode role + describe "build a cluster with no role priority for the newnode role" + try { - def name2 = clustername + "-2" - buildAgentCluster(name2, + def cluster03 = clustername + "_03" + buildAgentCluster(cluster03, [ (ROLE_NODE): 2, "role3": 1, @@ -166,14 +171,16 @@ class TestBuildBasicAgent extends AgentTestBase { ], true, false, false) - failWithBuildSucceeding(name2, "no priority for one role") + failWithBuildSucceeding(cluster03, "no priority for one role") } catch (BadConfigException expected) { } + describe "build a cluster with the number of agents out of range" try { - launcher = buildAgentCluster(clustername + "-10", + def cluster04 = clustername + "_04" + launcher = buildAgentCluster(cluster04, [ - (ROLE_NODE): 4, + (ROLE_NODE): 2000, ], [ ARG_OPTION, CONTROLLER_URL, "http://localhost";, @@ -185,15 +192,18 @@ class TestBuildBasicAgent extends AgentTestBase { ], true, false, false) - failWithBuildSucceeding(ROLE_NODE, "too many instances") + failWithBuildSucceeding(cluster04, "too many instances") } catch (BadConfigException expected) { - assert expected.message.contains("Expected minimum is 1 and maximum is 2") - assert expected.message.contains("Component echo, yarn.component.instances value 4 out of range.") + assertExceptionDetails(expected, SliderExitCodes.EXIT_BAD_CONFIGURATION, + "Expected minimum is 1 and maximum is") + assertExceptionDetails(expected, SliderExitCodes.EXIT_BAD_CONFIGURATION, + "out of range.") } - //duplicate priorities + + describe "build a cluster with duplicate priorities for roles" try { - def name3 = clustername + "-3" - buildAgentCluster(name3, + def cluster05 = clustername + "_05" + buildAgentCluster(cluster05, [ (ROLE_NODE): 5, (master): 1, @@ -206,16 +216,16 @@ class TestBuildBasicAgent extends AgentTestBase { true, false, false) - failWithBuildSucceeding(name3, "duplicate priorities") + failWithBuildSucceeding(cluster05, "duplicate priorities") } catch (BadConfigException expected) { } - def cluster4 = clustername + "-4" + def cluster06 = clustername + "_06" def jvmopts = "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005" - buildAgentCluster(cluster4, + buildAgentCluster(cluster06, [ (master): 1, (rs): 5 @@ -235,7 +245,7 @@ class TestBuildBasicAgent extends AgentTestBase { false) //now we want to look at the value - AggregateConf instanceDefinition = loadInstanceDefinition(cluster4) + AggregateConf instanceDefinition = loadInstanceDefinition(cluster06) def opt = instanceDefinition.getAppConfOperations().getComponentOpt( SliderKeys.COMPONENT_AM, RoleKeys.JVM_OPTS, @@ -245,8 +255,8 @@ class TestBuildBasicAgent extends AgentTestBase { // now create an instance with no component options, hence no // entry in the app config - def name5 = clustername + "-5" - buildAgentCluster(name5, + def name07 = clustername + "_07" + buildAgentCluster(name07, [ "hbase-rs": 1, ], @@ -403,9 +413,8 @@ class TestBuildBasicAgent extends AgentTestBase { def rscomponent2 = resource2.getMandatoryComponent(rs) assert "6" == rscomponent2.getMandatoryOption(ResourceKeys.COMPONENT_INSTANCES) } - + public AggregateConf loadInstanceDefinition(String name) { - def cluster4 def sliderFS = createSliderFileSystem() def dirPath = sliderFS.buildClusterDirPath(name) ConfPersister persister = new ConfPersister(sliderFS, dirPath) @@ -697,10 +706,8 @@ class TestBuildBasicAgent extends AgentTestBase { } public void failWithBuildSucceeding(String name, String reason) { - def badArgs1 AggregateConf instanceDefinition = loadInstanceDefinition(name) - log.error( - "Build operation should have failed from $reason : \n$instanceDefinition") + log.error("Build operation should have failed from $reason : \n$instanceDefinition") fail("Build operation should have failed from $reason") } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c776b1ad/slider-core/src/test/python/metainfo.xml ---------------------------------------------------------------------- diff --git a/slider-core/src/test/python/metainfo.xml b/slider-core/src/test/python/metainfo.xml index 2a8c9e0..7f9cd23 100644 --- a/slider-core/src/test/python/metainfo.xml +++ b/slider-core/src/test/python/metainfo.xml @@ -51,7 +51,7 @@ <name>echo</name> <category>MASTER</category> <minInstanceCount>1</minInstanceCount> - <maxInstanceCount>200</maxInstanceCount> + <maxInstanceCount>100</maxInstanceCount> <commandScript> <script>echo.py</script> <scriptType>PYTHON</scriptType>
