Repository: incubator-slider Updated Branches: refs/heads/develop 329cab03e -> ffbc3a3c3
SLIDER-380. Add ability for a application to specify what system configs they need SLIDER-379. HBase package should by default set hbase.master.port to 0 Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ae4c076c Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ae4c076c Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ae4c076c Branch: refs/heads/develop Commit: ae4c076cd24f86f6ad1ef665e1375fa42d3b12f0 Parents: 329cab0 Author: Sumit Mohanty <[email protected]> Authored: Fri Aug 29 20:54:56 2014 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Fri Aug 29 20:55:05 2014 -0700 ---------------------------------------------------------------------- app-packages/hbase/appConfig.json | 4 +++- .../slider/providers/agent/AgentKeys.java | 1 + .../providers/agent/AgentProviderService.java | 20 ++++++++++++++++++-- .../agent/TestAgentProviderService.java | 17 +++++++++++++++++ 4 files changed, 39 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ae4c076c/app-packages/hbase/appConfig.json ---------------------------------------------------------------------- diff --git a/app-packages/hbase/appConfig.json b/app-packages/hbase/appConfig.json index 9c37ed9..d324e6d 100644 --- a/app-packages/hbase/appConfig.json +++ b/app-packages/hbase/appConfig.json @@ -6,6 +6,7 @@ "application.def": "${app.package.name}.zip", "create.default.zookeeper.node": "true", "java_home": "/usr/jdk64/jdk1.7.0_45", + "system_configs": "core-site", "site.global.app_user": "yarn", "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${hbase.version}", @@ -33,7 +34,8 @@ "site.hbase-site.zookeeper.znode.parent": "${DEF_ZK_PATH}", "site.hbase-site.hbase.regionserver.info.port": "0", "site.hbase-site.hbase.master.info.port": "${HBASE_MASTER.ALLOCATED_PORT}", - "site.hbase-site.hbase.regionserver.port": "0" + "site.hbase-site.hbase.regionserver.port": "0", + "site.hbase-site.hbase.master.port": "0" }, "components": { "slider-appmaster": { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ae4c076c/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java index 419fa1a..1622367 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java @@ -85,6 +85,7 @@ public interface AgentKeys { String JAVA_HOME = "java_home"; String PACKAGE_LIST = "package_list"; + String SYSTEM_CONFIGS = "system_configs"; String WAIT_HEARTBEAT = "wait.heartbeat"; String PYTHON_EXE = "python"; String CREATE_DEF_ZK_NODE = "create.default.zookeeper.node"; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ae4c076c/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java index 669a19a..be5b1ac 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java @@ -1348,9 +1348,10 @@ public class AgentProviderService extends AbstractProviderService implements new TreeMap<String, Map<String, String>>(); Map<String, String> tokens = getStandardTokenMap(appConf); - List<String> configs = getApplicationConfigurationTypes(); + Set<String> configs = new HashSet<String>(); + configs.addAll(getApplicationConfigurationTypes()); + configs.addAll(getSystemConfigurationsRequested(appConf)); - //Add global for (String configType : configs) { addNamedConfiguration(configType, appConf.getGlobalOptions().options, configurations, tokens, containerId); @@ -1374,6 +1375,21 @@ public class AgentProviderService extends AbstractProviderService implements } @VisibleForTesting + protected List<String> getSystemConfigurationsRequested(ConfTreeOperations appConf) { + List<String> configList = new ArrayList<String>(); + + String configTypes = appConf.get(AgentKeys.SYSTEM_CONFIGS); + if (configTypes != null && configTypes.length() > 0) { + String[] configs = configTypes.split(","); + for(String config :configs) + configList.add(config.trim()); + } + + return new ArrayList<String>(new HashSet<String>(configList)); + } + + + @VisibleForTesting protected List<String> getApplicationConfigurationTypes() { List<String> configList = new ArrayList<String>(); configList.add(GLOBAL_CONFIG_TAG); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ae4c076c/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java index b0b059d..428d767 100644 --- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java +++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java @@ -18,6 +18,7 @@ package org.apache.slider.providers.agent; +import com.sun.jersey.spi.container.servlet.WebConfig; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FilterFileSystem; @@ -1057,7 +1058,10 @@ public class TestAgentProviderService { List<String> configurations = new ArrayList<String>(); configurations.add("hbase-site"); configurations.add("global"); + List<String> sysConfigurations = new ArrayList<String>(); + configurations.add("core-site"); doReturn(configurations).when(mockAps).getApplicationConfigurationTypes(); + doReturn(sysConfigurations).when(mockAps).getSystemConfigurationsRequested(any(ConfTreeOperations.class)); Map<String, Map<String, ClusterNode>> roleClusterNodeMap = new HashMap<String, Map<String, ClusterNode>>(); Map<String, ClusterNode> container = new HashMap<String, ClusterNode>(); @@ -1078,6 +1082,7 @@ public class TestAgentProviderService { mockAps.addStartCommand("HBASE_MASTER", "cid1", hbr, "", 0, Boolean.FALSE); Assert.assertTrue(hbr.getExecutionCommands().get(0).getConfigurations().containsKey("hbase-site")); + Assert.assertTrue(hbr.getExecutionCommands().get(0).getConfigurations().containsKey("core-site")); Map<String, String> hbaseSiteConf = hbr.getExecutionCommands().get(0).getConfigurations().get("hbase-site"); Assert.assertTrue(hbaseSiteConf.containsKey("a.port")); Assert.assertEquals("10023", hbaseSiteConf.get("a.port")); @@ -1095,4 +1100,16 @@ public class TestAgentProviderService { Assert.assertFalse(cmd.getConfigurations().get("hbase-site").containsKey("defaultB")); } + @Test + public void testParameterParsing() { + AgentProviderService aps = new AgentProviderService(); + AggregateConf aggConf = new AggregateConf(); + ConfTreeOperations treeOps = aggConf.getAppConfOperations(); + treeOps.getGlobalOptions().put(AgentKeys.SYSTEM_CONFIGS, "core-site,yarn-site, core-site "); + List<String> configs = aps.getSystemConfigurationsRequested(treeOps); + Assert.assertEquals(2, configs.size()); + Assert.assertTrue(configs.contains("core-site")); + Assert.assertFalse(configs.contains("bore-site")); + } + }
