Repository: incubator-slider Updated Branches: refs/heads/feature/auto_generate_storm_pkg 26b4153eb -> 02faa944d
SLIDER-391. appConfig.json should allow reference to other config properties and resolve them automatically Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/02faa944 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/02faa944 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/02faa944 Branch: refs/heads/feature/auto_generate_storm_pkg Commit: 02faa944da56cd6252b48afbdb6ee427291f1fe0 Parents: 26b4153 Author: Sumit Mohanty <[email protected]> Authored: Wed Sep 3 18:17:13 2014 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Wed Sep 3 18:17:13 2014 -0700 ---------------------------------------------------------------------- app-packages/hbase/appConfig.json | 8 +++-- app-packages/storm/appConfig.json | 7 ++-- .../providers/agent/AgentProviderService.java | 35 ++++++++++++++++---- .../agent/TestAgentProviderService.java | 29 ++++++++++++++++ 4 files changed, 66 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/02faa944/app-packages/hbase/appConfig.json ---------------------------------------------------------------------- diff --git a/app-packages/hbase/appConfig.json b/app-packages/hbase/appConfig.json index 7df0dc3..07d1b4e 100644 --- a/app-packages/hbase/appConfig.json +++ b/app-packages/hbase/appConfig.json @@ -11,14 +11,16 @@ "site.global.app_user": "yarn", "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${hbase.version}", + "site.global.ganglia_server_host": "${NN_HOST}", + "site.global.ganglia_server_port": "8667", + "site.global.ganglia_server_id": "Application1", + "site.global.ganglia_enabled":"true", + "site.global.hbase_instance_name": "instancename", "site.global.hbase_root_password": "secret", "site.global.user_group": "hadoop", "site.global.security_enabled": "false", "site.global.monitor_protocol": "http", - "site.global.ganglia_server_host": "${NN_HOST}", - "site.global.ganglia_server_port": "8667", - "site.global.ganglia_server_id": "Application1", "site.global.hbase_thrift_port": "${HBASE_THRIFT.ALLOCATED_PORT}", "site.global.hbase_thrift2_port": "${HBASE_THRIFT2.ALLOCATED_PORT}", "site.global.hbase_rest_port": "${HBASE_REST.ALLOCATED_PORT}", http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/02faa944/app-packages/storm/appConfig.json ---------------------------------------------------------------------- diff --git a/app-packages/storm/appConfig.json b/app-packages/storm/appConfig.json index df28e8f..ac9bd49 100644 --- a/app-packages/storm/appConfig.json +++ b/app-packages/storm/appConfig.json @@ -11,6 +11,7 @@ "site.global.app_user": "yarn", "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}", "site.global.user_group": "hadoop", + "site.global.ganglia_server_host": "${NN_HOST}", "site.global.ganglia_server_id": "Application2", "site.global.ganglia_enabled":"true", @@ -22,8 +23,8 @@ "site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm", "site.storm-site.transactional.zookeeper.root": "/transactional", "site.storm-site.storm.zookeeper.port": "2181", - "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM", - "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM", + "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${site.global.ganglia_server_host},port=${site.global.ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM", + "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${site.global.ganglia_server_host},port=${site.global.ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM", "site.storm-site.dev.zookeeper.path": "${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper", "site.storm-site.drpc.invocations.port": "0", "site.storm-site.storm.zookeeper.root": "${DEF_ZK_PATH}", @@ -31,7 +32,7 @@ "site.storm-site.nimbus.host": "${NIMBUS_HOST}", "site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}", "site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]", - "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM", + "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${site.global.ganglia_server_host},port=${site.global.ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM", "site.storm-site.drpc.port": "0", "site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}" }, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/02faa944/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java index 61866fb..d188d49 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java @@ -1196,7 +1196,8 @@ public class AgentProviderService extends AbstractProviderService implements hostLevelParams.put(CONTAINER_ID, containerId); cmd.setHostLevelParams(hostLevelParams); - setInstallCommandConfigurations(cmd, containerId); + Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId); + cmd.setConfigurations(configurations); cmd.setCommandParams(setCommandParameters(scriptPath, timeout, false)); @@ -1246,12 +1247,6 @@ public class AgentProviderService extends AbstractProviderService implements return cmdParams; } - private void setInstallCommandConfigurations(ExecutionCommand cmd, String containerId) throws SliderException { - ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); - Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId); - cmd.setConfigurations(configurations); - } - @VisibleForTesting protected void addStatusCommand(String roleName, String containerId, @@ -1380,9 +1375,35 @@ public class AgentProviderService extends AbstractProviderService implements configurations, tokens, containerId); } + //do a final replacement of re-used configs + dereferenceAllConfigs(configurations); + return configurations; } + protected void dereferenceAllConfigs(Map<String, Map<String, String>> configurations) { + Map<String, String> allConfigs = new HashMap<String, String>(); + String lookupFormat = "${site.%s.%s}"; + for (String configType : configurations.keySet()) { + Map<String, String> configBucket = configurations.get(configType); + for (String configName : configBucket.keySet()) { + allConfigs.put(String.format(lookupFormat, configType, configName), configBucket.get(configName)); + } + } + + for (String configType : configurations.keySet()) { + Map<String, String> configBucket = configurations.get(configType); + for (String configName : configBucket.keySet()) { + String configValue = configBucket.get(configName); + for (String lookUpKey : allConfigs.keySet()) { + if (configValue != null && configValue.contains(lookUpKey)) { + configBucket.put(configName, configValue.replace(lookUpKey, allConfigs.get(lookUpKey))); + } + } + } + } + } + private Map<String, String> getStandardTokenMap(ConfTreeOperations appConf) throws SliderException { Map<String, String> tokens = new HashMap<String, String>(); String nnuri = appConf.get("site.fs.defaultFS"); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/02faa944/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java index 16f5347..0f0e744 100644 --- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java +++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java @@ -1128,4 +1128,33 @@ public class TestAgentProviderService { Assert.assertFalse(configs.contains("bore-site")); } + @Test + public void testDereferenceAllConfig() { + AgentProviderService aps = new AgentProviderService(); + Map<String, Map<String, String>> allConfigs = new HashMap<String, Map<String, String>>(); + Map<String, String> cfg1 = new HashMap<String, String>(); + cfg1.put("a1", "${site.cfg-2.A1}"); + cfg1.put("b1", "22"); + cfg1.put("c1", "33"); + cfg1.put("d1", "${site.cfg1.c1}AA"); + Map<String, String> cfg2 = new HashMap<String, String>(); + cfg2.put("A1", "11"); + cfg2.put("B1", "${site.cfg-2.A1},${site.cfg-2.A1}"); + cfg2.put("C1", "DD${site.cfg1.c1}"); + cfg2.put("D1", "${14}"); + + allConfigs.put("cfg1", cfg1); + allConfigs.put("cfg-2", cfg2); + aps.dereferenceAllConfigs(allConfigs); + Assert.assertEquals("11", cfg1.get("a1")); + Assert.assertEquals("22", cfg1.get("b1")); + Assert.assertEquals("33", cfg1.get("c1")); + Assert.assertEquals("33AA", cfg1.get("d1")); + + Assert.assertEquals("11", cfg2.get("A1")); + Assert.assertEquals("11,11", cfg2.get("B1")); + Assert.assertEquals("DD33", cfg2.get("C1")); + Assert.assertEquals("${14}", cfg2.get("D1")); + } + }
