Repository: incubator-slider Updated Branches: refs/heads/develop f14d32c6f -> 486ef84f4
SLIDER-181. Storm package updates (changes to accomodate storm script changing to be a shell script from python) SLIDER-402. Add support for general purpose cross referencing of named properties (handle multiple replacements for single property) Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/486ef84f Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/486ef84f Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/486ef84f Branch: refs/heads/develop Commit: 486ef84f4cb7e6e716e70a82ac08e1c43b3c497d Parents: f14d32c Author: Sumit Mohanty <[email protected]> Authored: Fri Sep 5 20:59:04 2014 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Fri Sep 5 20:59:24 2014 -0700 ---------------------------------------------------------------------- app-packages/storm/appConfig.json | 6 +-- app-packages/storm/package/scripts/params.py | 2 +- app-packages/storm/package/scripts/service.py | 43 ++++++++------------ .../providers/agent/AgentProviderService.java | 5 ++- .../agent/TestAgentProviderService.java | 10 ++--- 5 files changed, 28 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/486ef84f/app-packages/storm/appConfig.json ---------------------------------------------------------------------- diff --git a/app-packages/storm/appConfig.json b/app-packages/storm/appConfig.json index ac9bd49..1eae599 100644 --- a/app-packages/storm/appConfig.json +++ b/app-packages/storm/appConfig.json @@ -23,8 +23,8 @@ "site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm", "site.storm-site.transactional.zookeeper.root": "/transactional", "site.storm-site.storm.zookeeper.port": "2181", - "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${site.global.ganglia_server_host},port=${site.global.ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM", - "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${site.global.ganglia_server_host},port=${site.global.ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM", + "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM", + "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM", "site.storm-site.dev.zookeeper.path": "${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper", "site.storm-site.drpc.invocations.port": "0", "site.storm-site.storm.zookeeper.root": "${DEF_ZK_PATH}", @@ -32,7 +32,7 @@ "site.storm-site.nimbus.host": "${NIMBUS_HOST}", "site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}", "site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]", - "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${site.global.ganglia_server_host},port=${site.global.ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM", + "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM", "site.storm-site.drpc.port": "0", "site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}" }, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/486ef84f/app-packages/storm/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/app-packages/storm/package/scripts/params.py b/app-packages/storm/package/scripts/params.py index a70bcb9..38e25fb 100644 --- a/app-packages/storm/package/scripts/params.py +++ b/app-packages/storm/package/scripts/params.py @@ -34,7 +34,7 @@ user_group = config['configurations']['global']['user_group'] java64_home = config['hostLevelParams']['java_home'] nimbus_host = config['configurations']['storm-site']['nimbus.host'] nimbus_port = config['configurations']['storm-site']['nimbus.thrift.port'] -storm_bin = format("{app_root}/bin/storm") +storm_bin = format("{app_root}/bin/storm.py") storm_env_sh_template = config['configurations']['storm-env']['content'] ganglia_installed = config['configurations']['global']['ganglia_enabled'] http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/486ef84f/app-packages/storm/package/scripts/service.py ---------------------------------------------------------------------- diff --git a/app-packages/storm/package/scripts/service.py b/app-packages/storm/package/scripts/service.py index dc3c733..3a5e795 100644 --- a/app-packages/storm/package/scripts/service.py +++ b/app-packages/storm/package/scripts/service.py @@ -21,6 +21,8 @@ limitations under the License. from resource_management import * import time +import os +import sys """ Slider package uses jps as pgrep does not list the whole process start command @@ -31,6 +33,7 @@ def service( import params import status_params + python_binary = os.environ['PYTHON_EXE'] if 'PYTHON_EXE' in os.environ else sys.executable pid_file = status_params.pid_files[name] container_id = status_params.container_id no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1") @@ -40,17 +43,11 @@ def service( if name == 'ui': crt_pid_cmd = format("{jps_path} -vl | grep \"^[0-9 ]*backtype.storm.ui.core\" {grep_and_awk} > {pid_file}") - elif name == "rest_api": - rest_process_cmd = format("{java64_home}/bin/java -jar {rest_lib_dir}/`ls {rest_lib_dir} | grep -wE storm-rest-[0-9.-]+\.jar` server") - crt_pid_cmd = format("pgrep -f \"{rest_process_cmd}\" > {pid_file}") else: crt_pid_cmd = format("{jps_path} -vl | grep \"^[0-9 ]*backtype.storm.daemon.{name}\" {grep_and_awk} > {pid_file}") if action == "start": - if name == "rest_api": - cmd = format("{rest_process_cmd} {rest_api_conf_file} > {log_dir}/restapi.log") - else: - cmd = format("env JAVA_HOME={java64_home} PATH=$PATH:{java64_home}/bin STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {storm_bin} {name} > {log_dir}/{name}.out 2>&1") + cmd = format("env JAVA_HOME={java64_home} PATH=$PATH:{java64_home}/bin STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {python_binary} {storm_bin} {name} > {log_dir}/{name}.out 2>&1") if params.security_enabled: if name == "nimbus": @@ -67,30 +64,22 @@ def service( wait_for_finish=False ) - if name == "rest_api": + content = None + for i in xrange(12): Execute(crt_pid_cmd, user=params.storm_user, - logoutput=True, - tries=6, - try_sleep=10 + logoutput=True ) - else: - content = None - for i in xrange(12): - Execute(crt_pid_cmd, - user=params.storm_user, - logoutput=True - ) - with open(pid_file) as f: - content = f.readline().strip() - if content.isdigit(): - break; - File(pid_file, action = "delete") - time.sleep(10) - pass + with open(pid_file) as f: + content = f.readline().strip() + if content.isdigit(): + break; + File(pid_file, action = "delete") + time.sleep(10) + pass - if not content.isdigit(): - raise Fail(format("Unable to start {name}")) + if not content.isdigit(): + raise Fail(format("Unable to start {name}")) elif action == "stop": process_dont_exist = format("! ({no_op_test})") http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/486ef84f/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java index 02ef625..063e61c 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java @@ -1382,7 +1382,7 @@ public class AgentProviderService extends AbstractProviderService implements protected void dereferenceAllConfigs(Map<String, Map<String, String>> configurations) { Map<String, String> allConfigs = new HashMap<String, String>(); - String lookupFormat = "${site.%s.%s}"; + String lookupFormat = "${@//site/%s/%s}"; for (String configType : configurations.keySet()) { Map<String, String> configBucket = configurations.get(configType); for (String configName : configBucket.keySet()) { @@ -1396,9 +1396,10 @@ public class AgentProviderService extends AbstractProviderService implements String configValue = configBucket.get(configName); for (String lookUpKey : allConfigs.keySet()) { if (configValue != null && configValue.contains(lookUpKey)) { - configBucket.put(configName, configValue.replace(lookUpKey, allConfigs.get(lookUpKey))); + configValue = configValue.replace(lookUpKey, allConfigs.get(lookUpKey)); } } + configBucket.put(configName, configValue); } } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/486ef84f/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java index 0f0e744..3711e12 100644 --- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java +++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java @@ -1133,14 +1133,14 @@ public class TestAgentProviderService { AgentProviderService aps = new AgentProviderService(); Map<String, Map<String, String>> allConfigs = new HashMap<String, Map<String, String>>(); Map<String, String> cfg1 = new HashMap<String, String>(); - cfg1.put("a1", "${site.cfg-2.A1}"); + cfg1.put("a1", "${@//site/cfg-2/A1}"); cfg1.put("b1", "22"); cfg1.put("c1", "33"); - cfg1.put("d1", "${site.cfg1.c1}AA"); + cfg1.put("d1", "${@//site/cfg1/c1}AA"); Map<String, String> cfg2 = new HashMap<String, String>(); cfg2.put("A1", "11"); - cfg2.put("B1", "${site.cfg-2.A1},${site.cfg-2.A1}"); - cfg2.put("C1", "DD${site.cfg1.c1}"); + cfg2.put("B1", "${@//site/cfg-2/A1},${@//site/cfg-2/A1},AA,${@//site/cfg1/c1}"); + cfg2.put("C1", "DD${@//site/cfg1/c1}"); cfg2.put("D1", "${14}"); allConfigs.put("cfg1", cfg1); @@ -1152,7 +1152,7 @@ public class TestAgentProviderService { Assert.assertEquals("33AA", cfg1.get("d1")); Assert.assertEquals("11", cfg2.get("A1")); - Assert.assertEquals("11,11", cfg2.get("B1")); + Assert.assertEquals("11,11,AA,33", cfg2.get("B1")); Assert.assertEquals("DD33", cfg2.get("C1")); Assert.assertEquals("${14}", cfg2.get("D1")); }
