Repository: incubator-slider Updated Branches: refs/heads/develop 0fe9f5711 -> 66b1aa4a4
SLIDER-525. Storm app package for windows does not report component status correctly HBase Win package has the ability to provide custom ports for Thrift and Thrift2 Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/66b1aa4a Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/66b1aa4a Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/66b1aa4a Branch: refs/heads/develop Commit: 66b1aa4a403bfe60c026b63f201f8cdc539f2dfe Parents: 0fe9f57 Author: Sumit Mohanty <[email protected]> Authored: Wed Nov 5 09:00:06 2014 -0800 Committer: Sumit Mohanty <[email protected]> Committed: Wed Nov 5 09:00:10 2014 -0800 ---------------------------------------------------------------------- app-packages/hbase-win/README.txt | 2 +- .../hbase-win/package/scripts/hbase_service.py | 13 +++++++++ .../hbase-win/package/scripts/params.py | 10 ++++--- app-packages/hbase-win/resources-default.json | 14 ++++----- .../storm-win/package/scripts/nimbus.py | 2 +- .../storm-win/package/scripts/rest_api.py | 2 +- .../storm-win/package/scripts/service.py | 30 ++++++++++++++++---- .../storm-win/package/scripts/supervisor.py | 2 +- .../storm-win/package/scripts/ui_server.py | 2 +- .../src/main/python/agent/Controller.py | 2 +- 10 files changed, 56 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66b1aa4a/app-packages/hbase-win/README.txt ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/README.txt b/app-packages/hbase-win/README.txt index a413e76..6389fb2 100644 --- a/app-packages/hbase-win/README.txt +++ b/app-packages/hbase-win/README.txt @@ -30,7 +30,7 @@ mvn clean package -Phbase-app-package-win -Dpkg.version=<version> -Dpkg.name=<file name of app zip file> -Dpkg.src=<folder location where the pkg is available> Example: -mvn clean package -Phbase-app-package -Dpkg.version=0.98.5-hadoop2 +mvn clean package -Phbase-app-package-win -Dpkg.version=0.98.5-hadoop2 -Dpkg.name=hbase-0.98.5-hadoop2-bin.zip -Dpkg.src=/Users/user1/Downloads http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66b1aa4a/app-packages/hbase-win/package/scripts/hbase_service.py ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/package/scripts/hbase_service.py b/app-packages/hbase-win/package/scripts/hbase_service.py index 9a8dc56..e269531 100644 --- a/app-packages/hbase-win/package/scripts/hbase_service.py +++ b/app-packages/hbase-win/package/scripts/hbase_service.py @@ -28,6 +28,8 @@ def hbase_service( import params pid_file = format("{pid_dir}/hbase-{hbase_user}-{name}.pid") + custom_port = None + custom_info_port = None heap_size = params.master_heapsize main_class = "org.apache.hadoop.hbase.master.HMaster" if name == "regionserver": @@ -36,12 +38,17 @@ def hbase_service( if name == "rest": heap_size = params.restserver_heapsize main_class = "org.apache.hadoop.hbase.rest.RESTServer" + custom_port = params.rest_port if name == "thrift": heap_size = params.thriftserver_heapsize main_class = "org.apache.hadoop.hbase.thrift.ThriftServer" + custom_port = params.thrift_port + custom_info_port = params.thrift_info_port if name == "thrift2": heap_size = params.thrift2server_heapsize main_class = "org.apache.hadoop.hbase.thrift2.ThriftServer" + custom_port = params.thrift2_port + custom_info_port = params.thrift2_info_port role_user = format("{hbase_user}-{name}") @@ -49,6 +56,12 @@ def hbase_service( process_cmd = format("{java64_home}\\bin\\java {rest_of_the_command} {main_class} {action}") + if custom_port: + process_cmd = format("{process_cmd} -p {custom_port}") + + if custom_info_port: + process_cmd = format("{process_cmd} --infoport {custom_info_port}") + Execute(process_cmd, logoutput=False, wait_for_finish=False, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66b1aa4a/app-packages/hbase-win/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/package/scripts/params.py b/app-packages/hbase-win/package/scripts/params.py index 4f55cd0..90d274c 100644 --- a/app-packages/hbase-win/package/scripts/params.py +++ b/app-packages/hbase-win/package/scripts/params.py @@ -42,11 +42,13 @@ input_conf_files_dir = config['configurations']['global']['app_input_conf_dir'] hbase_hdfs_root_dir = config['configurations']['hbase-site']['hbase.rootdir'] """ -Read various ports, unused as of now +Read various ports """ -rest_port = config['configurations']['global']['hbase_rest_port'] -thrift_port = config['configurations']['global']['hbase_thrift_port'] -thrift2_port = config['configurations']['global']['hbase_thrift2_port'] +rest_port = default("configurations/global/hbase_rest_port", 1700) +thrift_port = default("configurations/global/hbase_thrift_port", 9090) +thrift2_port = default("configurations/global/hbase_thrift2_port", 9091) +thrift_info_port = default("configurations/global/hbase_info_thrift_port", 9095) +thrift2_info_port = default("configurations/global/hbase_info_thrift2_port", 9096) """ Compute or read various heap sizes http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66b1aa4a/app-packages/hbase-win/resources-default.json ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/resources-default.json b/app-packages/hbase-win/resources-default.json index e451c51..93dc17c 100644 --- a/app-packages/hbase-win/resources-default.json +++ b/app-packages/hbase-win/resources-default.json @@ -10,7 +10,7 @@ "HBASE_MASTER": { "yarn.role.priority": "1", "yarn.component.instances": "1", - "yarn.memory": "256" + "yarn.memory": "512" }, "slider-appmaster": { "yarn.memory": "1024" @@ -18,22 +18,22 @@ "HBASE_REGIONSERVER": { "yarn.role.priority": "2", "yarn.component.instances": "1", - "yarn.memory": "256" + "yarn.memory": "512" }, "HBASE_REST": { "yarn.role.priority": "3", "yarn.component.instances": "1", - "yarn.memory": "256" + "yarn.memory": "512" }, "HBASE_THRIFT": { "yarn.role.priority": "4", - "yarn.component.instances": "0", - "yarn.memory": "256" + "yarn.component.instances": "1", + "yarn.memory": "512" }, "HBASE_THRIFT2": { "yarn.role.priority": "5", - "yarn.component.instances": "1", - "yarn.memory": "256" + "yarn.component.instances": "0", + "yarn.memory": "512" } } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66b1aa4a/app-packages/storm-win/package/scripts/nimbus.py ---------------------------------------------------------------------- diff --git a/app-packages/storm-win/package/scripts/nimbus.py b/app-packages/storm-win/package/scripts/nimbus.py index fafc97a..c7c3120 100644 --- a/app-packages/storm-win/package/scripts/nimbus.py +++ b/app-packages/storm-win/package/scripts/nimbus.py @@ -49,7 +49,7 @@ class Nimbus(Script): def status(self, env): import status_params env.set_params(status_params) - #check_process_status(status_params.pid_nimbus) + check_process_status(status_params.pid_nimbus) if __name__ == "__main__": Nimbus().execute() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66b1aa4a/app-packages/storm-win/package/scripts/rest_api.py ---------------------------------------------------------------------- diff --git a/app-packages/storm-win/package/scripts/rest_api.py b/app-packages/storm-win/package/scripts/rest_api.py index 6a09e4e..33d8924 100644 --- a/app-packages/storm-win/package/scripts/rest_api.py +++ b/app-packages/storm-win/package/scripts/rest_api.py @@ -51,7 +51,7 @@ class StormRestApi(Script): def status(self, env): import status_params env.set_params(status_params) - #check_process_status(status_params.pid_rest_api) + check_process_status(status_params.pid_rest_api) if __name__ == "__main__": StormRestApi().execute() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66b1aa4a/app-packages/storm-win/package/scripts/service.py ---------------------------------------------------------------------- diff --git a/app-packages/storm-win/package/scripts/service.py b/app-packages/storm-win/package/scripts/service.py index b17c636..aa7d339 100644 --- a/app-packages/storm-win/package/scripts/service.py +++ b/app-packages/storm-win/package/scripts/service.py @@ -20,9 +20,9 @@ limitations under the License. from resource_management import * -import time import os import sys +import xml.etree.ElementTree as et """ Slider package uses jps as pgrep does not list the whole process start command @@ -40,14 +40,32 @@ def service( os.environ['STORM_LOG_DIR'] = params.log_dir os.environ['STORM_HOME'] = params.app_root os.environ['STORM_CONF_DIR'] = params.conf_dir - cmd = format("{storm_bin} {name} > {log_dir}/{name}.out 2>&1") - Execute(cmd, - logoutput=False, - wait_for_finish=False, - pid_file = pid_file + generate_xml = format("{storm_bin} --service {name} > {log_dir}/{name}.cmd") + + Execute(generate_xml, + logoutput=True, + wait_for_finish=True ) + tree = et.parse(format("{log_dir}/{name}.cmd")) + root = tree.getroot() + cmd_part = None + for child in root: + if child.tag == "arguments": + cmd_part = child.text + + if cmd_part: + cmd = format("{java64_home}\\bin\\java {cmd_part}") + + Execute(cmd, + logoutput=False, + wait_for_finish=False, + pid_file=pid_file + ) + else: + Logger.warn("Valid command file did not get generated at " + format("{log_dir}/{name}.cmd")) + elif action == "stop": pid = format("`cat {pid_file}` >/dev/null 2>&1") Execute(format("kill {pid}") http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66b1aa4a/app-packages/storm-win/package/scripts/supervisor.py ---------------------------------------------------------------------- diff --git a/app-packages/storm-win/package/scripts/supervisor.py b/app-packages/storm-win/package/scripts/supervisor.py index eba9fa9..47c20c9 100644 --- a/app-packages/storm-win/package/scripts/supervisor.py +++ b/app-packages/storm-win/package/scripts/supervisor.py @@ -53,7 +53,7 @@ class Supervisor(Script): import status_params env.set_params(status_params) - #check_process_status(status_params.pid_supervisor) + check_process_status(status_params.pid_supervisor) if __name__ == "__main__": http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66b1aa4a/app-packages/storm-win/package/scripts/ui_server.py ---------------------------------------------------------------------- diff --git a/app-packages/storm-win/package/scripts/ui_server.py b/app-packages/storm-win/package/scripts/ui_server.py index 0875b76..0fe7cd2 100644 --- a/app-packages/storm-win/package/scripts/ui_server.py +++ b/app-packages/storm-win/package/scripts/ui_server.py @@ -49,7 +49,7 @@ class UiServer(Script): def status(self, env): import status_params env.set_params(status_params) - #check_process_status(status_params.pid_ui) + check_process_status(status_params.pid_ui) if __name__ == "__main__": UiServer().execute() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66b1aa4a/slider-agent/src/main/python/agent/Controller.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/Controller.py b/slider-agent/src/main/python/agent/Controller.py index fe7047d..387bc7e 100644 --- a/slider-agent/src/main/python/agent/Controller.py +++ b/slider-agent/src/main/python/agent/Controller.py @@ -295,7 +295,7 @@ class Controller(threading.Thread): if 'restartEnabled' in response: restartEnabled = response['restartEnabled'] if restartEnabled: - logger.info("Component auto-restart is enabled.") + logger.debug("Component auto-restart is enabled.") if 'hasMappedComponents' in response.keys(): self.hasMappedComponents = response['hasMappedComponents'] != False
