Repository: ambari Updated Branches: refs/heads/trunk 1813b0062 -> 0d6027983
AMBARI-14310: HAWQ status check fails and reports service as stopped. (adenissov via jaoki) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0d602798 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0d602798 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0d602798 Branch: refs/heads/trunk Commit: 0d6027983a56942ad6d7f0cd794322cf5e2c7c90 Parents: 1813b00 Author: Jun Aoki <ja...@apache.org> Authored: Thu Dec 10 11:00:11 2015 -0800 Committer: Jun Aoki <ja...@apache.org> Committed: Thu Dec 10 11:00:11 2015 -0800 ---------------------------------------------------------------------- .../HAWQ/2.0.0/package/scripts/common.py | 84 ++++++++++---------- .../HAWQ/2.0.0/package/scripts/constants.py | 61 -------------- .../2.0.0/package/scripts/hawq_constants.py | 61 ++++++++++++++ .../HAWQ/2.0.0/package/scripts/hawqmaster.py | 4 +- .../HAWQ/2.0.0/package/scripts/hawqsegment.py | 16 ++-- .../HAWQ/2.0.0/package/scripts/hawqstandby.py | 4 +- .../HAWQ/2.0.0/package/scripts/hawqstatus.py | 10 +-- .../HAWQ/2.0.0/package/scripts/master_helper.py | 36 ++++----- .../HAWQ/2.0.0/package/scripts/service_check.py | 22 ++--- .../HAWQ/2.0.0/package/scripts/utils.py | 18 ++--- 10 files changed, 158 insertions(+), 158 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/0d602798/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py index 8a62172..a0bc925 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py @@ -30,7 +30,7 @@ from resource_management.core.resources.accounts import Group, User import xml.etree.ElementTree as ET import utils -import constants +import hawq_constants def update_bashrc(source_file, target_file): @@ -41,7 +41,7 @@ def update_bashrc(source_file, target_file): """ append_src_cmd = "echo 'source {0}' >> {1}".format(source_file, target_file) src_cmd_exists = "grep 'source {0}' {1}".format(source_file, target_file) - Execute(append_src_cmd, user=constants.hawq_user, timeout=constants.default_exec_timeout, not_if=src_cmd_exists) + Execute(append_src_cmd, user=hawq_constants.hawq_user, timeout=hawq_constants.default_exec_timeout, not_if=src_cmd_exists) def setup_user(): @@ -57,12 +57,12 @@ def __create_hawq_user(): Creates HAWQ user with password and default group. """ import params - Group(constants.hawq_group, ignore_failures=True) + Group(hawq_constants.hawq_group, ignore_failures=True) - User(constants.hawq_user, - gid=constants.hawq_group, + User(hawq_constants.hawq_user, + gid=hawq_constants.hawq_group, password=crypt.crypt(params.hawq_password, "salt"), - groups=[constants.hawq_group, params.user_group], + groups=[hawq_constants.hawq_group, params.user_group], ignore_failures=True) @@ -70,8 +70,8 @@ def __set_home_dir_ownership(): """ Updates the HAWQ user home directory to be owned by gpadmin:gpadmin. """ - command = "chown -R {0}:{1} {2}".format(constants.hawq_user, constants.hawq_group, constants.hawq_home_dir) - Execute(command, timeout=constants.default_exec_timeout) + command = "chown -R {0}:{1} {2}".format(hawq_constants.hawq_user, hawq_constants.hawq_group, hawq_constants.hawq_home_dir) + Execute(command, timeout=hawq_constants.default_exec_timeout) def setup_common_configurations(): @@ -108,11 +108,11 @@ def __update_hdfs_client(): hdfs_client_dict[key] = params.hdfs_site[key] XmlConfig("hdfs-client.xml", - conf_dir=constants.hawq_config_dir, + conf_dir=hawq_constants.hawq_config_dir, configurations=ConfigDictionary(hdfs_client_dict), configuration_attributes=params.config['configuration_attributes']['hdfs-client'], - owner=constants.hawq_user, - group=constants.hawq_group, + owner=hawq_constants.hawq_user, + group=hawq_constants.hawq_group, mode=0644) @@ -160,11 +160,11 @@ def __update_yarn_client(): yarn_client_dict['yarn.resourcemanager.scheduler.ha'] = "{0},{1}".format(rm_scheduler_address_host1, rm_scheduler_address_host2) XmlConfig("yarn-client.xml", - conf_dir=constants.hawq_config_dir, + conf_dir=hawq_constants.hawq_config_dir, configurations=ConfigDictionary(yarn_client_dict), configuration_attributes=params.config['configuration_attributes']['yarn-client'], - owner=constants.hawq_user, - group=constants.hawq_group, + owner=hawq_constants.hawq_user, + group=hawq_constants.hawq_group, mode=0644) @@ -175,11 +175,11 @@ def __update_hawq_site(): import params XmlConfig("hawq-site.xml", - conf_dir=constants.hawq_config_dir, + conf_dir=hawq_constants.hawq_config_dir, configurations=params.hawq_site, configuration_attributes=params.config['configuration_attributes']['hawq-site'], - owner=constants.hawq_user, - group=constants.hawq_group, + owner=hawq_constants.hawq_user, + group=hawq_constants.hawq_group, mode=0644) @@ -188,7 +188,7 @@ def __set_osparams(): Updates parameters in sysctl.conf and limits.conf required by HAWQ. """ # Create a temp scratchpad directory - utils.create_dir_as_hawq_user(constants.hawq_tmp_dir) + utils.create_dir_as_hawq_user(hawq_constants.hawq_tmp_dir) # Suse doesn't supports loading values from files in /etc/sysctl.d # So we will have to directly edit the sysctl file @@ -208,7 +208,7 @@ def __update_limits_file(): """ import params # Ensure limits directory exists - Directory(constants.limits_conf_dir, recursive=True, owner=constants.root_user, group=constants.root_user) + Directory(hawq_constants.limits_conf_dir, recursive=True, owner=hawq_constants.root_user, group=hawq_constants.root_user) # Generate limits for hawq user limits_file_content = "#### HAWQ Limits Parameters ###########\n" @@ -221,9 +221,9 @@ def __update_limits_file(): gpadmin hard nofile 290000 key used in the configuration is of the format soft_nofile, thus strip '_' and replace with 'space' """ - limits_file_content += "{0} {1} {2}\n".format(constants.hawq_user, re.sub("_", " ", key), value.strip()) - File('{0}/{1}.conf'.format(constants.limits_conf_dir, constants.hawq_user), content=limits_file_content, - owner=constants.hawq_user, group=constants.hawq_group) + limits_file_content += "{0} {1} {2}\n".format(hawq_constants.hawq_user, re.sub("_", " ", key), value.strip()) + File('{0}/{1}.conf'.format(hawq_constants.limits_conf_dir, hawq_constants.hawq_user), content=limits_file_content, + owner=hawq_constants.hawq_user, group=hawq_constants.hawq_group) def __valid_input(value): @@ -251,26 +251,26 @@ def __update_sysctl_file(): Updates /etc/sysctl.d/hawq_sysctl.conf file with the HAWQ parameters on CentOS/RHEL. """ # Ensure sys ctl sub-directory exists - Directory(constants.sysctl_conf_dir, recursive=True, owner=constants.root_user, group=constants.root_user) + Directory(hawq_constants.sysctl_conf_dir, recursive=True, owner=hawq_constants.root_user, group=hawq_constants.root_user) # Generate temporary file with kernel parameters needed by hawq - File(constants.hawq_sysctl_tmp_file, content=__convert_sysctl_dict_to_text(), owner=constants.hawq_user, - group=constants.hawq_group) + File(hawq_constants.hawq_sysctl_tmp_file, content=__convert_sysctl_dict_to_text(), owner=hawq_constants.hawq_user, + group=hawq_constants.hawq_group) is_changed = True - if os.path.exists(constants.hawq_sysctl_tmp_file) and os.path.exists(constants.hawq_sysctl_file): - is_changed = not filecmp.cmp(constants.hawq_sysctl_file, constants.hawq_sysctl_tmp_file) + if os.path.exists(hawq_constants.hawq_sysctl_tmp_file) and os.path.exists(hawq_constants.hawq_sysctl_file): + is_changed = not filecmp.cmp(hawq_constants.hawq_sysctl_file, hawq_constants.hawq_sysctl_tmp_file) if is_changed: # Generate file with kernel parameters needed by hawq, only if something # has been changed by user - Execute("cp -p {0} {1}".format(constants.hawq_sysctl_tmp_file, constants.hawq_sysctl_file)) + Execute("cp -p {0} {1}".format(hawq_constants.hawq_sysctl_tmp_file, hawq_constants.hawq_sysctl_file)) # Reload kernel sysctl parameters from hawq file. - Execute("sysctl -e -p {0}".format(constants.hawq_sysctl_file), timeout=constants.default_exec_timeout) + Execute("sysctl -e -p {0}".format(hawq_constants.hawq_sysctl_file), timeout=hawq_constants.default_exec_timeout) # Wipe out temp file - File(constants.hawq_sysctl_tmp_file, action='delete') + File(hawq_constants.hawq_sysctl_tmp_file, action='delete') def __update_sysctl_file_suse(): @@ -278,33 +278,33 @@ def __update_sysctl_file_suse(): Updates /etc/sysctl.conf file with the HAWQ parameters on SUSE. """ # Backup file - backup_file_name = constants.sysctl_backup_file.format(str(int(time.time()))) + backup_file_name = hawq_constants.sysctl_backup_file.format(str(int(time.time()))) try: # Generate file with kernel parameters needed by hawq to temp file - File(constants.hawq_sysctl_tmp_file, content=__convert_sysctl_dict_to_text(), owner=constants.hawq_user, - group=constants.hawq_group) + File(hawq_constants.hawq_sysctl_tmp_file, content=__convert_sysctl_dict_to_text(), owner=hawq_constants.hawq_user, + group=hawq_constants.hawq_group) - sysctl_file_dict = utils.read_file_to_dict(constants.sysctl_suse_file) + sysctl_file_dict = utils.read_file_to_dict(hawq_constants.sysctl_suse_file) sysctl_file_dict_original = sysctl_file_dict.copy() - hawq_sysctl_dict = utils.read_file_to_dict(constants.hawq_sysctl_tmp_file) + hawq_sysctl_dict = utils.read_file_to_dict(hawq_constants.hawq_sysctl_tmp_file) # Merge common system file with hawq specific file sysctl_file_dict.update(hawq_sysctl_dict) if sysctl_file_dict_original != sysctl_file_dict: # Backup file - Execute("cp {0} {1}".format(constants.sysctl_suse_file, backup_file_name), timeout=constants.default_exec_timeout) + Execute("cp {0} {1}".format(hawq_constants.sysctl_suse_file, backup_file_name), timeout=hawq_constants.default_exec_timeout) # Write merged properties to file - utils.write_dict_to_file(sysctl_file_dict, constants.sysctl_suse_file) + utils.write_dict_to_file(sysctl_file_dict, hawq_constants.sysctl_suse_file) # Reload kernel sysctl parameters from /etc/sysctl.conf - Execute("sysctl -e -p", timeout=constants.default_exec_timeout) + Execute("sysctl -e -p", timeout=hawq_constants.default_exec_timeout) except Exception as e: Logger.error("Error occurred while updating sysctl.conf file, reverting the contents" + str(e)) - Execute("cp {0} {1}".format(constants.sysctl_suse_file, constants.hawq_sysctl_tmp_file)) - Execute("mv {0} {1}".format(backup_file_name, constants.sysctl_suse_file), timeout=constants.default_exec_timeout) + Execute("cp {0} {1}".format(hawq_constants.sysctl_suse_file, hawq_constants.hawq_sysctl_tmp_file)) + Execute("mv {0} {1}".format(backup_file_name, hawq_constants.sysctl_suse_file), timeout=hawq_constants.default_exec_timeout) Logger.error("Please execute `sysctl -e -p` on the command line manually to reload the contents of file {0}".format( - constants.hawq_sysctl_tmp_file)) + hawq_constants.hawq_sysctl_tmp_file)) raise Fail("Failed to update sysctl.conf file ") @@ -314,7 +314,7 @@ def get_local_hawq_site_property(property_name): """ hawq_site_path = None try: - hawq_site_path = os.path.join(constants.hawq_config_dir, "hawq-site.xml") + hawq_site_path = os.path.join(hawq_constants.hawq_config_dir, "hawq-site.xml") hawq_site_root = ET.parse(hawq_site_path).getroot() for property in hawq_site_root.findall("property"): for item in property: http://git-wip-us.apache.org/repos/asf/ambari/blob/0d602798/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/constants.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/constants.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/constants.py deleted file mode 100644 index 6edb327..0000000 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/constants.py +++ /dev/null @@ -1,61 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import os - -MASTER = "master" -STANDBY = "standby" -SEGMENT = "segment" -START = "start" -INIT = "init" -STOP = "stop" -YARN = "yarn" - -# Users -root_user = "root" -hawq_user = "gpadmin" -hawq_group = hawq_user - -# Directories -hawq_home_dir = "/usr/local/hawq/" -hawq_config_dir = "/usr/local/hawq/etc/" -hawq_pid_dir = "/var/run/hawq/" -hawq_tmp_dir = '/tmp/hawq/' -hawq_user_home_dir = os.path.expanduser("~{0}".format(hawq_user)) -limits_conf_dir = "/etc/security/limits.d" -sysctl_conf_dir = "/etc/sysctl.d" - -# Files -hawq_slaves_file = os.path.join(hawq_config_dir, "slaves") -hawq_user_bashrc_file = os.path.join(hawq_user_home_dir, ".bashrc") -hawq_greenplum_path_file = os.path.join(hawq_home_dir, "greenplum_path.sh") -hawq_hosts_file = "/tmp/hawq_hosts" -hawq_check_file = os.path.join(hawq_config_dir, "gpcheck.cnf") -sysctl_suse_file = "/etc/sysctl.conf" -sysctl_backup_file = "/etc/sysctl.conf.backup.{0}" -hawq_sysctl_filename = "hawq_sysctl.conf" -hawq_sysctl_tmp_file = os.path.join(hawq_tmp_dir, hawq_sysctl_filename) -hawq_sysctl_file = os.path.join(sysctl_conf_dir, hawq_sysctl_filename) -postmaster_opts_filename = "postmaster.opts" -postmaster_pid_filename = "postmaster.pid" - -# Smoke check table -smoke_check_table_name = "ambari_hawq_smoke_test" - -# Timeouts -default_exec_timeout = 600 -hawq_operation_exec_timeout = 900 http://git-wip-us.apache.org/repos/asf/ambari/blob/0d602798/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawq_constants.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawq_constants.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawq_constants.py new file mode 100644 index 0000000..6edb327 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawq_constants.py @@ -0,0 +1,61 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import os + +MASTER = "master" +STANDBY = "standby" +SEGMENT = "segment" +START = "start" +INIT = "init" +STOP = "stop" +YARN = "yarn" + +# Users +root_user = "root" +hawq_user = "gpadmin" +hawq_group = hawq_user + +# Directories +hawq_home_dir = "/usr/local/hawq/" +hawq_config_dir = "/usr/local/hawq/etc/" +hawq_pid_dir = "/var/run/hawq/" +hawq_tmp_dir = '/tmp/hawq/' +hawq_user_home_dir = os.path.expanduser("~{0}".format(hawq_user)) +limits_conf_dir = "/etc/security/limits.d" +sysctl_conf_dir = "/etc/sysctl.d" + +# Files +hawq_slaves_file = os.path.join(hawq_config_dir, "slaves") +hawq_user_bashrc_file = os.path.join(hawq_user_home_dir, ".bashrc") +hawq_greenplum_path_file = os.path.join(hawq_home_dir, "greenplum_path.sh") +hawq_hosts_file = "/tmp/hawq_hosts" +hawq_check_file = os.path.join(hawq_config_dir, "gpcheck.cnf") +sysctl_suse_file = "/etc/sysctl.conf" +sysctl_backup_file = "/etc/sysctl.conf.backup.{0}" +hawq_sysctl_filename = "hawq_sysctl.conf" +hawq_sysctl_tmp_file = os.path.join(hawq_tmp_dir, hawq_sysctl_filename) +hawq_sysctl_file = os.path.join(sysctl_conf_dir, hawq_sysctl_filename) +postmaster_opts_filename = "postmaster.opts" +postmaster_pid_filename = "postmaster.pid" + +# Smoke check table +smoke_check_table_name = "ambari_hawq_smoke_test" + +# Timeouts +default_exec_timeout = 600 +hawq_operation_exec_timeout = 900 http://git-wip-us.apache.org/repos/asf/ambari/blob/0d602798/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqmaster.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqmaster.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqmaster.py index 4443264..c886325 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqmaster.py +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqmaster.py @@ -21,7 +21,7 @@ from resource_management.libraries.functions.check_process_status import check_p import master_helper import common -import constants +import hawq_constants class HawqMaster(Script): """ @@ -36,7 +36,7 @@ class HawqMaster(Script): def configure(self, env): import params env.set_params(params) - env.set_params(constants) + env.set_params(hawq_constants) master_helper.configure_master() def start(self, env): http://git-wip-us.apache.org/repos/asf/ambari/blob/0d602798/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqsegment.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqsegment.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqsegment.py index e0bc75c..a99f0f7 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqsegment.py +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqsegment.py @@ -23,7 +23,7 @@ from resource_management.libraries.functions.check_process_status import check_p import utils import common -import constants +import hawq_constants class HawqSegment(Script): """ @@ -39,17 +39,17 @@ class HawqSegment(Script): import params env.set_params(params) - env.set_params(constants) + env.set_params(hawq_constants) common.setup_user() common.setup_common_configurations() - common.update_bashrc(constants.hawq_greenplum_path_file, constants.hawq_user_bashrc_file) + common.update_bashrc(hawq_constants.hawq_greenplum_path_file, hawq_constants.hawq_user_bashrc_file) def __start_segment(self): import params return utils.exec_hawq_operation( - constants.START, - "{0} -a".format(constants.SEGMENT), + hawq_constants.START, + "{0} -a".format(hawq_constants.SEGMENT), not_if=utils.chk_hawq_process_status_cmd(params.hawq_segment_address_port)) def start(self, env): @@ -67,7 +67,7 @@ class HawqSegment(Script): def stop(self, env): import params - utils.exec_hawq_operation(constants.STOP, "{0} -a".format(constants.SEGMENT), only_if=utils.chk_hawq_process_status_cmd( + utils.exec_hawq_operation(hawq_constants.STOP, "{0} -a".format(hawq_constants.SEGMENT), only_if=utils.chk_hawq_process_status_cmd( params.hawq_segment_address_port)) @@ -85,14 +85,14 @@ class HawqSegment(Script): utils.create_dir_as_hawq_user(params.hawq_segment_temp_dir.split(',')) # Initialize hawq segment - utils.exec_hawq_operation(constants.INIT, "{0} -a -v".format(constants.SEGMENT)) + utils.exec_hawq_operation(hawq_constants.INIT, "{0} -a -v".format(hawq_constants.SEGMENT)) def __is_segment_initialized(self): """ Check whether the HAWQ Segment is initialized """ import params - return os.path.exists(os.path.join(params.hawq_segment_dir, constants.postmaster_opts_filename)) + return os.path.exists(os.path.join(params.hawq_segment_dir, hawq_constants.postmaster_opts_filename)) if __name__ == "__main__": http://git-wip-us.apache.org/repos/asf/ambari/blob/0d602798/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqstandby.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqstandby.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqstandby.py index d8254f9..ad81319 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqstandby.py +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqstandby.py @@ -21,7 +21,7 @@ from resource_management.libraries.functions.check_process_status import check_p import master_helper import common -import constants +import hawq_constants class HawqStandby(Script): """ @@ -36,7 +36,7 @@ class HawqStandby(Script): def configure(self, env): import params env.set_params(params) - env.set_params(constants) + env.set_params(hawq_constants) master_helper.configure_master() def start(self, env): http://git-wip-us.apache.org/repos/asf/ambari/blob/0d602798/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqstatus.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqstatus.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqstatus.py index 26dfdd0..a1b6dd9 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqstatus.py +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqstatus.py @@ -24,7 +24,7 @@ from resource_management.core.exceptions import Fail import utils import common -import constants +import hawq_constants def get_pid_file(): @@ -37,10 +37,10 @@ def get_pid_file(): component_name = config['componentName'] component = "master" if component_name in ["HAWQMASTER", "HAWQSTANDBY"] else "segment" - hawq_pid_file = os.path.join(constants.hawq_pid_dir, "hawq-{0}.pid".format(component)) + hawq_pid_file = os.path.join(hawq_constants.hawq_pid_dir, "hawq-{0}.pid".format(component)) File(hawq_pid_file, action='delete') - utils.create_dir_as_hawq_user(constants.hawq_pid_dir) + utils.create_dir_as_hawq_user(hawq_constants.hawq_pid_dir) #Get hawq_master_directory or hawq_segment_directory value from hawq-site.xml depending #on the component @@ -49,7 +49,7 @@ def get_pid_file(): #hawq-site content from Ambari server will not be available when the #command type is STATUS_COMMAND. Hence, reading it directly from the local file postmaster_pid_file = os.path.join(common.get_local_hawq_site_property( - hawq_site_directory_property), constants.postmaster_pid_filename) + hawq_site_directory_property), hawq_constants.postmaster_pid_filename) pid = "" if os.path.exists(postmaster_pid_file): @@ -59,6 +59,6 @@ def get_pid_file(): if not pid: raise Fail("Failed to fetch pid from {0}".format(postmaster_pid_file)) - File(hawq_pid_file, content=pid, owner=constants.hawq_user, group=constants.hawq_user) + File(hawq_pid_file, content=pid, owner=hawq_constants.hawq_user, group=hawq_constants.hawq_user) return hawq_pid_file http://git-wip-us.apache.org/repos/asf/ambari/blob/0d602798/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/master_helper.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/master_helper.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/master_helper.py index 4255573..f0a0593 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/master_helper.py +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/master_helper.py @@ -24,7 +24,7 @@ from resource_management.libraries.functions.format import format import utils import common -import constants +import hawq_constants def __setup_master_specific_conf_files(): """ @@ -32,13 +32,13 @@ def __setup_master_specific_conf_files(): """ import params - File(constants.hawq_check_file, content=params.gpcheck_content, owner=constants.hawq_user, group=constants.hawq_group, + File(hawq_constants.hawq_check_file, content=params.gpcheck_content, owner=hawq_constants.hawq_user, group=hawq_constants.hawq_group, mode=0644) - File(constants.hawq_slaves_file, content=Template("slaves.j2"), owner=constants.hawq_user, group=constants.hawq_group, + File(hawq_constants.hawq_slaves_file, content=Template("slaves.j2"), owner=hawq_constants.hawq_user, group=hawq_constants.hawq_group, mode=0644) - File(constants.hawq_hosts_file, content=Template("hawq-hosts.j2"), owner=constants.hawq_user, group=constants.hawq_group, + File(hawq_constants.hawq_hosts_file, content=Template("hawq-hosts.j2"), owner=hawq_constants.hawq_user, group=hawq_constants.hawq_group, mode=0644) @@ -47,18 +47,18 @@ def __setup_passwordless_ssh(): Exchanges ssh keys to setup passwordless ssh for the hawq_user between the HAWQ Master and the HAWQ Segment nodes """ import params - utils.exec_hawq_operation("ssh-exkeys", format('-f {hawq_hosts_file} -p {hawq_password!p}', hawq_hosts_file=constants.hawq_hosts_file, hawq_password=params.hawq_password)) + utils.exec_hawq_operation("ssh-exkeys", format('-f {hawq_hosts_file} -p {hawq_password!p}', hawq_hosts_file=hawq_constants.hawq_hosts_file, hawq_password=params.hawq_password)) - File(constants.hawq_hosts_file, action='delete') + File(hawq_constants.hawq_hosts_file, action='delete') def __setup_hawq_user_profile(): """ Sets up the ENV variables for hawq_user as a convenience for the command line users """ - hawq_profile_file = os.path.join(os.path.expanduser("~{0}".format(constants.hawq_user)), ".hawq-profile.sh") - File(hawq_profile_file, content=Template("hawq-profile.sh.j2"), owner=constants.hawq_user, group=constants.hawq_group) - common.update_bashrc(hawq_profile_file, constants.hawq_user_bashrc_file) + hawq_profile_file = os.path.join(os.path.expanduser("~{0}".format(hawq_constants.hawq_user)), ".hawq-profile.sh") + File(hawq_profile_file, content=Template("hawq-profile.sh.j2"), owner=hawq_constants.hawq_user, group=hawq_constants.hawq_group) + common.update_bashrc(hawq_profile_file, hawq_constants.hawq_user_bashrc_file) def configure_master(): @@ -81,7 +81,7 @@ def __create_local_dirs(): utils.create_dir_as_hawq_user(params.hawq_master_dir) utils.create_dir_as_hawq_user(params.hawq_master_temp_dir.split(',')) - Execute("chmod 700 {0}".format(params.hawq_master_dir), user=constants.root_user, timeout=constants.default_exec_timeout) + Execute("chmod 700 {0}".format(params.hawq_master_dir), user=hawq_constants.root_user, timeout=hawq_constants.default_exec_timeout) def __create_hdfs_dirs(): @@ -89,7 +89,7 @@ def __create_hdfs_dirs(): Creates the required HDFS directories for HAWQ """ import params - params.HdfsResource(params.hawq_hdfs_data_dir, type="directory", action="create_on_execute", owner=constants.hawq_user, group=constants.hawq_group, mode=0755) + params.HdfsResource(params.hawq_hdfs_data_dir, type="directory", action="create_on_execute", owner=hawq_constants.hawq_user, group=hawq_constants.hawq_group, mode=0755) params.HdfsResource(None, action="execute") @@ -98,21 +98,21 @@ def __init_active(): Initializes the active master """ __create_hdfs_dirs() - utils.exec_hawq_operation(constants.INIT, "{0} -a -v".format(constants.MASTER)) + utils.exec_hawq_operation(hawq_constants.INIT, "{0} -a -v".format(hawq_constants.MASTER)) def __init_standby(): """ Initializes the HAWQ Standby Master """ - utils.exec_hawq_operation(constants.INIT, "{0} -a -v".format(constants.STANDBY)) + utils.exec_hawq_operation(hawq_constants.INIT, "{0} -a -v".format(hawq_constants.STANDBY)) def __get_component_name(): """ Identifies current node as either HAWQ Master or HAWQ Standby Master """ - return constants.MASTER if __is_active_master() else constants.STANDBY + return hawq_constants.MASTER if __is_active_master() else hawq_constants.STANDBY def __start_local_master(): @@ -122,7 +122,7 @@ def __start_local_master(): import params component_name = __get_component_name() utils.exec_hawq_operation( - constants.START, + hawq_constants.START, "{0} -a".format(component_name), not_if=utils.chk_hawq_process_status_cmd(params.hawq_master_address_port, component_name)) @@ -132,7 +132,7 @@ def __is_local_initialized(): Checks if the local node has been initialized """ import params - return os.path.exists(os.path.join(params.hawq_master_dir, constants.postmaster_opts_filename)) + return os.path.exists(os.path.join(params.hawq_master_dir, hawq_constants.postmaster_opts_filename)) def __get_standby_host(): @@ -149,7 +149,7 @@ def __is_standby_initialized(): """ import params - file_path = os.path.join(params.hawq_master_dir, constants.postmaster_opts_filename) + file_path = os.path.join(params.hawq_master_dir, hawq_constants.postmaster_opts_filename) (retcode, _, _) = utils.exec_ssh_cmd(__get_standby_host(), "[ -f {0} ]".format(file_path)) return retcode == 0 @@ -185,7 +185,7 @@ def stop_master(): import params component_name = __get_component_name() utils.exec_hawq_operation( - constants.STOP, + hawq_constants.STOP, "{0} -a".format(component_name), only_if=utils.chk_hawq_process_status_cmd(params.hawq_master_address_port, component_name)) http://git-wip-us.apache.org/repos/asf/ambari/blob/0d602798/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/service_check.py index 7a99f5e..c35fdce 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/service_check.py +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/service_check.py @@ -16,7 +16,7 @@ See the License for the specific language governing permissions and limitations under the License. """ import common -import constants +import hawq_constants from utils import exec_psql_cmd, exec_ssh_cmd from resource_management.libraries.script import Script from resource_management.core.exceptions import Fail @@ -54,33 +54,33 @@ class HAWQServiceCheck(Script): def drop_table(self): - Logger.info("Dropping {0} table if exists".format(constants.smoke_check_table_name)) - sql_cmd = "drop table if exists {0}".format(constants.smoke_check_table_name) + Logger.info("Dropping {0} table if exists".format(hawq_constants.smoke_check_table_name)) + sql_cmd = "drop table if exists {0}".format(hawq_constants.smoke_check_table_name) exec_psql_cmd(sql_cmd, self.active_master_host) def create_table(self): - Logger.info("Creating table {0}".format(constants.smoke_check_table_name)) - sql_cmd = "create table {0} (col1 int) distributed randomly".format(constants.smoke_check_table_name) + Logger.info("Creating table {0}".format(hawq_constants.smoke_check_table_name)) + sql_cmd = "create table {0} (col1 int) distributed randomly".format(hawq_constants.smoke_check_table_name) exec_psql_cmd(sql_cmd, self.active_master_host) def insert_data(self): - Logger.info("Inserting data to table {0}".format(constants.smoke_check_table_name)) - sql_cmd = "insert into {0} select * from generate_series(1,10)".format(constants.smoke_check_table_name) + Logger.info("Inserting data to table {0}".format(hawq_constants.smoke_check_table_name)) + sql_cmd = "insert into {0} select * from generate_series(1,10)".format(hawq_constants.smoke_check_table_name) exec_psql_cmd(sql_cmd, self.active_master_host) def query_data(self): - Logger.info("Querying data from table {0}".format(constants.smoke_check_table_name)) - sql_cmd = "select * from {0}".format(constants.smoke_check_table_name) + Logger.info("Querying data from table {0}".format(hawq_constants.smoke_check_table_name)) + sql_cmd = "select * from {0}".format(hawq_constants.smoke_check_table_name) exec_psql_cmd(sql_cmd, self.active_master_host) def check_data_correctness(self): expected_data = "55" Logger.info("Validating data inserted, finding sum of all the inserted entries. Expected output: {0}".format(expected_data)) - sql_cmd = "select sum(col1) from {0}".format(constants.smoke_check_table_name) + sql_cmd = "select sum(col1) from {0}".format(hawq_constants.smoke_check_table_name) _, stdout, _ = exec_psql_cmd(sql_cmd, self.active_master_host, tuples_only=False) if expected_data != stdout.strip(): Logger.error("Incorrect data returned. Expected Data: {0} Actual Data: {1}".format(expected_data, stdout)) @@ -89,7 +89,7 @@ class HAWQServiceCheck(Script): def check_state(self): import params - command = "source {0} && hawq state -d {1}".format(constants.hawq_greenplum_path_file, params.hawq_master_dir) + command = "source {0} && hawq state -d {1}".format(hawq_constants.hawq_greenplum_path_file, params.hawq_master_dir) Logger.info("Executing hawq status check..") (retcode, out, err) = exec_ssh_cmd(self.active_master_host, command) if retcode: http://git-wip-us.apache.org/repos/asf/ambari/blob/0d602798/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/utils.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/utils.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/utils.py index 8e2b157..316c7f2 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/utils.py +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/utils.py @@ -21,13 +21,13 @@ from resource_management.core.resources.system import Execute, Directory from resource_management.core.exceptions import Fail from resource_management.core.logger import Logger -import constants +import hawq_constants def chk_hawq_process_status_cmd(port, component_name=None): """ Check if hawq postgres / gpsyncmaster process is running """ - process = "gpsyncmaster" if component_name == constants.STANDBY else "postgres" + process = "gpsyncmaster" if component_name == hawq_constants.STANDBY else "postgres" return "netstat -tupln | egrep ':{0}\s' | egrep {1}".format(port, process) @@ -35,18 +35,18 @@ def create_dir_as_hawq_user(directory): """ Creates directories with hawq_user and hawq_group (defaults to gpadmin:gpadmin) """ - Directory(directory, recursive=True, owner=constants.hawq_user, group=constants.hawq_group) + Directory(directory, recursive=True, owner=hawq_constants.hawq_user, group=hawq_constants.hawq_group) def exec_hawq_operation(operation, option, not_if=None, only_if=None, logoutput=True): """ Sets up execution environment and runs a given command as HAWQ user """ - hawq_cmd = "source {0} && hawq {1} {2}".format(constants.hawq_greenplum_path_file, operation, option) + hawq_cmd = "source {0} && hawq {1} {2}".format(hawq_constants.hawq_greenplum_path_file, operation, option) Execute( hawq_cmd, - user=constants.hawq_user, - timeout=constants.hawq_operation_exec_timeout, + user=hawq_constants.hawq_user, + timeout=hawq_constants.hawq_operation_exec_timeout, not_if=not_if, only_if=only_if, logoutput=logoutput) @@ -82,9 +82,9 @@ def exec_ssh_cmd(hostname, cmd): import params # Only gpadmin should be allowed to run command via ssh, thus not exposing user as a parameter if params.hostname != hostname: - cmd = "su - {0} -c 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null {1} \"{2} \" '".format(constants.hawq_user, hostname, cmd) + cmd = "su - {0} -c 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null {1} \"{2} \" '".format(hawq_constants.hawq_user, hostname, cmd) else: - cmd = "su - {0} -c \"{1}\"".format(constants.hawq_user, cmd) + cmd = "su - {0} -c \"{1}\"".format(hawq_constants.hawq_user, cmd) Logger.info("Command executed: {0}".format(cmd)) process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) (stdout, stderr) = process.communicate() @@ -95,7 +95,7 @@ def exec_psql_cmd(command, host, db="template1", tuples_only=True): """ Sets up execution environment and runs the HAWQ queries """ - src_cmd = "source {0}".format(constants.hawq_greenplum_path_file) + src_cmd = "source {0}".format(hawq_constants.hawq_greenplum_path_file) if tuples_only: cmd = src_cmd + " && psql -d {0} -c \\\"{1};\\\"".format(db, command) else: