Repository: incubator-hawq Updated Branches: refs/heads/master 5c512afc4 -> 1ad7dcdf4
HAWQ-1362. Add RPS start/stop script in HAWQ start/stop script Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/1ad7dcdf Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/1ad7dcdf Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/1ad7dcdf Branch: refs/heads/master Commit: 1ad7dcdf442220e427a6dd6d5bd93391a0a03582 Parents: 5c512af Author: stanlyxiang <[email protected]> Authored: Tue Mar 7 18:02:20 2017 +0800 Committer: Wen Lin <[email protected]> Committed: Wed Mar 8 14:09:15 2017 +0800 ---------------------------------------------------------------------- ranger-plugin/Makefile | 2 +- ranger-plugin/conf/rps.properties | 8 +++++- ranger-plugin/scripts/rps.sh | 2 +- tools/bin/hawq_ctl | 51 ++++++++++++++++++++++++++++++++++ tools/bin/hawqstate | 42 +++++++++++++++++++++++++++- 5 files changed, 101 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1ad7dcdf/ranger-plugin/Makefile ---------------------------------------------------------------------- diff --git a/ranger-plugin/Makefile b/ranger-plugin/Makefile index 48ba32f..735a473 100644 --- a/ranger-plugin/Makefile +++ b/ranger-plugin/Makefile @@ -30,7 +30,7 @@ ifeq ($(enable_rps), yes) all: ifdef MAVEN $(MAVEN) $(MVN_OPTS) clean - $(MAVEN) $(MVN_OPTS) packag + $(MAVEN) $(MVN_OPTS) package $(MAVEN) $(MVN_OPTS) install else @$(missing) mvn $< $(MVN_OPTS) clean http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1ad7dcdf/ranger-plugin/conf/rps.properties ---------------------------------------------------------------------- diff --git a/ranger-plugin/conf/rps.properties b/ranger-plugin/conf/rps.properties index 60545c1..a08a9f2 100644 --- a/ranger-plugin/conf/rps.properties +++ b/ranger-plugin/conf/rps.properties @@ -29,4 +29,10 @@ RPS_SHUTDOWN_PORT=8405 RANGER_HAWQ_INSTANCE=hawq # version of the software -RPS_VERSION=${project.version} \ No newline at end of file +RPS_VERSION=${project.version} + +# catalina home path of Tomcat/6.0.x for Ranger Plugin Service +SYS_CATALINA_HOME= + +# use SYS_CATALINA_HOME in default and use /usr/lib/bigtop-tomcat/ if SYS_CATALINA_HOME not set. +CATALINA_HOME=${SYS_CATALINA_HOME:-/usr/lib/bigtop-tomcat} http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1ad7dcdf/ranger-plugin/scripts/rps.sh ---------------------------------------------------------------------- diff --git a/ranger-plugin/scripts/rps.sh b/ranger-plugin/scripts/rps.sh index 476d0d6..cc63af0 100755 --- a/ranger-plugin/scripts/rps.sh +++ b/ranger-plugin/scripts/rps.sh @@ -32,7 +32,7 @@ BASEDIR=$( dirname ${CWDIR} ) # read properties from the file source ${BASEDIR}/etc/rps.properties -export CATALINA_HOME=/usr/lib/bigtop-tomcat +export CATALINA_HOME export CATALINA_BASE=${BASEDIR}/plugin-service export CATALINA_PID=${CATALINA_BASE}/work/rps.pid http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1ad7dcdf/tools/bin/hawq_ctl ---------------------------------------------------------------------- diff --git a/tools/bin/hawq_ctl b/tools/bin/hawq_ctl index 81296d7..5a9697c 100755 --- a/tools/bin/hawq_ctl +++ b/tools/bin/hawq_ctl @@ -513,6 +513,7 @@ class HawqStart: self.ignore_bad_hosts = opts.ignore_bad_hosts self._get_config() + self.hawq_acl_type = self.check_hawq_acl_type() def _get_config(self): logger.info("Gathering information and validating the environment...") @@ -544,6 +545,11 @@ class HawqStart: logger.info("No standby host configured") self.standby_host_name = '' + def check_hawq_acl_type(self): + if 'hawq_acl_type' in self.hawq_dict and self.hawq_dict['hawq_acl_type'].lower() == 'ranger': + return "ranger" + return "standalone" + def _check_recovery_start(self): cmd = "%s; %s/bin/pg_controldata %s |grep 'Database cluster state';" % (source_hawq_env, self.GPHOME, self.master_data_directory) result, stdout, stderr = remote_ssh_output(cmd, self.master_host_name, '') @@ -684,6 +690,8 @@ class HawqStart: def _start_all_nodes(self): logger.info("Start all the nodes in hawq cluster") + if self.hawq_acl_type == 'ranger': + self.start_rps() if self.standby_host_name.lower() not in ('', 'none'): logger.info("Starting standby master '%s'" % self.standby_host_name) check_return_code(self.start_standby(), logger, "Standby master start failed, exit", @@ -729,8 +737,20 @@ class HawqStart: logger.info("Segments started successfully") return node_init.return_flag + def _start_rps(self): + logger.info("Start ranger plugin service") + cmd_str = "%s/ranger/bin/rps.sh start" % (self.GPHOME) + result = remote_ssh(cmd_str, self.master_host_name, self.user) + return result + + def start_rps(self): + check_return_code(self._start_rps(), logger, \ + "Ranger plugin service start failed, exit", "Ranger plugin service started successfully") + def run(self): if self.node_type == "master": + if self.hawq_acl_type == 'ranger': + self.start_rps() check_return_code(self.start_master(), logger, \ "Master start failed, exit", "Master started successfully") elif self.node_type == "standby": @@ -774,6 +794,7 @@ class HawqStop: self.conn = None self._get_config() self.ignore_bad_hosts = opts.ignore_bad_hosts + self.hawq_acl_type = self.check_hawq_acl_type() def _get_config(self): check_items = ('hawq_master_address_host', 'hawq_master_address_port', @@ -803,6 +824,22 @@ class HawqStop: logger.info("No standby host configured") self.standby_host_name = '' + def check_hawq_acl_type(self): + try: + dburl = dbconn.DbURL(port=self.master_port, username=self.user, dbname='template1') + conn = dbconn.connect(dburl, True) + query = "select name, setting from pg_catalog.pg_settings where name='hawq_acl_type';" + rows = dbconn.execSQL(conn, query) + conn.close() + except DatabaseError, ex: + logger.error("Failed to connect to database, this script can only be run when the database is up") + sys.exit(1) + + for row in rows: + if row[1].lower() == 'ranger': + return "ranger" + return "standalone" + def _stop_master_checks(self): try: total_connections = 0 @@ -924,6 +961,8 @@ class HawqStop: logger.error("Standby master %s failed" % self.stop_action) else: logger.info("Standby master %s successfully" % self.stop_action_past) + if self.hawq_acl_type == 'ranger': + self.stop_rps() # Execute segment stop command on each node. segments_return_flag = self._stopAllSegments() @@ -999,10 +1038,22 @@ class HawqStop: logger.info("Segments %s successfully" % self.stop_action_past) return total_return_flag + def _stop_rps(self): + cmd_str = "%s/ranger/bin/rps.sh stop" % (self.GPHOME) + result = remote_ssh(cmd_str, self.master_host_name, self.user) + return result + + def stop_rps(self): + logger.info("Stop Ranger plugin service") + check_return_code(self._stop_rps(), logger, \ + "Ranger plugin service stop failed, exit", "Ranger plugin service stopped successfully") + def run(self): if self.node_type == "master": check_return_code(self._stop_master(), logger, \ "Master %s failed, exit" % self.stop_action, "Master %s successfully" % self.stop_action_past) + if self.hawq_acl_type == 'ranger': + self.stop_rps() elif self.node_type == "standby": if self.standby_host_name.lower() not in ('', 'none'): check_return_code(self._stop_standby(), logger, \ http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1ad7dcdf/tools/bin/hawqstate ---------------------------------------------------------------------- diff --git a/tools/bin/hawqstate b/tools/bin/hawqstate index d9e41d3..7dd1f4f 100755 --- a/tools/bin/hawqstate +++ b/tools/bin/hawqstate @@ -18,6 +18,8 @@ import os import sys +import re +import commands from optparse import Option, OptionParser from hawqpylib.hawqlib import HawqXMLParser, parse_hosts_file, check_file_exist_list from gppylib.db import dbconn @@ -53,6 +55,38 @@ def check_status(status_value): status_tag = 'Unknown' return status_tag +def get_guc_value(guc): + try: + master_host_name = hawq_site.hawq_dict['hawq_master_address_host'] + master_port_num = hawq_site.hawq_dict['hawq_master_address_port'] + dburl = dbconn.DbURL(hostname=master_host_name, port=master_port_num, dbname='template1') + conn = dbconn.connect(dburl, True) + query = "select name, setting from pg_catalog.pg_settings where name='%s';" % guc + rows = dbconn.execSQL(conn, query) + conn.close() + except DatabaseError, ex: + print "Failed to connect to database, this script can only be run when the database is up." + sys.exit(1) + for row in rows: + if row[0] == guc: + return row[1] + +def check_rps_status(): + hawq_rps_address_host = get_guc_value("hawq_rps_address_host") + hawq_rps_address_port = get_guc_value("hawq_rps_address_port") + hawq_rps_address_suffix = get_guc_value("hawq_rps_address_suffix") + params = ["curl --connect-timeout 2 http://";, hawq_rps_address_host.strip(), ":", hawq_rps_address_port.strip(), "/", hawq_rps_address_suffix.strip(), "/version"] + check_cmd = ''.join(params) + status, output = commands.getstatusoutput(check_cmd) + pattern1 = re.compile("\"version\":\"\d\.\d\.\d\.\d\"") + pattern2 = re.compile("Connection refused") + if pattern1.search(output): + status = 'Active' + elif pattern2.search(output): + status = 'Down' + else: + status = 'Unknown' + return status def show_brief_status(hawq_site, segment_list, standby_host): try: @@ -92,7 +126,7 @@ def show_brief_status(hawq_site, segment_list, standby_host): seg_pid_file_path = hawq_site.hawq_dict['hawq_segment_directory'] + "/postmaster.pid" total_seg_pid_file_found = len(check_file_exist_list(seg_pid_file_path, segment_list, '' )) total_seg_pid_file_miss = total_seg_num - total_seg_pid_file_found - logger.info("-HAWQ instance status summary") + logger.info("- HAWQ instance status summary") logger.info("-----------------------------------------------------") logger.info("- Master instance = %s" % master_status) if 'hawq_standby_address_host' in hawq_site.hawq_dict: @@ -101,6 +135,12 @@ def show_brief_status(hawq_site, segment_list, standby_host): else: logger.info("- No Standby master defined ") logger.info("- Total segment instance count from config file = %s"% total_seg_num) + if 'hawq_acl_type' in hawq_site.hawq_dict: + hawq_acl_type = hawq_site.hawq_dict['hawq_acl_type'] + logger.info("- Current HAWQ acl type = %s" % hawq_acl_type) + if hawq_acl_type == "ranger": + hawq_rps_status = check_rps_status() + logger.info("- HAWQ Ranger plugin service state = %s" % hawq_rps_status) logger.info("----------------------------------------------------- ") logger.info("- Segment Status ") logger.info("----------------------------------------------------- ")
