Author: degenaro Date: Tue Apr 10 19:21:20 2018 New Revision: 1828851 URL: http://svn.apache.org/viewvc?rev=1828851&view=rev Log: UIMA-5742 Reliable DUCC
- support ducc.head.reliable.list in ducc.properties Modified: uima/uima-ducc/branches/reliable-ducc/src/main/admin/ducc_head_mode.py uima/uima-ducc/branches/reliable-ducc/src/main/admin/ducc_util.py uima/uima-ducc/branches/reliable-ducc/src/main/resources/default.ducc.properties Modified: uima/uima-ducc/branches/reliable-ducc/src/main/admin/ducc_head_mode.py URL: http://svn.apache.org/viewvc/uima/uima-ducc/branches/reliable-ducc/src/main/admin/ducc_head_mode.py?rev=1828851&r1=1828850&r2=1828851&view=diff ============================================================================== --- uima/uima-ducc/branches/reliable-ducc/src/main/admin/ducc_head_mode.py (original) +++ uima/uima-ducc/branches/reliable-ducc/src/main/admin/ducc_head_mode.py Tue Apr 10 19:21:20 2018 @@ -37,19 +37,36 @@ class DuccHeadMode(DuccUtil): def debug(self,text): if(self.debug_flag): print(text) - + + keepalivd_conf = '/etc/keepalived/keepalived.conf' + + # eligible when keepalived config comprises the ip + def is_reliable_eligible(self, ip): + retVal = False + if ( os.path.exists(self.keepalivd_conf) ): + with open(self.keepalivd_conf) as f: + for line in f: + if ip in line: + retVal = True + break + return retVal + + # master when current node keepalived answers for head node ip + # backup when current node keepalived does not answer for head ip, but is capable in config + # unspecified otherwise def main(self): result = 'unspecified' try: - head = self.ducc_properties.get('ducc.head') - if(self.is_reliable_head_eligible(head)): + ducc_head = self.ducc_properties.get('ducc.head') + head_ip = self.get_ip_address(ducc_head) + if(self.is_reliable_eligible(head_ip)): text = 'cmd: ', '/sbin/ip', 'addr', 'list' self.debug(text) p = subprocess.Popen(['/sbin/ip', 'addr', 'list'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, err = p.communicate() text = "output: "+output self.debug(text) - if(head in output): + if(head_ip in output): result = 'master' else: result = 'backup' Modified: uima/uima-ducc/branches/reliable-ducc/src/main/admin/ducc_util.py URL: http://svn.apache.org/viewvc/uima/uima-ducc/branches/reliable-ducc/src/main/admin/ducc_util.py?rev=1828851&r1=1828850&r2=1828851&view=diff ============================================================================== --- uima/uima-ducc/branches/reliable-ducc/src/main/admin/ducc_util.py (original) +++ uima/uima-ducc/branches/reliable-ducc/src/main/admin/ducc_util.py Tue Apr 10 19:21:20 2018 @@ -65,6 +65,12 @@ import db_util as dbu global use_threading use_threading = True +ducc_util_debug_flag = False + +def debug(label,data): + if(ducc_util_debug_flag): + print label, data + # The "ducc" userid is the user that installed DUCC and created this file. # If the admin dir's permissions were 700 then could assume the current user is the ducc user def find_ducc_uid(): @@ -539,31 +545,119 @@ class DuccUtil(DuccBase): return False return True - keepalivd_conf = '/etc/keepalived/keepalived.conf' + # determine if string represent an integer + def is_int(self,string): + result = True + try: + number = int(string) + except: + result = False + return result + + # transform hostname into ip address + def get_ip_address(self,hostname): + result = None + try: + p = subprocess.Popen(['/usr/bin/nslookup', hostname], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output, err = p.communicate() + #print hostname, output, err + name = None + for line in output.splitlines(): + tokens = line.split() + if(len(tokens) == 2): + t0 = tokens[0] + t1 = tokens[1] + if(t0 == 'Address:'): + if(name != None): + result = t1 + break + elif(t0 == 'Name:'): + name = t1 + except Exception as e: + print e + debug('ip_address: ', str(result)) + return result + + # get all possible hostnames & ip addresses for a head node + def get_head_node_list_variations(self): + head_node_list = [] + # add ducc.head.reliable.list node(s) + ducc_head_list = self.ducc_properties.get("ducc.head.reliable.list") + if(ducc_head_list != None): + ducc_head_nodes = ducc_head_list.split() + if(len(ducc_head_nodes)== 0): + pass + elif(len(ducc_head_nodes)== 1): + print '>>> ERROR - "ducc.head.reliable.list" missing or invalid.' + sys.exit(1); + else: + head_node_list = ducc_head_nodes + # add ducc.head node + ducc_head = self.ducc_properties.get("ducc.head") + if(ducc_head == None): + print '>>> ERROR - "ducc.head" missing or invalid.' + sys.exit(1); + ducc_head_nodes = ducc_head.split() + if(len(ducc_head_nodes) != 1): + print '>>> ERROR - "ducc.head" missing or invalid.' + sys.exit(1); + head_node = ducc_head_nodes[0] + if(not head_node in head_node_list): + head_node_list.append(head_node) + # add short names + list = head_node_list + for node in list: + short_name = node.split('.')[0] + if(not self.is_int(short_name)): + if(not short_name in head_node_list): + head_node_list.append(short_name) + # add ip addresses + list = head_node_list + for node in list: + ip = self.get_ip_address(node) + if(ip != None): + if(not ip in head_node_list): + head_node_list.append(ip) + # + debug('head_node_list: ', head_node_list) + return head_node_list - def is_reliable_head_eligible(self, head): - retVal = False - if ( os.path.exists(self.keepalivd_conf) ): - with open(self.keepalivd_conf) as f: - for line in f: - if head in line: - retVal = True - break - return retVal + # drop domain and whitespace + def normalize(self,name): + result = name + if(name != None): + result = name + result = result.strip() + result = result.split('.')[0] + return result + + # get current host's name + def get_node_name(self): + node_name = 'unknown' + cmd = '/bin/hostname' + resp = self.popen(cmd) + lines = resp.readlines() + if(len(lines)== 1): + name = lines[0] + node_name = self.normalize(name) + debug('node_name: ', node_name) + return node_name # Exit if this is not the head node. Ignore the domain as uname sometimes drops it. # Also check that ssh to this node works # Also restrict operations to the userid that installed ducc def verify_head(self): - head = self.ducc_properties.get("ducc.head").split('.')[0] - if(self.is_reliable_head_eligible(head)): - node = 'localhost' + head_node_list = self.get_head_node_list_variations() + node = self.get_node_name() + if(node in head_node_list): + pass else: - local = self.localhost.split('.')[0] - if local != head: - print ">>> ERROR - this script must be run from the head node" + ip = self.get_ip_address(node) + if(ip in head_node_list): + pass + else: + print ">>> ERROR - "+node+" not configured as head node." sys.exit(1); - node = head if(self.ssh_operational(node)): text = "ssh is operational to "+node #print text Modified: uima/uima-ducc/branches/reliable-ducc/src/main/resources/default.ducc.properties URL: http://svn.apache.org/viewvc/uima/uima-ducc/branches/reliable-ducc/src/main/resources/default.ducc.properties?rev=1828851&r1=1828850&r2=1828851&view=diff ============================================================================== --- uima/uima-ducc/branches/reliable-ducc/src/main/resources/default.ducc.properties (original) +++ uima/uima-ducc/branches/reliable-ducc/src/main/resources/default.ducc.properties Tue Apr 10 19:21:20 2018 @@ -47,11 +47,18 @@ # Resource Manager, Process Manager, Service Manager). This property is required and MUST be # configured in new installation. The installation script ducc_post_install initializes this # property to the node the script is executed on. -# Reliable DUCC: if running reliably, then this value must be the same as that specified -# for the virtual_ipaddress in /etc/keepalived/keepalived.conf. DUCC CLI and Agents employ -# this value to connect to the current reliable DUCC head node. +# Reliable DUCC: if running reliably, then this value must resolve to the same ip address +# specified for the virtual_ipaddress in /etc/keepalived/keepalived.conf for master and +# backup nodes. DUCC CLI and Agents employ this value to connect to the current reliable +# DUCC head node. ducc.head = <head-node> +# Reliable DUCC: if running reliably, then this value must comprise the blank delimited list +# of nodes that are eligible to become the DUCC head node. Admin commands start_ducc and +# stop_ducc are only allowed on the ducc.head node or any node in the ducc.head.reliable.list. +# An empty ducc.head.reliable.list indicates that DUCC is not running in reliably. +ducc.head.reliable.list = + # The full name of the Java command. # This specifies the full path to the JVM to be used by the DUCC processes. This MUST be # configured. The installation script ducc_post_install initializes this property to