Author: degenaro Date: Thu Feb 7 21:33:32 2019 New Revision: 1853161 URL: http://svn.apache.org/viewvc?rev=1853161&view=rev Log: UIMA-5742 Reliable DUCC
> autostart.py insert host+agent start into DB of never before seen node (one > not already in DB) > autostart.py don't use ssh for utility local host operations (e.g. > query/start agent) > ducc_util.py don't create database directories when automanage == false Added: uima/uima-ducc/trunk/src/main/admin/db_autostart_insert.py (with props) Modified: uima/uima-ducc/trunk/src/main/admin/autostart.py uima/uima-ducc/trunk/src/main/admin/ducc_util.py Modified: uima/uima-ducc/trunk/src/main/admin/autostart.py URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/autostart.py?rev=1853161&r1=1853160&r2=1853161&view=diff ============================================================================== --- uima/uima-ducc/trunk/src/main/admin/autostart.py (original) +++ uima/uima-ducc/trunk/src/main/admin/autostart.py Thu Feb 7 21:33:32 2019 @@ -114,6 +114,10 @@ class AutoStart(DuccUtil): 'ws':'ws', } + def __init__(self): + DuccUtil.__init__(self) + self.ssh_enabled = False + # return file name def _fn(self): fpath = __file__.split('/') @@ -144,7 +148,6 @@ class AutoStart(DuccUtil): LOGLEVEL = os.environ.get('LOGLEVEL','info') self.logger = Logger(LOGFILE,LOGLEVEL) - # check if host names with domain match def is_host_match_with_domain(self,h1,h2): retVal = False @@ -195,9 +198,10 @@ class AutoStart(DuccUtil): pass return retVal - # get daemons started (from DB) - def get_daemons_started(self): - daemons = [] + # get daemons started/all in DB for host + def get_daemons_host(self): + db = [] + started = [] jclass = 'org.apache.uima.ducc.database.lifetime.DbDaemonLifetimeUI' option = '--query' cmd = [self.jvm, '-DDUCC_HOME='+self.DUCC_HOME, jclass, option] @@ -207,8 +211,9 @@ class AutoStart(DuccUtil): for line in lines: host, daemon, state = self.parse_line(line) if(self.is_host_match(self.LOCAL_HOST, host)): + db.append(daemon) if(state == 'Start'): - daemons.append(daemon) + started.append(daemon) text = 'add'+' '+host+' '+daemon self.logger.debug(self._mn(),text) else: @@ -217,16 +222,16 @@ class AutoStart(DuccUtil): else: text = 'skip'+' '+host+' '+daemon self.logger.debug(self._mn(),text) - text = 'daemons'+' '+str(daemons) + text = 'started'+' '+str(started)+' '+'db'+' '+str(db) self.logger.debug(self._mn(),text) - return daemons + return db, started def normalize_component(self,component): daemon = component[:2] return daemon # get daemons running (from system) - def get_daemons_running(self): + def get_components_running(self): daemons = [] result = self.find_ducc_process(self.LOCAL_HOST) find_status = result[0] @@ -271,19 +276,50 @@ class AutoStart(DuccUtil): out, err = p.communicate() text = str(out) self.logger.info(self._mn(),text) - + + def insert(self): + python_script = os.path.join(self.DUCC_HOME,'admin','db_autostart_insert.py') + node = self.get_node_name() + component = 'ag' + cmd = [ python_script, '--host', node, '--name', component] + text = str(cmd) + self.logger.info(self._mn(),text) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + text = str(out) + self.logger.info(self._mn(),text) + # autostart: start head or agent daemons, as required def main(self, argv): NAME = 'autostart' self.setup_logging(NAME) self.get_args() try: - daemons_started = self.get_daemons_started() - daemons_running = self.get_daemons_running() + daemons_db, daemons_started = self.get_daemons_host() + text = 'daemons_db '+str(len(daemons_db)) + self.logger.debug(self._mn(),text) + text = 'daemons_started '+str(len(daemons_started)) + self.logger.debug(self._mn(),text) + # if agent node is not in db, then insert it! + if(len(daemons_db) == 0): + if(self.is_head_node()): + pass + else: + self.insert() + daemons_db, daemons_started = self.get_daemons_host() + components_running = self.get_components_running() + text = 'components_running '+str(components_running) + self.logger.debug(self._mn(),text) for daemon in daemons_started: - if(not daemon in daemons_running): + component = self.normalize_component(daemon) + text = 'component '+str(component) + self.logger.debug(self._mn(),text) + if(component in components_running): + pass + else: + text = 'start daemon '+str(daemon) + self.logger.debug(self._mn(),text) self.start(daemon) - except Exception,e: lines = traceback.format_exc().splitlines() for line in lines: Added: uima/uima-ducc/trunk/src/main/admin/db_autostart_insert.py URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/db_autostart_insert.py?rev=1853161&view=auto ============================================================================== --- uima/uima-ducc/trunk/src/main/admin/db_autostart_insert.py (added) +++ uima/uima-ducc/trunk/src/main/admin/db_autostart_insert.py Thu Feb 7 21:33:32 2019 @@ -0,0 +1,93 @@ +#!/usr/bin/env python +# ----------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ----------------------------------------------------------------------- + +import sys + +version_min = [2, 7] +version_info = sys.version_info +version_error = False +if(version_info[0] < version_min[0]): + version_error = True +elif(version_info[0] == version_min[0]): + if(version_info[1] < version_min[1]): + version_error = True +if(version_error): + print('Python minimum requirement is version '+str(version_min[0])+'.'+str(version_min[1])) + sys.exit(1) + +import argparse +import os +import subprocess + +from ducc_util import DuccUtil + +# command to insert into the autostart database table the specified host & daemon + +class AutostartInsert(DuccUtil): + + valid_names = [ 'ag', 'br', 'or', 'pm', 'rm', 'sm', 'ws' ] + jclass = 'org.apache.uima.ducc.database.lifetime.DbDaemonLifetimeUI' + + description = 'Insert an entry into the autostart database table, if one does not already exist.' + + def get_args(self): + parser = argparse.ArgumentParser(description=self.description) + parser.add_argument('--host', action='store', required=True, help='the DUCC daemon host') + parser.add_argument('--name', action='store', required=True, choices=self.valid_names, help='the DUCC daemon name') + self.args = parser.parse_args() + + def find(self): + retVal = False + option = '--query' + cmd = [self.jvm, '-DDUCC_HOME='+self.DUCC_HOME, self.jclass, option] + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + lines = out.split('\n') + for line in lines: + tokens = line.split('.') + if(len(tokens) == 2): + host = tokens[0] + name = tokens[1].split('=')[0] + if(host == self.args.host): + if(name == self.args.name): + retVal = True + return retVal + + def insert(self): + option = '--start' + cmd = [self.jvm, '-DDUCC_HOME='+self.DUCC_HOME, self.jclass, option, self.args.host, self.args.name] + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + + def main(self, argv): + self.get_args() + if(self.find()): + print 'already exists' + else: + self.insert() + if(self.find()): + print 'inserted' + else: + print 'insert failed' + +if __name__ == "__main__": + + instance = AutostartInsert() + instance.main(sys.argv[1:]) Propchange: uima/uima-ducc/trunk/src/main/admin/db_autostart_insert.py ------------------------------------------------------------------------------ svn:executable = * Propchange: uima/uima-ducc/trunk/src/main/admin/db_autostart_insert.py ------------------------------------------------------------------------------ svn:mime-type = text/plain Modified: uima/uima-ducc/trunk/src/main/admin/ducc_util.py URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1853161&r1=1853160&r2=1853161&view=diff ============================================================================== --- uima/uima-ducc/trunk/src/main/admin/ducc_util.py (original) +++ uima/uima-ducc/trunk/src/main/admin/ducc_util.py Thu Feb 7 21:33:32 2019 @@ -1451,10 +1451,17 @@ class DuccUtil(DuccBase): if ( dbhost == None ): dbhost = 'localhost' - dir_db_state = self.DUCC_HOME + '/state/database/'+dbhost - self.makedirs(dir_db_state) - dir_db_logs = self.DUCC_HOME + '/logs/database/'+dbhost - self.makedirs(dir_db_logs) + + manage_database = self.ducc_properties.get('ducc.database.automanage') + self.automanage_database = False + if (manage_database in ('t', 'true', 'T', 'True')) : + self.automanage_database = True + + if(manage_database): + dir_db_state = self.DUCC_HOME + '/state/database/'+dbhost + self.makedirs(dir_db_state) + dir_db_logs = self.DUCC_HOME + '/logs/database/'+dbhost + self.makedirs(dir_db_logs) self.db_pidfile = dir_db_state+ '/cassandra.pid' self.db_logfile = dir_db_logs + '/cassandra.console' @@ -1472,11 +1479,6 @@ class DuccUtil(DuccBase): if (manage_broker in ('t', 'true', 'T', 'True')) : self.automanage_broker = True - manage_database = self.ducc_properties.get('ducc.database.automanage') - self.automanage_database = False - if (manage_database in ('t', 'true', 'T', 'True')) : - self.automanage_database = True - py_version = platform.python_version().split('.') if ( int(py_version[0]) > 2 ): print "Warning, only Python Version 2 is supported."