Author: degenaro Date: Tue Jan 15 15:22:50 2019 New Revision: 1851378 URL: http://svn.apache.org/viewvc?rev=1851378&view=rev Log: UIMA-5958 DUCC resources distribution tool
Added: uima/uima-ducc/trunk/src/main/admin/ducc_rsync (with props) Modified: uima/uima-ducc/trunk/src/main/admin/stop_ducc Added: uima/uima-ducc/trunk/src/main/admin/ducc_rsync URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/ducc_rsync?rev=1851378&view=auto ============================================================================== --- uima/uima-ducc/trunk/src/main/admin/ducc_rsync (added) +++ uima/uima-ducc/trunk/src/main/admin/ducc_rsync Tue Jan 15 15:22:50 2019 @@ -0,0 +1,171 @@ +#!/usr/bin/env python +# ----------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ----------------------------------------------------------------------- + +import sys + +version_min = [2, 7] +version_info = sys.version_info +version_error = False +if(version_info[0] < version_min[0]): + version_error = True +elif(version_info[0] == version_min[0]): + if(version_info[1] < version_min[1]): + version_error = True +if(version_error): + print('Python minimum requirement is version '+str(version_min[0])+'.'+str(version_min[1])) + sys.exit(1) + +import argparse +import os + +from argparse import RawDescriptionHelpFormatter + +from ducc_util import * + +class DuccRsync(DuccUtil): + + merge = False + + key_ducc_head = 'ducc.head' + key_ducc_head_reliable_list = 'ducc.head.reliable.list' + + rsync_cmd = 'rsync' + rsync_flags = '-avz --delete --ignore-errors' + + def __init__(self): + DuccUtil.__init__(self, self.merge) + + def _fn(self): + fpath = __file__.split('/') + flen = len(fpath) + return fpath[flen-1] + + # extra help! + def get_epilog(self): + epilog = '' + epilog = epilog+'Purpose: synchronize DUCC directory(s) from present node to other head & agent nodes.' + epilog = epilog+'\n\n' + epilog = epilog+'Example:' + epilog = epilog+'\n\n' + epilog = epilog+'> '+self._fn()+' '+'--dirs "resources admin" --nodelist ducc.nodes' + return epilog + + default_dirs = 'resources' + default_nodelist = 'ducc.nodes' + + help_dirs = 'Blank separated list of directories to synchronize, default='+default_dirs+'.' + help_nodelist = 'Blank separated list of file names, default='+default_nodelist+'.' + help_debug = 'Display debugging messages.' + help_quiet = 'Suppress informational messages.' + + # parse command line + def get_args(self): + self.parser = argparse.ArgumentParser(formatter_class=RawDescriptionHelpFormatter,epilog=self.get_epilog()) + self.parser.add_argument('--dirs' , '-d', action='store', default=self.default_dirs, help=self.help_dirs) + self.parser.add_argument('--nodelist' , '-n', action='store', default=self.default_nodelist, help=self.help_nodelist) + self.parser.add_argument('--debug', '-g', action='store_true', help=self.help_debug) + self.parser.add_argument('--quiet', '-q', action='store_true', help=self.help_quiet) + self.args = self.parser.parse_args() + + # conditionally add node to list of nodes to be updated + def add(self,list,node): + if(node == self.localhost): + pass + elif(node in list): + pass + else: + list.append(node) + + # update the specified node, dir + def rsync(self,node,user,subdir): + dir = os.path.join(DUCC_HOME,subdir) + cmd = self.rsync_cmd+' '+self.rsync_flags+' '+dir+' '+user+'@'+node+':'+dir + if(self.args.debug): + print cmd + proc = subprocess.Popen(cmd, shell=True, bufsize=0, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + lines = [] + for line in proc.stdout: + lines.append(line.strip()) + proc.wait() + rc = proc.returncode + if(self.args.debug): + for line in lines: + print line + + # create list of node to be updated + def create_node_list(self): + node_list = [] + # head + head = self.ducc_properties.get(self.key_ducc_head) + if(self.args.debug): + print 'head:'+head + self.add(node_list,head) + #reliable + reliable = self.ducc_properties.get(self.key_ducc_head_reliable_list) + for node in reliable: + if(self.args.debug): + print 'reliable:'+head + self.add(node_list,node) + # agents + map = {} + if(self.args.debug): + print 'nodelist:'+str(self.args.nodelist) + nodefiles = self.args.nodelist.split() + if(self.args.debug): + print 'nodefiles:'+str(nodefiles) + for nodefile in nodefiles: + nodes, map = self.read_nodefile(nodefile,map) + if(self.args.debug): + print nodes, str(map) + for key in map: + value = map[key] + if(self.args.debug): + print 'key:'+key+' '+'value:'+str(value) + for node in value: + if(self.args.debug): + print 'node:'+node + self.add(node_list,node) + return node_list + + # do rsync of dir(s) from present node to peers + def main(self, argv): + try: + node_list = [] + user = find_ducc_uid() + self.get_args() + dir_list = self.args.dirs.split() + if(self.args.debug): + print 'dirs:'+str(dir_list) + node_list = self.create_node_list() + for node in node_list: + #print 'updating:'+node + for dir in dir_list: + if(self.args.quiet): + pass + else: + print 'updating:'+node+'->'+dir + self.rsync(node,user,dir) + except Exception,e: + print e + +if __name__ == "__main__": + + instance = DuccRsync() + instance.main(sys.argv[1:]) Propchange: uima/uima-ducc/trunk/src/main/admin/ducc_rsync ------------------------------------------------------------------------------ svn:executable = * Modified: uima/uima-ducc/trunk/src/main/admin/stop_ducc URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/stop_ducc?rev=1851378&r1=1851377&r2=1851378&view=diff ============================================================================== --- uima/uima-ducc/trunk/src/main/admin/stop_ducc (original) +++ uima/uima-ducc/trunk/src/main/admin/stop_ducc Tue Jan 15 15:22:50 2019 @@ -260,7 +260,7 @@ class StopDucc(DuccUtil): self.parser.add_argument(self.option_maxthreads, '-m', action='store', type=int, default=None, help=self.help_maxthreads) self.parser.add_argument(self.option_debug, '-d', action='store_true', help=self.help_debug) self.args = self.parser.parse_args() - # sepcial cases + # special cases if(self.args.kill): if(self.args.maxthreads == None): self.args.maxthreads = self.maxthreads