Author: challngr Date: Tue Sep 15 14:31:24 2015 New Revision: 1703203 URL: http://svn.apache.org/r1703203 Log: UIMA-4577 First delivery, DB project.
Added: uima/sandbox/uima-ducc/trunk/src/main/admin/database.py (with props) uima/sandbox/uima-ducc/trunk/src/main/admin/db.py (with props) uima/sandbox/uima-ducc/trunk/src/main/admin/db_console (with props) uima/sandbox/uima-ducc/trunk/src/main/resources/private/database.xml uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/NullStateServices.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/ uima/sandbox/uima-ducc/trunk/uima-ducc-database/pom.xml uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbConstants.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbObject.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbTester.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/ProcessToJobList.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/misc/ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/misc/DbQuery.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/misc/Server.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/misc/Shutdown.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/misc/SmLoader.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/resources/ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/test/ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/test/java/ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/test/resources/ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/NullHistoryManager.java Modified: uima/sandbox/uima-ducc/trunk/pom.xml uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/IDuccEnv.java uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/IStateServices.java uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServices.java uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesDirectory.java uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesFactory.java uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesSet.java uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/DuccProperties.java uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/test/java/org/apache/uima/ducc/common/test/cmd/StateServicesTest.java uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/runducc uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/start_sim uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/stop_sim uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/pom.xml uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCheckpoint.java uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCommonArea.java uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/utilities/Checkpointable.java uima/sandbox/uima-ducc/trunk/uima-ducc-parent/pom.xml uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java uima/sandbox/uima-ducc/trunk/uima-ducc-transport/pom.xml uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccPerWorkItemStatistics.java uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccWorkReservation.java uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccPerWorkItemStatistics.java uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryFactory.java uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryPersistenceManager.java uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/IHistoryPersistenceManager.java uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccData.java uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/IServicesRegistry.java uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/ServicesRegistry.java uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/ServicesRegistryMap.java uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/sort/ServicesSortCache.java Modified: uima/sandbox/uima-ducc/trunk/pom.xml URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/pom.xml?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/pom.xml (original) +++ uima/sandbox/uima-ducc/trunk/pom.xml Tue Sep 15 14:31:24 2015 @@ -121,6 +121,11 @@ </dependency> <dependency> <groupId>org.apache.uima</groupId> + <artifactId>uima-ducc-database</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.uima</groupId> <artifactId>uima-ducc-user</artifactId> <version>${project.version}</version> </dependency> @@ -175,6 +180,10 @@ </dependency> <dependency> <groupId>org.apache.uima</groupId> + <artifactId>uima-ducc-database</artifactId> + </dependency> + <dependency> + <groupId>org.apache.uima</groupId> <artifactId>uima-ducc-user</artifactId> </dependency> <dependency> @@ -182,28 +191,13 @@ <artifactId>uima-ducc-container</artifactId> </dependency> </dependencies> - - <build> - - - <!--resources> - <resource> - <directory>${baseidr}/src/main/resources</directory> - <includes> - <include>README</include> - <include>RELEASE_NOTES.html</include> - </includes> - <filtering>true</filtering> - <targetPath>${basedir}/target</targetPath> - </resource> - </resources--> - + <build> + <resources /> <plugins> - <!-- This java doc config is for building the ones distributed with the bin packaging, and also posted on our website. --> <plugin> @@ -466,9 +460,6 @@ </plugin --> </plugins> <!-- /pluginManagement --> - - - </build> <!-- Include all ducc modules --> @@ -485,6 +476,7 @@ <module>uima-ducc-duccdocs</module> <module>uima-ducc-web</module> <module>uima-ducc-examples</module> + <module>uima-ducc-database</module> <module>uima-ducc-spawn</module> <module>uima-ducc-user</module> <!-- Note: uima-ducc-container has a test dependency on uima-ducc-user so must be built after it --> Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc (original) +++ uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc Tue Sep 15 14:31:24 2015 @@ -51,6 +51,16 @@ class CheckDucc(DuccUtil): return + def verify_database(self): + if ( self.db_parms == self.db_disabled ): + return True + + ret = self.db_alive() + if ( ret ): + print 'The database is running' + else: + print 'The database is not running' + def verify_activemq(self): if ( self.is_amq_active() ): print 'ActiveMQ is found listening at', self.broker_protocol + "://" + self.broker_host + ':' + self.broker_port @@ -267,6 +277,8 @@ class CheckDucc(DuccUtil): if ( not self.verify_activemq() ): print 'ActiveMQ broker is not running on', self.broker_protocol + "://" + self.broker_host + ':' + self.broker_port + self.verify_database() + # init the PID file self.pids = Properties() self.pids.load_if_exists(self.pid_file) Added: uima/sandbox/uima-ducc/trunk/src/main/admin/database.py URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/database.py?rev=1703203&view=auto ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/admin/database.py (added) +++ uima/sandbox/uima-ducc/trunk/src/main/admin/database.py Tue Sep 15 14:31:24 2015 @@ -0,0 +1,91 @@ +#!/usr/bin/python + +# ----------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ----------------------------------------------------------------------- + +# This is common code to deal wiht the orientdb configuration xml + + +global domloaded +try: + from xml.dom.minidom import parse + domloaded = True +except ImportError: + domloaded = False + + +def db_dom(dbconfig): + return parse(dbconfig) + + +# <users> +# <user resources="*" password="asdfasdf" name="root"/> +# </users> + +def get_db_password(ducc_home, dbconfig): + dom = db_dom(ducc_home + '/' + dbconfig) + + lst = dom.getElementsByTagName('users') # returns a Nodelist object + for u in lst: + childs = u.childNodes + for c in childs: + if (c.nodeName == 'user'): + atts = c.attributes + name = atts.getNamedItem('name').nodeValue + pw = atts.getNamedItem('password').nodeValue + if ( name == 'root' ): + return pw + return None + +def update_head(dbconfig, head): + # update the listeners with the correct head, return the dom + dom = db_dom(dbconfig) + + lst = dom.getElementsByTagName('listener') + for l in lst: + atts = l.attributes + l.setAttribute('ip-address', head) + print 'Listener for protocol', atts.getNamedItem('protocol').value, 'set to', head + return dom + +def write_config(dbconfig, dom): + config = open(dbconfig, 'w') + config.write(dom.toxml()) + config.close() + +def configure(dbconfig, head, pw): + print 'Configure', dbconfig, 'at', head, 'pw', pw + + # parse the config, update the head node in the 'listeners' section + dom = update_head(dbconfig, head) + + # set the root pw + lst = dom.getElementsByTagName('users') # returns a Nodelist object + for u in lst: + childs = u.childNodes + for c in childs: + if (c.nodeName == 'user'): + atts = c.attributes + name = atts.getNamedItem('name').nodeValue + if ( name == 'root' ): + c.setAttribute('password', pw) + + # and rewrite the config + write_config(dbconfig, dom) + Propchange: uima/sandbox/uima-ducc/trunk/src/main/admin/database.py ------------------------------------------------------------------------------ svn:executable = * Added: uima/sandbox/uima-ducc/trunk/src/main/admin/db.py URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/db.py?rev=1703203&view=auto ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/admin/db.py (added) +++ uima/sandbox/uima-ducc/trunk/src/main/admin/db.py Tue Sep 15 14:31:24 2015 @@ -0,0 +1,192 @@ +#!/usr/bin/env python + +# ----------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ----------------------------------------------------------------------- + +import os +import sys +import getopt +import httplib +import base64 +import json +import textwrap + +from ducc_util import DuccUtil +from properties import Properties +import database as db + +class DbUtil(DuccUtil): + + def __init__(self): + DuccUtil.__init__(self) + + def pprint(self, j): + print json.dumps(j, sort_keys=True, indent=4, separators=(',',':')) + + + #"clientId":"-", + #"commandDetail":"-", + #"commandInfo":"Listening", + #"connectedOn":"2015-08-05 06:21:30", + #"connectionId":"12", + #"db":"DuccHistory", + #"driver":"OrientDB Java v2.1-rc6 Protocol v30", + #"lastCommandDetail":"SELECT * FROM VRESERVATION WHERE ducc_dbid=944", + #"lastCommandInfo":"Execute remote command", + #"lastCommandOn":"2015-08-10 12:36:11", + #"lastExecutionTime":"3", + #"protocol":"binary", + #"remoteAddress":"/192.168.2.196:48994", + #"totalRequests":"39171", + #"totalWorkingTime":"196975", + #"user":"admin" + + def format_connections(self, j): + keys = j[0].keys() + slen = {} + for k in keys: + maxv = len(k) + print 'initial', maxv + for c in j: + maxv = max(maxv, len(c[k])) + slen[k] = maxv + + print slen + format = '' + for k in keys: + format = format + '%' + str(slen[k]) + 's | ' + + + tmp = j[0] + self.pprint(j[0]) + print format + print tmp + print len(tmp) + print format % (tuple(tmp.keys())) + print format % (tuple(tmp.values())) + return + + i = 1 + for c in j: # connectins in json + print 'Connection', i + i = i + 1 + self.pprint(c) + + maxl = 0 + for k in c.keys(): + maxl = max(maxl, len(k)) + print 'max:', maxl + + + def show_connections(self): + print 'Show connections' + + pw = base64.b64encode('root:asdfasdf') + + try: + conn = httplib.HTTPConnection(self.dbrest) + conn.request('GET', '/server', None, {'Authorization':'Basic ' + pw}) + except Exception, (e): + print " Checking connection: ", e + return False + + + resp = conn.getresponse() + #print 'response code', resp.status, resp.reason + data = resp.read() + #print 'Data:', data + + if ( resp.status == 200 ): + #print data + conns = json.loads(data) + #print json.dumps(conns, sort_keys=True, indent=4, separators=(',',':')) + + #self.pprint(conns['connections']) + #print conns['dbs'] + #print conns['properties'] + #print conns['storages'] + print 'Connections:', len(conns['connections']) + self.format_connections(conns['connections']) + + + # + # Open an OrientDb console on the database defined for this DUCC instance + # + def open_console(self): + + main = 'com.orientechnologies.orient.graph.console.OGremlinConsole' + + jp = '' + for k in self.jvm_parms.keys(): + v = self.jvm_parms[k] + if ( v == None ): + jp = jp + k + ' ' + else: + jp = jp + k + '=' + v + ' ' + + cmd = ' '.join([self.java(), jp, '-cp', self.classpath, main, ]) + + here = os.getcwd() + os.chdir(self.db_rt) + pid = self.spawn(cmd) + + def usage(self, *msg): + if ( msg[0] != None ): + print ' '.join(msg) + + print 'Usage:' + print ' db.py [options]' + print '' + print 'Where options include:' + print ' -c show connection status' + print '' + print 'If no options are given a console to the database is opened.' + sys.exit(0) + + + def main(self, argv): + + # The dom gets loaded by the import above - or not + if ( not db.domloaded ): + print "Unable to read database configuration; insure the installed Python supports xml.dom.minidom" + print "Note that Python must be at least version 2.6 but not 3.x. You are running version", sys.version_info + return + + (self.jvm_parms, self.classpath, self.db_rt, self.dburl, self.dbrest, self.dbroot) = self.db_parms + try: + opts, args = getopt.getopt(argv, 'ch?', ['--connections', '--help']) + except: + self.usage('Bad arguments ' + ' '.join(argv)) + + + if ( len(opts) == 0 ): + self.open_console() + return + + for ( o, a ) in opts: + print 'o', o, 'a', a + if ( o in ['-c', '--connections']): + self.show_connections() + elif ( o in ['-h', '-?', '--help'] ): + self.usage() + + +if __name__ == "__main__": + ducc = DbUtil() + ducc.main(sys.argv[1:]) Propchange: uima/sandbox/uima-ducc/trunk/src/main/admin/db.py ------------------------------------------------------------------------------ svn:executable = * Added: uima/sandbox/uima-ducc/trunk/src/main/admin/db_console URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/db_console?rev=1703203&view=auto ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/admin/db_console (added) +++ uima/sandbox/uima-ducc/trunk/src/main/admin/db_console Tue Sep 15 14:31:24 2015 @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# ----------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ----------------------------------------------------------------------- + + +import os +import sys + +from ducc_util import DuccUtil +from properties import Properties +import database as db + +class DbConsole(DuccUtil): + + def __init__(self): + DuccUtil.__init__(self) + + def main(self, rgv): + # The dom gets loaded by the import above - or not + if ( not db.domloaded ): + print "Unable to read database configuration; insure the installed Python supports xml.dom.minidom" + print "Note that Python must be at least version 2.6 but not 3.x. You are running version", sys.version_info + return + + + (jvm_parms, classpath, db_rt, dburl, dbrest, dbroot) = self.db_parms() + + + main = 'com.orientechnologies.orient.graph.console.OGremlinConsole' + + jp = '' + for k in jvm_parms.keys(): + v = jvm_parms[k] + if ( v == None ): + jp = jp + k + ' ' + else: + jp = jp + k + '=' + v + ' ' + + cmd = ' '.join([self.java(), jp, '-cp', classpath, main, ]) + + here = os.getcwd() + os.chdir(db_rt) + pid = self.spawn(cmd) + + +if __name__ == "__main__": + console = DbConsole() + console.main(sys.argv[1:]) + + Propchange: uima/sandbox/uima-ducc/trunk/src/main/admin/db_console ------------------------------------------------------------------------------ svn:executable = * Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py (original) +++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py Tue Sep 15 14:31:24 2015 @@ -27,9 +27,55 @@ import time from ducc_util import DuccUtil from local_hooks import verify_slave_node +import database as db + +from properties import Properties +from properties import Property class Ducc(DuccUtil): + + def run_db(self, background): + # The dom gets loaded by the import above - or not + if ( not db.domloaded ): + print "Unable to read database configuration; insure the installed Python supports xml.dom.minidom" + print "Note that Python must be at least version 2.6 but not 3.x. You are running version", sys.version_info + return + + print '(re)Starting the database' + + # first write this out so it's easy to find in java without having to parse xml + (jvm_parms, classpath, db_rt, dburl, dbrest, dbroot) = self.db_parms + db_pw_file = Properties() + comment = [] + comment.append('# Do not modify this file, it is auto-generated.') + comment.append('# This is extracted from the database configuration and written to this file on every boot of the db') + p = Property('db_password', dbroot, comment) + db_pw_file.put_property(p) + db_pw_filename = self.DUCC_HOME + '/resources.private/db_password' + db_pw_file.write(db_pw_filename) + os.chmod(db_pw_filename, 0700) + + main = 'com.orientechnologies.orient.server.OServerMain' + + jp = '' + for k in jvm_parms.keys(): + v = jvm_parms[k] + if ( v == None ): + jp = jp + k + ' ' + else: + jp = jp + k + '=' + v + ' ' + + cmd = ' '.join(['nohup', self.java(), jp, '-cp', classpath, main, '&']) + print cmd + + here = os.getcwd() + os.chdir(db_rt) + pid = self.spawn(cmd) + os.chdir(here) + + print 'OK' + def run_broker(self, component): broker_port = self.ducc_properties.get('ducc.broker.port') broker_jmx_port = self.ducc_properties.get('ducc.broker.jmx.port') @@ -149,6 +195,7 @@ class Ducc(DuccUtil): self.add_to_classpath(ducc_home + '/lib/http-client/*') self.add_to_classpath(ducc_home + '/webserver/lib/*') self.add_to_classpath(ducc_home + '/webserver/lib/jsp/*') + self.add_to_classpath(ducc_home + '/lib/orientdb/*') if ( c == 'orchestrator' ): if ( or_parms != None ): @@ -157,6 +204,7 @@ class Ducc(DuccUtil): jvm_opts.append(self.or_jvm_args) self.add_to_classpath(ducc_home + '/lib/http-client/*') self.add_to_classpath(ducc_home + '/webserver/lib/*') + self.add_to_classpath(ducc_home + '/lib/orientdb/*') if ( c == 'pm' ): if ( self.pm_jvm_args != None ): @@ -168,6 +216,7 @@ class Ducc(DuccUtil): self.add_to_classpath(ducc_home + '/apache-uima/apache-activemq/lib/optional/*') self.add_to_classpath(ducc_home + '/lib/http-client/*') self.add_to_classpath(ducc_home + '/webserver/lib/*') + self.add_to_classpath(ducc_home + '/lib/orientdb/*') if (component != 'agent'): service = 'org.apache.uima.ducc.common.main.DuccService' @@ -290,10 +339,17 @@ class Ducc(DuccUtil): if ( component == None ): self.usage("Must specify component") + if ( component == 'db' ): + self.run_db(background) + return + if ( component == 'broker' ): self.run_broker(background) - else: - self.run_component(component, or_parms, numagents, rmoverride, background, nodup, localdate) + return + + # fall-through, runs one of the ducc components proper + self.run_component(component, or_parms, numagents, rmoverride, background, nodup, localdate) + return def __call__(self, *args): Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install (original) +++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install Tue Sep 15 14:31:24 2015 @@ -37,6 +37,8 @@ from ducc_base import which from ducc import Ducc +import database as db + class PostInstall(): def usage(self, msg): @@ -78,6 +80,24 @@ class PostInstall(): print 'WARNING' print '' + def configure_database(self): + self.ducc_default_properties = Properties() + self.ducc_default_properties.load(self.DUCC_HOME + '/resources/default.ducc.properties') + self.ducc_default_properties.put('ducc.head', self.ducc_head) + + dburl = self.ducc_default_properties.get('ducc.database.url') + dbconfig = self.DUCC_HOME + '/' + self.ducc_default_properties.get('ducc.database.config.file') + print 'Configuring database at', dburl, 'from', dbconfig + print '' + + dbpw = '' + while ( dbpw == '' ): + dbpw = raw_input('Enter password for database root:') + + print '' + db.configure(dbconfig, self.ducc_head, dbpw) + + def get_keystore_pw(self, prompt): if ( self.keystore_pw == None ): @@ -354,6 +374,8 @@ class PostInstall(): self.setup_ducc_head() self.check_nodes() + self.configure_database() + # As of DUCC 2.0, always set here on installation print 'ActiveMQ is automanaged on node ', self.localhost Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py (original) +++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py Tue Sep 15 14:31:24 2015 @@ -28,11 +28,14 @@ import grp import resource import time import platform +import httplib from threading import * import traceback import Queue +import database as db + from stat import * from local_hooks import find_other_processes @@ -158,7 +161,190 @@ class DuccUtil(DuccBase): CMD = ' '.join(CMD) print 'Merging', base_props, 'with', site_props, 'into', run_props os.system(CMD) - + + def db_read_parms(self): + # + # common setup for orientdb management + # + + dbhost = self.ducc_properties.get('ducc.database.host') + if ( dbhost == self.db_disabled ): + self.db_parms = self.db_disabled + return self.db_parms + + dburl = self.ducc_properties.get('ducc.database.url') + dbrest = self.ducc_properties.get('ducc.database.rest.url') + dbconfig = self.ducc_properties.get('ducc.database.config.file') + dbroot = db.get_db_password(DUCC_HOME, dbconfig) + if ( dbroot == None ): + print 'Cannot find database passord in', dbconfig + sys.exit(1) + + rt = self.DUCC_HOME # (ducc runtime) + db_rt = rt + '/database' # ORIENTDB_HOME - the database "home" place + + jvm_parms = { + '-Dfile.encoding' : 'UTF8', + '-Drhino.opt.level' : '9', + '-Dprofile.enabled' : 'true', + '-Djna.nosys' : 'true', + '-Djava.awt.headless' : 'true', + '-Dorientdb.config.file' : rt + '/' + dbconfig, + '-Dorientdb.www.path' : db_rt + '/www', + '-Djava.util.logging.config.file' : rt + '/resources/database.log.config', + } + classpath = '"' + rt + '/lib/orientdb/*' + classpath = classpath + ':' + rt + '/lib/jna/*' + '"' + + os.environ['DUCC_HOME'] = self.DUCC_HOME + os.environ['ORIENTDB_HOME'] = db_rt + + self.db_parms = (jvm_parms, classpath, db_rt, dburl, dbrest, dbroot) + + def db_alive(self): + if ( self.db_parms == self.db_disabled ): + return True + + (jvm_parms, classpath, db_rt, dburl, dbrest, dbroot) = self.db_parms + + try: + conn = httplib.HTTPConnection(dbrest) + conn.request('GET', '/listDatabases') + except Exception, (e): + print " Checking connection: ", e + return False + + resp = conn.getresponse() + #print 'response code', resp.status, resp.reason + data = resp.read() + #print 'Data:', data + + if ( resp.status == 200 ): + # it will be simple json that Python will see as lists and maps so we can just eval it + data = eval(data) + dblist = data['databases'] + if ( len(dblist) == 0 ): + print ' Connection succeeded, no databases found.' + pass + else: + print 'Found these databases:' + for d in data['databases']: + print ' ', d + return True + + def db_start(self): + + # bypass all of this for the initial delivery + if ( self.db_parms == self.db_disabled ): + return True + + print 'Starting database' + + self.sync_db_config() # insure db is running on head node, in case the head changed in ducc.properties + node = self.ducc_properties.get('ducc.head') + + if ( node == 'local' ): + node = self.localhost + + if ( self.db_alive() ): + print 'Database is already started.' + return True + + max_attempts = 5 + attempt = 0 + while attempt < max_attempts: + lines = self.ssh(node, True, "'", self.DUCC_HOME + '/admin/ducc.py', '-c', 'db', '--nodup', "'") + # we'll capture anything that the python shell spews because it may be useful, and then drop the + # pipe when we see a PID message + while True: + try: + line = lines.readline().strip() + except: + break + #print '[]', line + + if ( not line ): + break + if ( line == '' ): + break + if ( line == 'OK' ): + print 'GOT OK from db start' + lines.close(); + + print 'waiting for database to start' + time.sleep(5) + if ( self.db_alive() ): + return True + + attempt = attempt + 1 + print 'Did not connect to database, retrying (', attempt, 'of', max_attempts, ')' + + return False + + def db_init(self): + + # bypass all of this for the initial delivery + if ( self.db_parms == self.db_disabled ): + return True + + print 'Initializing database.' + main = 'org.apache.uima.ducc.database.DbCreate' + (jvm_parms, classpath, db_rt, dburl, dbrest, dbroot) = self.db_parms + + classpath = self.DUCC_HOME + '/lib/uima-ducc/*:' + classpath + dburl = self.ducc_properties.get('ducc.state.database.url') + cmd = ' '.join([self.java(), '-DDUCC_HOME=/home/challngr/ducc_runtime_db', '-cp', classpath, main, dburl]) + print cmd + self.spawn(cmd) + + def db_stop(self): + # bypass all of this for the initial delivery + if ( self.db_parms == self.db_disabled ): + return True + + + cfgfile = 'ducc.database.config.file' + dbconfig = self.ducc_properties.get(cfgfile) + if ( dbconfig == None ): + print 'Database is not configured:', cfgfile, 'is not found in ducc.properties' + return + + rt = self.DUCC_HOME # (ducc runtime) + db_rt = rt + '/database' # ORIENTDB_HOME - the database "home" place + + jvm_parms = { + '-Djava.awt.headless' : 'true', + '-Dorientdb.config.file' : rt + '/' + dbconfig, + } + classpath = '"' + rt + '/lib/orientdb/*' + '"' + + main = 'com.orientechnologies.orient.server.OServerShutdownMain' + + jp = '' + for k in jvm_parms.keys(): + v = jvm_parms[k] + if ( v == None ): + jp = jp + k + ' ' + else: + jp = jp + k + '=' + v + ' ' + + + os.environ['ORIENTDB_HOME'] = db_rt + cmd = ' '.join([self.java(), jp, '-cp', classpath, main]) + print cmd + + here = os.getcwd() + os.chdir(db_rt) + self.spawn(cmd) + os.chdir(here) + + def sync_db_config(self): + dbconfig = self.DUCC_HOME + '/' + self.ducc_properties.get('ducc.database.config.file') + head = self.ducc_properties.get('ducc.head') + print 'Sync database node with head node on', head + dom = db.update_head(dbconfig, head) + db.write_config(dbconfig, dom) + def find_netstat(self): # don't you wish people would get together on where stuff lives? if ( os.path.exists('/sbin/netstat') ): @@ -765,12 +951,13 @@ class DuccUtil(DuccBase): global use_threading DuccBase.__init__(self, merge) + self.db_disabled = '--disabled--' self.duccling = None self.broker_url = 'tcp://localhost:61616' self.broker_protocol = 'tcp' self.broker_host = 'localhost' self.broker_port = '61616' - self.default_components = ['rm', 'pm', 'sm', 'or', 'ws', 'broker'] + self.default_components = ['rm', 'pm', 'sm', 'or', 'ws', 'db', 'broker'] self.default_nodefiles = [self.DUCC_HOME + '/resources/ducc.nodes'] if ( self.localhost == self.ducc_properties.get("ducc.head")): @@ -783,6 +970,9 @@ class DuccUtil(DuccBase): self.os_pagesize = self.get_os_pagesize() self.update_properties() + self.db_read_parms() + + manage_broker = self.ducc_properties.get('ducc.broker.automanage') self.automanage = False if (manage_broker in ('t', 'true', 'T', 'True')) : Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc (original) +++ uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc Tue Sep 15 14:31:24 2015 @@ -311,6 +311,15 @@ class StartDucc(DuccUtil): print sys.exc_info()[0], "DUCC may not be started correctly." sys.exit(1) + try: + if ( self.db_start() ): + self.db_init() + + except Exception (e): + # print e + print sys.exc_info()[0], "Can't start the database." + sys.exit(1) + if ( self.is_amq_active() ): print 'ActiveMQ broker is found on configured host and port:', self.broker_host + ':' + self.broker_port else: @@ -336,7 +345,7 @@ class StartDucc(DuccUtil): print 'Starting', or_parms for com in components: - if ( com == 'broker' ): + if ( com in ('broker', 'db') ): pass # already started else: try: Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc (original) +++ uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc Tue Sep 15 14:31:24 2015 @@ -38,6 +38,10 @@ class StopDucc(DuccUtil): self.stop_broker() return + if ( component == 'db' ): + self.db_stop() + return + # # If it's an unqualified management component, we need to get it's qualified name # @@ -78,6 +82,7 @@ class StopDucc(DuccUtil): self.pids.delete(com) self.pids.delete(component) + def quiesce_agents(self, components, nodes): allnodes = [] for ( nf, nl ) in nodes.items(): @@ -134,6 +139,9 @@ class StopDucc(DuccUtil): print ' ws - web server' print ' agent - node agent' print '' + print ' -w --wait' + print ' Time to wait for everything to come down, in seconds. Default is 60.' + print '' print ' -k --kill' print ' Stop the component forcibly and immediately using kill -9. Use this only if a' print ' normal stop does not work (e.g. the process may be hung).' @@ -167,9 +175,10 @@ class StopDucc(DuccUtil): force = False quiesce = False all = False + wait_time = 60 try: - opts, args = getopt.getopt(argv, 'ac:n:kn:qh?v', ['all', 'component=', 'components=', 'help', 'nodelist=', 'kill', 'quiesce', 'nothreading']) + opts, args = getopt.getopt(argv, 'ac:n:kn:w:qh?v', ['all', 'component=', 'components=', 'help', 'nodelist=', 'kill', 'quiesce', 'nothreading', 'wait']) except: self.invalid('Invalid arguments ' + ' '.join(argv)) @@ -187,6 +196,8 @@ class StopDucc(DuccUtil): force = True elif o in ( '-q', '--quiesce' ): quiesce = True + elif o in ( '-w', '--wait' ): + wait_time = int(a) elif o in ( '--nothreading' ): self.disable_threading() elif ( o == '-v' ) : @@ -222,14 +233,15 @@ class StopDucc(DuccUtil): self.clean_shutdown() # Agents may wait up to 60 secs for processes to quiesce - sleeptime = 60 - print "Waiting " + str(sleeptime) + " seconds to broadcast agent shutdown." - time.sleep(sleeptime) + print "Waiting " + str(wait_time) + " seconds to broadcast agent shutdown." + time.sleep(wait_time) if ( self.automanage ): print "Stopping broker" self.stop_broker() + self.db_stop() + if ( os.path.exists(self.pid_file) ): os.remove(self.pid_file) return @@ -237,17 +249,27 @@ class StopDucc(DuccUtil): if ( len(nodefiles) == 0 ): nodefiles = self.default_nodefiles + self.pids = Properties() - try: - self.pids.load(self.pid_file) - pass - except PropertiesException, (inst): - print inst.msg - print '' - print 'Run check_ducc to refresh the PIDs file, or check_ducc -k to search for and', - print 'kill all DUCC processes.' - print '' - sys.exit(1) + sc = set(components) + sb = set(['broker', 'db']) + read_pids = True + if ( sc.issubset(sb) ): + read_pids = False + + + # The broker and db do not set the pid file + if ( read_pids ): + try: + self.pids.load(self.pid_file) + pass + except PropertiesException, (inst): + print inst.msg + print '' + print 'Run check_ducc to refresh the PIDs file, or check_ducc -k to search for and', + print 'kill all DUCC processes.' + print '' + sys.exit(1) # # if not 'all', we use nodefiles and component names @@ -278,10 +300,11 @@ class StopDucc(DuccUtil): for n in nl: self.stop_agents(n, force) - if ( len(self.pids) > 0 ): - self.pids.write(self.pid_file) - else: - os.remove(self.pid_file) + if ( read_pids ): + if ( len(self.pids) > 0 ): + self.pids.write(self.pid_file) + else: + os.remove(self.pid_file) return Modified: uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml (original) +++ uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml Tue Sep 15 14:31:24 2015 @@ -69,6 +69,7 @@ under the License. <include>org.apache.uima:uima-ducc-orchestrator</include> <include>org.apache.uima:uima-ducc-web</include> <include>org.apache.uima:uima-ducc-container</include> + <include>org.apache.uima:uima-ducc-database</include> </includes> <outputFileNameMapping>${artifact.artifactId}-${project.version}.jar</outputFileNameMapping> <outputDirectory>lib/uima-ducc</outputDirectory> @@ -129,6 +130,80 @@ under the License. <directoryMode>755</directoryMode> </dependencySet> + <!-- orientdb --> + <dependencySet> + <useStrictFiltering>true</useStrictFiltering> + <includes> + <include>com.tinkerpop.gremlin:gremlin-groovy</include> + </includes> + <outputDirectory>lib/orientdb</outputDirectory> + <useProjectArtifact>false</useProjectArtifact> + <fileMode>644</fileMode> + <directoryMode>755</directoryMode> + </dependencySet> + + <dependencySet> + <useStrictFiltering>true</useStrictFiltering> + <includes> + <include>com.tinkerpop.blueprints:blueprints-core</include> + </includes> + <outputDirectory>lib/orientdb</outputDirectory> + <useProjectArtifact>false</useProjectArtifact> + <fileMode>644</fileMode> + <directoryMode>755</directoryMode> + </dependencySet> + + <dependencySet> + <useStrictFiltering>true</useStrictFiltering> + <includes> + <include>com.googlecode.concurrentlinkedhashmap:concurrentlinkedhashmap-lru</include> + </includes> + <outputDirectory>lib/orientdb</outputDirectory> + <useProjectArtifact>false</useProjectArtifact> + <fileMode>644</fileMode> + <directoryMode>755</directoryMode> + </dependencySet> + + <dependencySet> + <useStrictFiltering>true</useStrictFiltering> + <includes> + <include>com.orientechnologies:orientdb-graphdb</include> + <include>com.orientechnologies:orientdb-core</include> + <include>com.orientechnologies:orientdb-client</include> + <include>com.orientechnologies:orientdb-server</include> + <include>com.orientechnologies:orientdb-tools</include> + <include>com.orientechnologies:orientdb-enterprise</include> + </includes> + <outputDirectory>lib/orientdb</outputDirectory> + <useProjectArtifact>false</useProjectArtifact> + <fileMode>644</fileMode> + <directoryMode>755</directoryMode> + </dependencySet> + + <dependencySet> + <useStrictFiltering>true</useStrictFiltering> + <includes> + <include>com.tinkerpop.gremlin:gremlin-groovy</include> + </includes> + <outputDirectory>lib/orientdb</outputDirectory> + <useProjectArtifact>false</useProjectArtifact> + <fileMode>644</fileMode> + <directoryMode>755</directoryMode> + </dependencySet> + + <!-- dependencySet> + <useStrictFiltering>true</useStrictFiltering> + <includes> + <include>com.orientechnologies:orientdb-lucene</include> + </includes> + <outputDirectory>database/plugins</outputDirectory> + <useProjectArtifact>false</useProjectArtifact> + <fileMode>644</fileMode> + <directoryMode>755</directoryMode> + </dependencySet --> + + <!-- end orientdb --> + <!-- xmlbeans is in UIMA-AS 2.6.0 package --> <dependencySet> Modified: uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties (original) +++ uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties Tue Sep 15 14:31:24 2015 @@ -502,4 +502,35 @@ ducc.process.thread.sleep.time=3000 ducc.flow-controller.specifier=org.apache.uima.ducc.FlowController +# database things ----------------------------------------------------------------------------------- +# Directory where the databases and associated stuff is kept, relative to DUCC_HOME. This +# is ORIENTDB_HOME +ducc.database.location = database + +# DB defined config file, relative to DUCC_HOME. Must be in private because it has root pw in plaintext. +ducc.database.config.file = resources.private/database.xml + +ducc.database.host = --disabled-- + +# The binary url not HTTP/REST +ducc.database.url = remote:${ducc.database.host}:2424 +# The rest interface +ducc.database.rest.url = ${ducc.database.host}:2480 + +# Name of the database with active state. Svc registry, etc. +ducc.state.database.url = ${ducc.database.url}/DuccState + +# Name of the database with history +ducc.history.database.url = ${ducc.database.url}/DuccHistory + +# This class implementes the persistence interface for the services registry +#ducc.service.persistence.impl = org.apache.uima.ducc.database.StateServicesDb +ducc.service.persistence.impl = org.apache.uima.ducc.common.persistence.services.StateServices + +# This class implements the persistence interface for Job, Reservation, Service, and AP instances, plus +# the Orchestrator checkpoint. +#ducc.job.history.impl = org.apache.uima.ducc.database.HistoryManagerDb +ducc.job.history.impl = org.apache.uima.ducc.transport.event.common.history.HistoryPersistenceManager + +# end database things ------------------------------------------------------------------------------- Added: uima/sandbox/uima-ducc/trunk/src/main/resources/private/database.xml URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/resources/private/database.xml?rev=1703203&view=auto ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/resources/private/database.xml (added) +++ uima/sandbox/uima-ducc/trunk/src/main/resources/private/database.xml Tue Sep 15 14:31:24 2015 @@ -0,0 +1,126 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<orient-server> + <handlers> + <handler class="com.orientechnologies.orient.graph.handler.OGraphServerHandler"> + <parameters> + <parameter value="true" name="enabled"/> + <parameter value="50" name="graph.pool.max"/> + </parameters> + </handler> + + <!-- This sets up the redundant distributed server setup --> + <!-- Configured out for now, jrc 2015-06-18 --> + <!-- + <handler class="com.orientechnologies.orient.server.hazelcast.OHazelcastPlugin"> + <parameters> + <parameter value="${distributed}" name="enabled"/> + <parameter value="${ORIENTDB_HOME}/config/default-distributed-db-config.json" name="configuration.db.default"/> + <parameter value="${ORIENTDB_HOME}/config/hazelcast.xml" name="configuration.hazelcast"/> + </parameters> + </handler> + --> + + <!-- JMX Support. --> + <!-- Configured out for now. jrc 2-15-06-18 --> + <!-- + <handler class="com.orientechnologies.orient.server.handler.OJMXPlugin"> + <parameters> + <parameter value="true" name="enabled"/> + <parameter value="true" name="profilerManaged"/> + </parameters> + </handler> + --> + + <handler class="com.orientechnologies.orient.server.handler.OAutomaticBackup"> + <parameters> + <parameter value="true" name="enabled"/> + <parameter value="4h" name="delay"/> + <parameter value="backup" name="target.directory"/> + <parameter value="${DBNAME}-${DATE:yyyyMMddHHmmss}.zip" name="target.fileName"/> + <parameter value="9" name="compressionLevel"/> + <parameter value="1048576" name="bufferSize"/> + <parameter value="" name="db.include"/> + <parameter value="" name="db.exclude"/> + </parameters> + </handler> + + <!-- Server side scripting, dangerous and unnecessary at this point. --> + <!-- Disabled by jrc 2015-06-28 --> + <!-- + <handler class="com.orientechnologies.orient.server.handler.OServerSideScriptInterpreter"> + <parameters> + <parameter value="true" name="enabled"/> + <parameter value="SQL" name="allowedLanguages"/> + </parameters> + </handler> + --> + <handler class="com.orientechnologies.orient.server.token.OrientTokenHandler"> + <parameters> + <parameter value="false" name="enabled"/> + <parameter value="" name="oAuth2Key"/> + <parameter value="60" name="sessionLength"/> + <parameter value="HmacSHA256" name="encryptionAlgorithm"/> + </parameters> + </handler> + </handlers> + <network> + <sockets> + <socket implementation="com.orientechnologies.orient.server.network.OServerSSLSocketFactory" name="ssl"> + <parameters> + <parameter value="false" name="network.ssl.clientAuth"/> + <parameter value="config/cert/orientdb.ks" name="network.ssl.keyStore"/> + <parameter value="password" name="network.ssl.keyStorePassword"/> + <parameter value="config/cert/orientdb.ks" name="network.ssl.trustStore"/> + <parameter value="password" name="network.ssl.trustStorePassword"/> + </parameters> + </socket> + <socket implementation="com.orientechnologies.orient.server.network.OServerSSLSocketFactory" name="https"> + <parameters> + <parameter value="false" name="network.ssl.clientAuth"/> + <parameter value="config/cert/orientdb.ks" name="network.ssl.keyStore"/> + <parameter value="password" name="network.ssl.keyStorePassword"/> + <parameter value="config/cert/orientdb.ks" name="network.ssl.trustStore"/> + <parameter value="password" name="network.ssl.trustStorePassword"/> + </parameters> + </socket> + </sockets> + <protocols> + <protocol implementation="com.orientechnologies.orient.server.network.protocol.binary.ONetworkProtocolBinary" name="binary"/> + <protocol implementation="com.orientechnologies.orient.server.network.protocol.http.ONetworkProtocolHttpDb" name="http"/> + </protocols> + <listeners> + <!-- Original has a set of listeners, we want to restrict to just one in DUCC --> + <!-- + <listener protocol="binary" socket="default" port-range="2424-2430" ip-address="0.0.0.0"/> + <listener protocol="http" socket="default" port-range="2480-2490" ip-address="0.0.0.0"> + --> + <listener protocol="binary" socket="default" port-range="2424" ip-address="replaceme" /> + <listener protocol="http" socket="default" port-range="2480" ip-address="replaceme" > + <commands> + <command implementation="com.orientechnologies.orient.server.network.protocol.http.command.get.OServerCommandGetStaticContent" pattern="GET|www GET|studio/ GET| GET|*.htm GET|*.html GET|*.xml GET|*.jpeg GET|*.jpg GET|*.png GET|*.gif GET|*.js GET|*.css GET|*.swf GET|*.ico GET|*.txt GET|*.otf GET|*.pjs GET|*.svg GET|*.json GET|*.woff GET|*.ttf GET|*.svgz" stateful="false"> + <parameters> + <entry value="Cache-Control: no-cache, no-store, max-age=0, must-revalidate\r\nPragma: no-cache" name="http.cache:*.htm *.html"/> + <entry value="Cache-Control: max-age=120" name="http.cache:default"/> + </parameters> + </command> + <command implementation="com.orientechnologies.orient.graph.server.command.OServerCommandGetGephi" pattern="GET|gephi/*" stateful="false"/> + </commands> + <parameters> + <parameter value="utf-8" name="network.http.charset"/> + </parameters> + </listener> + </listeners> + </network> + <storages/> + <users> + <user resources="*" password="replace" name="root"/> + <user resources="connect,server.listDatabases" password="guest" name="guest"/> + </users> + <properties> + <entry value="1" name="db.pool.min"/> + <entry value="50" name="db.pool.max"/> + <entry value="false" name="profiler.enabled"/> <!-- was true, made false, jrc 2015-06-18 --> + <entry value="info" name="log.console.level"/> + <entry value="info" name="log.file.level"/> + </properties> +</orient-server> Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/IDuccEnv.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/IDuccEnv.java?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/IDuccEnv.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/IDuccEnv.java Tue Sep 15 14:31:24 2015 @@ -41,7 +41,8 @@ public interface IDuccEnv { public static final String DUCC_HISTORY_RESERVATIONS_DIR = DUCC_HOME_DIR+"history"+File.separator+"reservations"+File.separator; public static final String DUCC_HISTORY_SERVICES_DIR = DUCC_HOME_DIR+"history"+File.separator+"services"+File.separator; - public static final String DUCC_STATE_SERVICES_DIR = DUCC_HOME_DIR+"state"+File.separator+"services"+File.separator; + public static final String DUCC_STATE_SVCREG_DIR = DUCC_HOME_DIR+"state"+File.separator+"services"+File.separator; + public static final String DUCC_HISTORY_SVCREG_DIR = DUCC_HOME_DIR+"history"+File.separator+"services-registry"+File.separator; public static final String DUCC_LOGS_DIR = DUCC_HOME_DIR+"logs"+File.separator; public static final String DUCC_LOGS_WEBSERVER_DIR = DUCC_HOME_DIR+"logs"+File.separator+"webserver"+File.separator; Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/IStateServices.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/IStateServices.java?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/IStateServices.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/IStateServices.java Tue Sep 15 14:31:24 2015 @@ -18,51 +18,128 @@ */ package org.apache.uima.ducc.common.persistence.services; -import java.io.IOException; -import java.util.ArrayList; +import java.util.List; +import java.util.Properties; import org.apache.uima.ducc.common.IDuccEnv; +import org.apache.uima.ducc.common.utils.DuccLogger; +import org.apache.uima.ducc.common.utils.id.DuccId; public interface IStateServices { - public static String directory_state_services = IDuccEnv.DUCC_STATE_SERVICES_DIR; + public static String svc_reg_dir = IDuccEnv.DUCC_STATE_SVCREG_DIR; + public static String svc_hist_dir = IDuccEnv.DUCC_HISTORY_SVCREG_DIR; public static final String svc = "svc"; public static final String meta = "meta"; + + public enum SvcProps { + endpoint { public String pname() { return "endpoint"; } }, + instances { public String pname() { return "instances"; } }, + autostart { public String pname() { return "autostart"; } }, + reference { public String pname() { return "reference"; } }, + enabled { public String pname() { return "enabled"; } }, + + disable_reason { public String pname() { return "disable_reason"; } }, + + implementors { public String pname() { return "implementors"; } }, + numeric_id { public String pname() { return "numeric_id"; } }, + ping_active { public String pname() { return "ping_active"; } }, + ping_only { public String pname() { return "ping_only"; } }, + + service_alive { public String pname() { return "service_alive"; } }, + service_class { public String pname() { return "service_class"; } }, + service_dependency { public String pname() { return "service_dependency"; } }, + service_healthy { public String pname() { return "service_healthy"; } }, + service_state { public String pname() { return "service_state"; } }, + last_use { public String pname() { return "last_use"; } }, + last_use_readable { public String pname() { return "last_use_readable"; } }, + service_statistics { public String pname() { return "service_statistics"; } }, + service_type { public String pname() { return "service_type"; } }, + submit_error { public String pname() { return "submit_error"; } }, + user { public String pname() { return "user"; } }, + + references { public String pname() { return "references"; } }, + stopped { public String pname() { return "stopped"; } }, + + scheduling_class { public String pname() { return "scheduling_class"; } }, + process_memory_size { public String pname() { return "process_memory_size"; } }, + description { public String pname() { return "description"; } }, + log_directory { public String pname() { return "log_directory"; } }, + process_executable { public String pname() { return "process_executable"; } }, + + last_ping { public String pname() { return "last_ping"; } }, + last_ping_readable { public String pname() { return "last_ping_readable"; } }, + last_runnable { public String pname() { return "last_runnable"; } }, + last_runnable_readable { public String pname() { return "last_runnable_readable"; } }, + work_instances { public String pname() { return "work_instances"; } }, + registration_date { public String pname() { return "registration_date"; } }, + + instance_init_failures_limit { public String pname() { return "instance_init_failures_limit"; } }, + + UIMA_AS { public String pname() { return "UIMA-AS"; } }, + CUSTOM { public String pname() { return "CUSTOM"; } }, + + ; + public abstract String pname(); + + }; + + //public static final String endpoint { public String pname() { return "endpoint"; } }, + //public static final String instances = "instances"; + //public static final String autostart = "autostart"; + + //public static final String reference = "reference"; + //public static final String enabled = "enabled"; + //public static final String disable_reason = "disable-reason"; + //public static final String implementors = "implementors"; + //public static final String numeric_id = "numeric_id"; + //public static final String ping_active = "ping-active"; + //public static final String ping_only = "ping-only"; + //public static final String service_alive = "service-alive"; + //public static final String service_class = "service-class"; + //public static final String service_dependency = "service_dependency"; + //public static final String service_healthy = "service-healthy"; + //public static final String service_state = "service-state"; + //public static final String last_use = "last-use"; + //public static final String service_statistics = "service-statistics"; + //public static final String service_type = "service-type"; + //public static final String submit_error = "submit-error"; + //public static final String user = "user"; - public static final String endpoint = "endpoint"; - public static final String instances = "instances"; - public static final String autostart = "autostart"; - public static final String reference = "reference"; - public static final String enabled = "enabled"; - public static final String disable_reason = "disable-reason"; - public static final String implementors = "implementors"; - public static final String numeric_id = "numeric_id"; - public static final String ping_active = "ping-active"; - public static final String ping_only = "ping-only"; - public static final String service_alive = "service-alive"; - public static final String service_class = "service-class"; - public static final String service_dependency = "service_dependency"; - public static final String service_healthy = "service-healthy"; - public static final String service_state = "service-state"; - public static final String last_use = "last-use"; - public static final String service_statistics = "service-statistics"; - public static final String service_type = "service-type"; - public static final String submit_error = "submit-error"; - public static final String user = "user"; - - public static final String scheduling_class = "scheduling_class"; - public static final String process_memory_size = "process_memory_size"; - public static final String description = "description"; - public static final String log_directory = "log_directory"; - public static final String process_executable = "process_executable"; + //public static final String scheduling_class = "scheduling_class"; + //public static final String process_memory_size = "process_memory_size"; + //public static final String description = "description"; + //public static final String log_directory = "log_directory"; + //public static final String process_executable = "process_executable"; - public static final String service_type_UIMA_AS = "UIMA-AS"; - public static final String service_type_CUSTOM = "CUSTOM"; + //public static final String service_type_UIMA_AS = "UIMA-AS"; + //public static final String service_type_CUSTOM = "CUSTOM"; - public ArrayList<String> getSvcList(); - public ArrayList<String> getMetaList(); + // + // IMPORTANT IMPORTANT IMPORTANT + // As of now, 2015/08/17, there is no support for service registration history. There never has been in + // this class and it's derivatives. Until there is a use-case and demand it's not clear what the interfaces + // would be like. + // + // We may have to prune, or look at specific history files, but that can be done with direct database + // operations. If we need more, well add it later. + // + // The only nod to history we make here is the ability to move a registration and its meta from 'live' to + // 'history' state. + // IMPORTANT IMPORTANT IMPORTANT + + public List<Long> getSvcList() throws Exception; // list of registered services + public List<Long> getMetaList() throws Exception; // not used ? - public StateServicesDirectory getStateServicesDirectory() throws IOException; + public StateServicesDirectory getStateServicesDirectory() throws Exception; // all the registy in one blow + + public boolean storeProperties (DuccId serviceId, Properties svc, Properties meta) throws Exception; // save svc and meta in a transaction + public boolean updateJobProperties (DuccId serviceId, Properties props) throws Exception; // update just job props + public boolean updateMetaProperties(DuccId serviceId, Properties props) throws Exception; // update just metaprops + public void moveToHistory(DuccId serviceId, Properties svc, Properties meta) throws Exception; + + public void shutdown() throws Exception; + public boolean init(DuccLogger logger) throws Exception; } Added: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/NullStateServices.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/NullStateServices.java?rev=1703203&view=auto ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/NullStateServices.java (added) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/NullStateServices.java Tue Sep 15 14:31:24 2015 @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. +*/ +package org.apache.uima.ducc.common.persistence.services; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Properties; + +import org.apache.uima.ducc.common.utils.DuccLogger; +import org.apache.uima.ducc.common.utils.id.DuccId; + + +public class NullStateServices implements IStateServices +{ + + NullStateServices() { + } + + public boolean init(DuccLogger logger) throws Exception { return true; } + public ArrayList<Long> getSvcList() + throws Exception + { + return new ArrayList<Long>(); + } + + + public ArrayList<Long> getMetaList() + throws Exception + { + return new ArrayList<Long>(); + } + + + public StateServicesDirectory getStateServicesDirectory() + throws IOException + { + return new StateServicesDirectory(); + } + + public boolean storeProperties(DuccId serviceId, Properties svc, Properties meta) throws Exception {return false;} + public boolean updateProperties(DuccId serviceId, String type, Properties props) throws Exception {return false;} + public void deleteProperties(DuccId serviceId) throws Exception {} + public void shutdown() throws Exception {} + public void moveToHHistory() throws Exception {} + public boolean updateJobProperties(DuccId serviceId, Properties props) throws Exception {return false;} + public boolean updateMetaProperties(DuccId serviceId, Properties props) throws Exception {return false;} + public void moveToHistory(DuccId serviceId, Properties svc, Properties meta) throws Exception {} + +} Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServices.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServices.java?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServices.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServices.java Tue Sep 15 14:31:24 2015 @@ -20,70 +20,87 @@ package org.apache.uima.ducc.common.pers import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; import java.io.IOException; import java.util.ArrayList; import java.util.Properties; -import org.apache.uima.ducc.common.IDuccEnv; -import org.apache.uima.ducc.common.main.DuccService; import org.apache.uima.ducc.common.utils.DuccLogger; +import org.apache.uima.ducc.common.utils.DuccProperties; import org.apache.uima.ducc.common.utils.IOHelper; import org.apache.uima.ducc.common.utils.id.DuccId; public class StateServices implements IStateServices { - private static DuccLogger logger = DuccService.getDuccLogger(StateServices.class.getName()); - private static DuccId jobid = null; - - private String directory_state_services = IDuccEnv.DUCC_STATE_SERVICES_DIR; + private DuccLogger logger = null; StateServices() { - mkdirs(); } - + + public boolean init(DuccLogger logger) + { + this.logger = logger; + mkdirs(); + return true; + } + private void mkdirs() { - IOHelper.mkdirs(directory_state_services); + IOHelper.mkdirs(svc_reg_dir); + IOHelper.mkdirs(svc_hist_dir); } - - private ArrayList<String> getList(String type) { + + private String mkfilename(long id, String type) + { + return svc_reg_dir + Long.toString(id) + "." + type; + } + + private String mkfilename(DuccId id, String type) + { + return mkfilename(id.getFriendly(), type); + } + + private ArrayList<Long> getList(String type) { String location = "getList"; - ArrayList<String> retVal = new ArrayList<String>(); + ArrayList<Long> retVal = new ArrayList<Long>(); try { - logger.debug(location, jobid, directory_state_services); - File folder = new File(directory_state_services); + logger.debug(location, null, svc_reg_dir); + File folder = new File(svc_reg_dir); File[] listOfFiles = folder.listFiles(); if(listOfFiles != null) { for (int i = 0; i < listOfFiles.length; i++) { if (listOfFiles[i].isFile()) { String name = listOfFiles[i].getName(); if(name.endsWith("."+type)) { - retVal.add(name); + int ndx = name.lastIndexOf("."); + name = name.substring(0, ndx); + retVal.add(Long.parseLong(name)); } } } } } catch(Exception e) { - logger.error(location, jobid, e); + logger.error(location, null, e); } return retVal; } - public ArrayList<String> getSvcList() { + public ArrayList<Long> getSvcList() { return getList(IStateServices.svc); } - public ArrayList<String> getMetaList() { + public ArrayList<Long> getMetaList() { return getList(IStateServices.meta); } - private Properties getProperties(String name) { + private DuccProperties getProperties(String name) { String location = "getProperties"; - Properties properties = new Properties(); + DuccProperties properties = new DuccProperties(); try { FileInputStream fis = new FileInputStream(name); try { @@ -94,43 +111,201 @@ public class StateServices implements IS } } catch(Exception e) { - logger.error(location, jobid, e); + logger.error(location, null, e); } return properties; } - public StateServicesDirectory getStateServicesDirectory() throws IOException { + public StateServicesDirectory getStateServicesDirectory() + throws Exception + { String location = "getStateServicesDirectory"; StateServicesDirectory ssd = null; try { ssd = new StateServicesDirectory(); - ArrayList<String> svcList = getSvcList(); - logger.trace(location, jobid, svcList.size()); - for(String entry : svcList) { + ArrayList<Long> svcList = getSvcList(); + logger.trace(location, null, svcList.size()); + for(Long entry : svcList) { try { - StateServicesSet sss = new StateServicesSet(); - String num = entry.split("[.]")[0]; - Integer i = new Integer(num); - String base = directory_state_services+num; - logger.trace(location, jobid, base); - String fnSvc = base+"."+svc; - String fnMeta = base+"."+meta; - Properties propertiesSvc = getProperties(fnSvc); - sss.put(svc, propertiesSvc); - Properties propertiesMeta = getProperties(fnMeta); - sss.put(meta, propertiesMeta); - ssd.put(i, sss); + StateServicesSet sss = new StateServicesSet(); + String fnSvc = mkfilename(entry, svc); + String fnMeta = mkfilename(entry, meta); + DuccProperties propertiesSvc = getProperties(fnSvc); + sss.put(svc, propertiesSvc); + DuccProperties propertiesMeta = getProperties(fnMeta); + sss.put(meta, propertiesMeta); + ssd.put(entry, sss); } catch(Exception e) { - logger.error(location, jobid, e); + logger.error(location, null, e); } } } catch(Exception e) { - logger.error(location, jobid, e); + logger.error(location, null, e); } return ssd; } - + + // Try to write properties file, using a temp file as backup in case it fails. + private boolean writeProperties(DuccId id, Properties props, File pfile, File pfile_tmp, String type) + { + + String methodName = "saveProperties"; + FileOutputStream fos = null; + + long original_size = pfile.length(); + try { + if ( (!pfile.exists()) || pfile.renameTo(pfile_tmp) ) { + fos = new FileOutputStream(pfile); + props.store(fos, type + " Descriptor"); + } else { + logger.error(methodName, id, "Cannot save", type, "properties, rename of", pfile, "to", pfile_tmp, "fails."); + if ( (!pfile.exists()) && pfile_tmp.exists() ) { + if ( !pfile_tmp.renameTo(pfile) ) { + logger.error(methodName, id, "Cannot restore", pfile_tmp, "to", pfile, "after failed update."); + } + } + } + } catch (FileNotFoundException e) { + logger.error(methodName, id, "Cannot save " + type + " properties, file does not exist."); + } catch (IOException e) { + logger.error(methodName, id, "I/O Error saving " + type + " service properties: " + e.toString()); + } catch (Throwable t) { + logger.error(methodName, id, "Unexpected Error saving " + type + " service properties: " + t.toString()); + } finally { + try { + if ( fos != null ) fos.close(); + long updated_size = pfile.length(); + long tmp_size = pfile_tmp.length(); + + // updated size must be > 0 and tmp_size must match original size + if ( (updated_size > 0) && (original_size == tmp_size) ) { + pfile_tmp.delete(); + } else { + logger.error(methodName, id, "Update of", pfile.toString(), "failed. Original size:", original_size, "updated size", updated_size, "temp file size", tmp_size); + logger.error(methodName, id, "The updated size must be > 0 and the temp size must match the original size for sucess."); + logger.error(methodName, id, "Attempting to restore", pfile.toString(), "from", pfile_tmp.toString()); + if ( !pfile.exists() && pfile_tmp.exists() ) { + pfile_tmp.renameTo(pfile); + } + return false; + } + } catch (Throwable t) { + logger.error(methodName, id, "Cannot close", type, "properties:", t); + return false; + } + } + + return true; + } + + // Try, and retry, to save a single props file. Returns true if ok, false otherwise. + private boolean saveProperties(DuccId id, Properties props, File pfile, File pfile_tmp, String type) + { + int max = 5; + for ( int i = 0; i < max; i++ ) { + if (writeProperties(id, props, pfile, pfile_tmp, type)) return true; + } + return false; + } + + + // Attempt to transactionally save the two service props. If it returns null + // the caller can assume it worked. Otherwise and error string indicating the cause + // of failure is returned. In case of failure, we try to insure no properties + // file linger. + public boolean storeProperties (DuccId id, Properties svcprops, Properties metaprops) + { + // save svc and meta in a transaction + + File svcfile = new File(mkfilename(id, svc)); + File metafile = new File(mkfilename(id, meta)); + boolean ok = true; + + File tmpfile = new File(svcfile.toString() + ".tmp"); + if ( saveProperties(id, svcprops, svcfile, tmpfile, svc) ) { + tmpfile = new File(metafile.toString() + ".tmp"); + ok = saveProperties(id, metaprops, metafile, tmpfile, meta); + } + + if ( !ok ) { + metafile.delete(); + svcfile.delete(); + } + return ok; + } + + private boolean updateProperties(DuccId serviceId, Properties props, String type) + { + File f = new File(mkfilename(serviceId, type)); + File tmpf = new File(f.toString() + ".tmp"); + return saveProperties(serviceId, props, f, tmpf, type); + } + + public boolean updateJobProperties(DuccId serviceId, Properties props) + { + return updateProperties(serviceId, props, svc); + } + + public boolean updateMetaProperties(DuccId serviceId, Properties props) + { + return updateProperties(serviceId, props, meta); + } + + public void deleteProperties(long serviceId) + { + String svcfile = mkfilename(serviceId, svc); + String metafile = mkfilename(serviceId, meta); + + File pf = new File(svcfile); + pf.delete(); + + File mf = new File(metafile); + mf.delete(); + } + + public void deleteProperties(DuccId id) + { + deleteProperties(id.getFriendly()); + } + + + public void moveToHistory(DuccId id, Properties svc, Properties meta) + throws Exception + { + String methodName = "moveToHistory"; + + // Save a copy in history, and then delete the original + File mfh = new File(svc_hist_dir + id + ".meta"); + try { + FileOutputStream fos = new FileOutputStream(mfh); + meta.store(fos, "Archived meta descriptor"); + fos.close(); + } catch (Exception e) { + logger.warn(methodName, null, id + ": Unable to save history to \"" + mfh.toString(), ": ", e.toString() + "\""); + } + + String meta_filename = svc_reg_dir + id + ".meta"; + File mf = new File(meta_filename); + mf.delete(); + + + File pfh = new File(svc_hist_dir + id + ".svc"); + + try { + FileOutputStream fos = new FileOutputStream(pfh); + svc.store(fos, "Archived svc properties."); + fos.close(); + } catch (Exception e) { + logger.warn(methodName, null, id + ":Unable to save history to \"" + pfh.toString(), ": ", e.toString() + "\""); + } + + String props_filename = svc_reg_dir + id + ".svc"; + File pf = new File(props_filename); + pf.delete(); + } + + public void shutdown() {} } Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesDirectory.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesDirectory.java?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesDirectory.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesDirectory.java Tue Sep 15 14:31:24 2015 @@ -23,17 +23,17 @@ import java.util.TreeMap; public class StateServicesDirectory { - private TreeMap<Integer,StateServicesSet> map = new TreeMap<Integer,StateServicesSet>(); + private TreeMap<Long,StateServicesSet> map = new TreeMap<Long,StateServicesSet>(); - public void put(Integer key, StateServicesSet value) { + public void put(Long key, StateServicesSet value) { map.put(key, value); } - public StateServicesSet get(Integer key) { + public StateServicesSet get(Long key) { return map.get(key); } - public NavigableSet<Integer> getDescendingKeySet() { + public NavigableSet<Long> getDescendingKeySet() { return map.descendingKeySet(); } }