svn commit: r1713942 - /uima/sandbox/uima-ducc/trunk/src/main/admin/db_util.py
Author: degenaro Date: Wed Nov 11 22:40:51 2015 New Revision: 1713942 URL: http://svn.apache.org/viewvc?rev=1713942&view=rev Log: UIMA-4577 DUCC Database (DB) db_util.py does not cd back to original directory (causing subsequent failure to create ducc_ling) Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/db_util.py Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/db_util.py URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/db_util.py?rev=1713942&r1=1713941&r2=1713942&view=diff == --- uima/sandbox/uima-ducc/trunk/src/main/admin/db_util.py (original) +++ uima/sandbox/uima-ducc/trunk/src/main/admin/db_util.py Wed Nov 11 22:40:51 2015 @@ -74,7 +74,8 @@ def configure_database(DUCC_HOME, DUCC_H CMD = "bin/cassandra -p "+ pidfile + " > /dev/null 2>&1"; os.system(CMD); print "Database is started. Waiting for initialization"; - +os.chdir(here) + # Now start the db and create the schema CLASSPATH = '' CLASSPATH = addToCp(CLASSPATH, DUCC_HOME + '/lib/cassandra/*')
svn commit: r1713941 - /uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install
Author: degenaro Date: Wed Nov 11 22:38:33 2015 New Revision: 1713941 URL: http://svn.apache.org/viewvc?rev=1713941&view=rev Log: UIMA-4577 DUCC Database (DB) post-install should first create state directory then initialize DB Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install?rev=1713941&r1=1713940&r2=1713941&view=diff == --- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install (original) +++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install Wed Nov 11 22:38:33 2015 @@ -414,9 +414,6 @@ class PostInstall(): # insure java is configured and installed self.java_bindir = self.get_java_bindir() -# configure the database for local system and initialize the schema -self.configure_database() - # As of DUCC 2.0, always set here on installation print 'ActiveMQ is automanaged on node ', self.localhost @@ -433,6 +430,9 @@ class PostInstall(): if ( not os.path.exists(historydir) ): os.mkdir(historydir) +# configure the database for local system and initialize the schema +self.configure_database() + self.keytool = self.get_java_keytool(self.java_bindir) print 'Java version:', self.get_java_version() print 'Java is verified.'
svn commit: r1713913 - /uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
Author: challngr
Date: Wed Nov 11 18:52:27 2015
New Revision: 1713913
URL: http://svn.apache.org/viewvc?rev=1713913&view=rev
Log:
UIMA-4577 Updates to scripting.
Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1713913&r1=1713912&r2=1713913&view=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py Wed Nov 11
18:52:27 2015
@@ -162,7 +162,6 @@ class DuccUtil(DuccBase):
def db_configure(self):
dbhost = self.ducc_properties.get('ducc.database.host')
-print '- dbhost', dbhost
if ( dbhost == self.db_disabled ):
self.db_bypass = True
return;
svn commit: r1713912 - in /uima/sandbox/uima-ducc/trunk/src/main/admin: db_create db_util.py ducc_post_install
Author: challngr
Date: Wed Nov 11 18:45:13 2015
New Revision: 1713912
URL: http://svn.apache.org/viewvc?rev=1713912&view=rev
Log:
UIMA-4577 Updates to scripting.
Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/db_create
uima/sandbox/uima-ducc/trunk/src/main/admin/db_util.py
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/db_create
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/db_create?rev=1713912&r1=1713911&r2=1713912&view=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/db_create (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/db_create Wed Nov 11 18:45:13
2015
@@ -94,6 +94,9 @@ class DbCreate(DuccUtil):
# configure the database for local system and initialize the schema
db_node = self.ducc_properties.get("ducc.head")
if ( dbu.configure_database(self.DUCC_HOME, db_node, self.jvm,
self.database_pw)):
+private_props_name = self.DUCC_HOME +
'/resources.private/ducc.private.properties'
+
+print 'Writing database password to', private_props_name
ducc_site_properties = Properties();
site_props_name = self.DUCC_HOME +
'/resources/site.ducc.properties'
ducc_site_properties.load(site_props_name)
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/db_util.py
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/db_util.py?rev=1713912&r1=1713911&r2=1713912&view=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/db_util.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/db_util.py Wed Nov 11 18:45:13
2015
@@ -70,6 +70,7 @@ def configure_database(DUCC_HOME, DUCC_H
here = os.getcwd()
os.chdir(DUCC_HOME + "/cassandra-server")
pidfile = DUCC_HOME + '/state/cassandra.pid'
+print 'Starting the database. This might take a few moments if it is the
first time.'
CMD = "bin/cassandra -p "+ pidfile + " > /dev/null 2>&1";
os.system(CMD);
print "Database is started. Waiting for initialization";
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install?rev=1713912&r1=1713911&r2=1713912&view=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install Wed Nov 11
18:45:13 2015
@@ -60,7 +60,7 @@ class PostInstall():
print " [-j, --jvm] "
print "This is the full path to java command to be used to
start DUCC; e.g., /usr/bin/java"
print ""
-print " [-, --db-password] "
+print " [-d, --db-password] "
print "This is the password DUCC uses to manage the database."
print ""
print " [-h, -? --help]"
@@ -105,24 +105,30 @@ class PostInstall():
print 'Database is already defined. Not configuring'
return
-db_pw = raw_input("Enter database password OR 'bypass' to disable
database support:")
-if ( db_pw == '' ):
-print "Must enter a DB password to continue."
-sys.exit(1);
+if ( self.database_pw == None ):
+db_pw = raw_input("Enter database password OR 'bypass' to disable
database support:")
+if ( db_pw == '' ):
+print "Must enter a DB password to continue."
+sys.exit(1);
+else:
+db_pw = self.database_pw
if ( db_pw == 'bypass' ):
print 'Database support will be disabled'
self.update_property('ducc.database.host', '--disabled--', '#
Database support is disabled')
return;
-else:
-self.update_property('ducc.database.host', self.ducc_head, '#
Database location')
-dbu.configure_database(self.DUCC_HOME, self.ducc_head,
self.path_to_java, db_pw)
-private_props_name = self.DUCC_HOME +
'/resources.private/ducc.private.properties'
-private_properties = Properties()
-private_properties.load(private_props_name)
-private_properties.put('db_password', self.database_pw, ['#Db
password']);
-private_properties.write(private_props_name)
+if ( dbu.configure_database(self.DUCC_HOME, self.ducc_head,
self.path_to_java, db_pw) ):
+print 'Configuring DUCC to use the database.'
+
+self.update_property('ducc.database.host', self.ducc_head, '#
Database location')
+self.update_property('ducc.service.persistence.impl',
'org.apache.uima.ducc.database.StateServicesDb', '# Service manager
persistence')
+self
svn commit: r1713897 - /uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/agent/metrics/swap/DuccProcessMemoryPageLoadUsage.java
Author: cwiklik
Date: Wed Nov 11 17:19:58 2015
New Revision: 1713897
URL: http://svn.apache.org/viewvc?rev=1713897&view=rev
Log:
UIMA-4690 redirect stderr to stdout
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/agent/metrics/swap/DuccProcessMemoryPageLoadUsage.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/agent/metrics/swap/DuccProcessMemoryPageLoadUsage.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/agent/metrics/swap/DuccProcessMemoryPageLoadUsage.java?rev=1713897&r1=1713896&r2=1713897&view=diff
==
---
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/agent/metrics/swap/DuccProcessMemoryPageLoadUsage.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/agent/metrics/swap/DuccProcessMemoryPageLoadUsage.java
Wed Nov 11 17:19:58 2015
@@ -36,6 +36,7 @@ public class DuccProcessMemoryPageLoadUs
String[] command = new String[] {"/bin/ps","-o","maj_flt",pid};
ProcessBuilder builder = new ProcessBuilder(command);
+ builder.redirectErrorStream(true);
Process process = builder.start();
InputStream is = process.getInputStream();
InputStreamReader isr = new InputStreamReader(is);
@@ -56,9 +57,10 @@ public class DuccProcessMemoryPageLoadUs
if (is != null) {
is.close();
}
+ process.waitFor();
process.destroy();
}
- process.waitFor();
+
if ( faults != null) {
return Long.parseLong(faults.trim());
} else {
svn commit: r1713896 - /uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/processors/LinuxProcessMetricsProcessor.java
Author: cwiklik
Date: Wed Nov 11 17:17:55 2015
New Revision: 1713896
URL: http://svn.apache.org/viewvc?rev=1713896&view=rev
Log:
UIMA-4690 redirect stderr to stdout
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/processors/LinuxProcessMetricsProcessor.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/processors/LinuxProcessMetricsProcessor.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/processors/LinuxProcessMetricsProcessor.java?rev=1713896&r1=1713895&r2=1713896&view=diff
==
---
uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/processors/LinuxProcessMetricsProcessor.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/processors/LinuxProcessMetricsProcessor.java
Wed Nov 11 17:17:55 2015
@@ -552,7 +552,6 @@ public class LinuxProcessMetricsProcesso
pb.redirectErrorStream(true);
Process proc = pb.start();
- proc.waitFor();
// spawn ps command and scrape the output
stream = proc.getInputStream();
reader = new BufferedReader(new
InputStreamReader(stream));
@@ -571,6 +570,7 @@ public class LinuxProcessMetricsProcesso
cpuTime = cpuTime.substring(0,
cpuTime.indexOf("."));
}
stream.close();
+ proc.waitFor();
try {
cpuint = Integer.valueOf(cpuTime);
} catch (NumberFormatException e) {
svn commit: r1713863 - /uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java
Author: challngr
Date: Wed Nov 11 14:58:21 2015
New Revision: 1713863
URL: http://svn.apache.org/viewvc?rev=1713863&view=rev
Log:
UIMA-4577 Redirect i/o from stop into dev null.
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java?rev=1713863&r1=1713862&r2=1713863&view=diff
==
---
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java
Wed Nov 11 14:58:21 2015
@@ -19,7 +19,7 @@
package org.apache.uima.ducc.sm;
import java.io.BufferedReader;
-import java.io.IOException;
+import java.io.File;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
@@ -357,28 +357,15 @@ class ServiceInstance
Map env = pb.environment();
env.put("DUCC_HOME", System.getProperty("DUCC_HOME"));
-StdioListener sin_listener = null;
-StdioListener ser_listener = null;
+pb.redirectOutput(new File("/dev/null"));
+pb.redirectError(new File("/dev/null"));
int rc = 0;
try {
Process p = pb.start();
-
-InputStream stdout = p.getInputStream();
- InputStream stderr = p.getErrorStream();
-
-sin_listener = new StdioListener(1, stdout, true);
-ser_listener = new StdioListener(2, stderr, true);
-Thread sol = new Thread(sin_listener);
-Thread sel = new Thread(ser_listener);
-sol.start();
-sel.start();
rc = p.waitFor();
logger.info(methodName, sset.getId(), "DuccServiceCancel returns
with rc", rc);
-
-sin_listener.stop();
-ser_listener.stop();
} catch (Throwable t) {
logger.error(methodName, null, t);
}
svn commit: r1713860 - in /uima/sandbox/uima-ducc/trunk: src/main/admin/ src/main/resources/ uima-ducc-database/src/main/java/org/apache/uima/ducc/database/ uima-ducc-sm/src/main/java/org/apache/uima/
Author: challngr Date: Wed Nov 11 14:45:37 2015 New Revision: 1713860 URL: http://svn.apache.org/viewvc?rev=1713860&view=rev Log: UIME-4577 DB support updates. Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbUtil.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc?rev=1713860&r1=1713859&r2=1713860&view=diff == --- uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc (original) +++ uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc Wed Nov 11 14:45:37 2015 @@ -55,7 +55,7 @@ class CheckDucc(DuccUtil): if ( self.db_bypass == True ): return True -ret = self.db_alive(3) +ret = self.db_alive(1) if ( ret ): print 'The database is running' else: Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1713860&r1=1713859&r2=1713860&view=diff == --- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py (original) +++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py Wed Nov 11 14:45:37 2015 @@ -208,6 +208,11 @@ class DuccUtil(DuccBase): if ( dbnode == None ): print 'No database location defined.' return False + +pidfile = self.DUCC_HOME + '/state/cassandra.pid' +if ( not os.path.exists(pidfile) ): +print 'Database pid file does not exist. Checking DB connectivity.' + # get our log4j config into the path to shut up noisy logging os.environ['CLASSPATH'] = os.environ['CLASSPATH'] + ':' + self.DUCC_HOME + '/resources' @@ -269,11 +274,11 @@ class DuccUtil(DuccBase): return True pidfile = self.DUCC_HOME + '/state/cassandra.pid' - -# for cassandra, just send it a terminate signal. a pidfile is written on startup -CMD = ['kill', '-TERM', '`cat ' + pidfile + '`'] -CMD = ' '.join(CMD) -os.system(CMD) +if ( os.path.exists(pidfile) ): +# for cassandra, just send it a terminate signal. a pidfile is written on startup +CMD = ['kill', '-TERM', '`cat ' + pidfile + '`'] +CMD = ' '.join(CMD) +os.system(CMD) def find_netstat(self): # don't you wish people would get together on where stuff lives? Modified: uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml?rev=1713860&r1=1713859&r2=1713860&view=diff == --- uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml (original) +++ uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml Wed Nov 11 14:45:37 2015 @@ -140,12 +140,13 @@ - + - + + @@ -240,6 +241,11 @@ + + + + + Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java?rev=1713860&r1=1713859&r2=1713860&view=diff == --- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java Wed Nov 11 14:45:37 2015 @@ -33,6 +33,8 @@ import com.datastax.driver.core.Prepared import com.datastax.driver.core.ResultSet; import com.datastax.driver.core.Session; import com.datastax.driver.core.SimpleStatement; +import com.datastax.driver.
