svn commit: r1715427 - in /uima/sandbox/uima-ducc/trunk: uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/ uima-ducc-database/src/main/java/org/apache/uima/ducc/database/ uima
Author: challngr
Date: Fri Nov 20 21:48:11 2015
New Revision: 1715427
URL: http://svn.apache.org/viewvc?rev=1715427&view=rev
Log:
UIMA-4577 Support rmshares table.
Added:
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IDbShare.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/NullRmStatePersistence.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/ResourceManagerComponent.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ISchedulerMain.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Share.java
Added:
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IDbShare.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IDbShare.java?rev=1715427&view=auto
==
---
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IDbShare.java
(added)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IDbShare.java
Fri Nov 20 21:48:11 2015
@@ -0,0 +1,45 @@
+package org.apache.uima.ducc.common.persistence.rm;
+
+import org.apache.uima.ducc.common.Node;
+import org.apache.uima.ducc.common.NodeIdentity;
+import org.apache.uima.ducc.common.utils.id.DuccId;
+
+public interface IDbShare {
+
+
+ public abstract int getNodepoolDepth();
+
+ public abstract String getNodepoolId();
+
+ public abstract DuccId getId();
+
+ // UIMA-4142
+ public abstract boolean isBlacklisted();
+
+ // UIMA-4142
+ public abstract DuccId getBlJobId();
+
+ public abstract NodeIdentity getNodeIdentity();
+
+ public abstract Node getNode();
+
+ /**
+* The order of the share itself.
+*/
+ public abstract int getShareOrder();
+
+ /**
+* Returns only initialization time. Eventually getInvestment() may
take other things into
+* consideration so we separate these two (even though currently they
do the same thing.)
+*/
+ public abstract long getInitializationTime();
+
+ public abstract void setInitializationTime(long millis);
+
+ public abstract void setFixed();
+
+ public abstract boolean isPurged();
+public abstract boolean isEvicted();
+public abstract boolean isFixed();
+
+}
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java?rev=1715427&r1=1715426&r2=1715427&view=diff
==
---
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java
Fri Nov 20 21:48:11 2015
@@ -91,8 +91,9 @@ public interface IRmPersistence
* @param id The node name
* @param jobid The duccid of the job owning the new shoare
* @param shareid The DuccId of the new share.
+ * @param quantum The scheduling quantum in GB used for this share.
*/
-public void addAssignment(String id, DuccId jobid, DuccId shareid);
+public void addAssignment(String id, DuccId jobid, IDbShare share, int
quantum, String jobtype) throws Exception;
/**
* Remove a share from the machine.
@@ -100,7 +101,54 @@ public interface IRmPersistence
* @param jobid The duccid of the job owning the new shoare
* @param shareid The Ducc
svn commit: r1715378 - /uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/CliFixups.java
Author: burn
Date: Fri Nov 20 15:40:55 2015
New Revision: 1715378
URL: http://svn.apache.org/viewvc?rev=1715378&view=rev
Log:
UIMA-4696 Remove buggy unnecessary hack for pre-2.0 services
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/CliFixups.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/CliFixups.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/CliFixups.java?rev=1715378&r1=1715377&r2=1715378&view=diff
==
---
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/CliFixups.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/CliFixups.java
Fri Nov 20 15:40:55 2015
@@ -21,8 +21,6 @@ package org.apache.uima.ducc.cli;
import java.util.Properties;
-import org.apache.uima.ducc.common.utils.Utils;
-
/*
* Sites that have old code built against a pre-release version of DUCC may
replace this class
* by one that corrects any deprecated options, e.g. changing
--process_environment to --environment
@@ -74,30 +72,15 @@ public class CliFixups {
}
static void cleanupProps(Properties props, String className) {
- // Augment CP if the SM issues a DuccServiceSubmit with the
classpath_order option
- // (Always invoked with a service properties file)
- if (className.equals(DuccServiceSubmit.class.getName())) {
- String cpOrder = props.getProperty("classpath_order");
- String cp = props.getProperty("classpath");
- if (cpOrder != null && cp != null) {
- String duccHome = Utils.findDuccHome();
- String uimaCp = duccHome +
"/apache-uima/lib/*:" + duccHome + "/apache-uima/apache-activemq/lib/*:"
- + duccHome +
"/apache-uima/apache=activemq/lib/optional/*";
- if (cpOrder.equals("ducc-before-user")) {
- cp = uimaCp + ":" + cp;
- } else {
- cp = cp + ":" + uimaCp;
- }
- props.put("classpath", cp);
- System.out.println("CLI added UIMA jars to
classpath of pre-2.0 service");
- }
- }
+
+ // >>> Removed the hack that added UIMA jars to the classpath of
services registered with DUCC 1.x but
+ // >>> started with 2.0 --- had a bug and just delayed the need for the
user to provide a complete classpath.
changeOption("process_DD", "process_descriptor_DD", props);
changeOption("instance_memory_size", "memory_size", props);
changeOption("classpath_order", null, props);
changeOption("process_thread_count", "process_pipeline_count", props);
- // Ignore option if number_of_instances = 1 ... reject any other value
+ // Remove number_of_instaces only if = 1 ... leave any other value to
be rejected
String val = props.getProperty("number_of_instances");
if (val != null && val.trim().equals("1")) {
changeOption("number_of_instances", null, props);
svn commit: r1715374 - /uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/AllInOneLauncher.java
Author: burn
Date: Fri Nov 20 15:20:41 2015
New Revision: 1715374
URL: http://svn.apache.org/viewvc?rev=1715374&view=rev
Log:
UIMA-4695 Send all_in_one local output to new stdout id# 0
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/AllInOneLauncher.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/AllInOneLauncher.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/AllInOneLauncher.java?rev=1715374&r1=1715373&r2=1715374&view=diff
==
---
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/AllInOneLauncher.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/AllInOneLauncher.java
Fri Nov 20 15:20:41 2015
@@ -752,9 +752,9 @@ public class AllInOneLauncher extends Cl
InputStream is = process.getInputStream();
InputStreamReader isr = new InputStreamReader(is);
BufferedReader ibr = new BufferedReader(isr);
-// Pass all process output through the console callback
+// Pass all process output through the console callback to stdout
while ((line = ibr.readLine()) != null) {
-consoleCb.console(1, line);
+consoleCb.console(0, line);
}
ibr.close();
try {
svn commit: r1715360 - /uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ConsoleListener.java
Author: burn
Date: Fri Nov 20 13:36:34 2015
New Revision: 1715360
URL: http://svn.apache.org/viewvc?rev=1715360&view=rev
Log:
UIMA-4695 Use ID#s 0,1 for the separate AP stdout/stderr streams and start
others from 2 for jd & JPs
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ConsoleListener.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ConsoleListener.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ConsoleListener.java?rev=1715360&r1=1715359&r2=1715360&view=diff
==
---
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ConsoleListener.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ConsoleListener.java
Fri Nov 20 13:36:34 2015
@@ -44,7 +44,8 @@ class ConsoleListener
private boolean in_shutdown = false;
private boolean start_stdin = false;
-private int numConnected = 0;
+private int nextIdNum = 2; // id# for JD ... JPs get 3,4,...
0&1 reserved for an AP
+private volatile PrintWriter shared_logout = null;// Shared by ids 0 &
1 (stdout & stderr)
private IDuccCallback consoleCb;
// private int callers; // number of remote processes we expect
to listen for
@@ -134,7 +135,7 @@ class ConsoleListener
int p = s.getPort();
synchronized(this) {
listeners.put(p, new Pair(sr,
sw));
-sr.idNum = ++numConnected;
+sr.idNum = nextIdNum++;
}
Thread t = new Thread(sr, "STDOUT");
@@ -164,9 +165,11 @@ class ConsoleListener
String remote_host;
private PrintWriter logout = null;
+static final String stream_tag = "1500 Stream: ";
static final String console_tag = "1002 CONSOLE_REDIRECT ";
int tag_len = 0;
private int idNum;
+private boolean is_stderr = false;
StdioReader(Socket sock)
{
@@ -186,6 +189,7 @@ class ConsoleListener
if ( debug ) System.out.println("= Listener completing: " +
remote_host + ":" + sock.getPort());
shutdown = true;
is.close();
+// It's OK to close the shared logout twice
if (logout != null) {
logout.close();
}
@@ -198,15 +202,35 @@ class ConsoleListener
void doWrite(String line)
{
+// Check for the initial stream identifier
+// Only APs have split streams, and only stdout carries the
logfile name
+if (line.startsWith(stream_tag)) {
+ String name = line.substring(stream_tag.length());
+ is_stderr = name.startsWith("STDERR");
+ if (is_stderr) {
+do_console_out = true; // Don't wait for the magic 1001
message
+idNum = 1;
+ } else {
+idNum = 0;
+ }
+ return;
+}
+// The stderr stream shares the stdout stream's logfile
if ( line.startsWith(console_tag) && !suppress_log) {
String logfile = line.substring(tag_len);
try {
logout = new PrintWriter(logfile);
- return;
+ if (idNum == 0) {
+shared_logout = logout;
+ }
} catch (FileNotFoundException e) {
consoleCb.status("Failed to create log file: " + logfile);
e.printStackTrace();
}
+return;
+}
+if (logout == null && idNum == 1) {
+ logout = shared_logout;
}
if (logout != null) {
logout.println(line);
@@ -285,7 +309,12 @@ class ConsoleListener
}
if ( debug ) System.out.println(remote_host + ": EOF:
exiting");
} catch ( Throwable t ) {
-t.printStackTrace();
+if (shutdown) {
+ if (debug) System.out.println(remote_host + ":" +
sock.getPort() + " ignore read error after shutdoen - id# " + idNum);
+} else {
+ System.out.println(remote_host + ":" + sock.getPort() + "
read error - id# " + idNum);
+ t.printStackTrace();
+}
} finally {
try {
shutdown();
svn commit: r1715357 - /uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/CliBase.java
Author: burn
Date: Fri Nov 20 13:31:52 2015
New Revision: 1715357
URL: http://svn.apache.org/viewvc?rev=1715357&view=rev
Log:
UIMA-4695 Add a query string to the AP callback addr so ducc_ling creates 2
streams
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/CliBase.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/CliBase.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/CliBase.java?rev=1715357&r1=1715356&r2=1715357&view=diff
==
---
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/CliBase.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/CliBase.java
Fri Nov 20 13:31:52 2015
@@ -422,6 +422,8 @@ public abstract class CliBase
/*
* Resolve any ${..} placeholders against user's system properties and
environment
+ * NOTE - this resolves against the caller's sys-props & environment ...
the one in DuccUiUtilities
+ *resolves against the process JVM args to match what is done by
Spring in UIMA-AS.
* 2.0: Leave unresolved entries as is & warn if not one of the
always-propagated ones
*/
private String resolvePlaceholders(String contents, List
envNameList) {
@@ -673,6 +675,9 @@ public abstract class CliBase
if (console_attach) {
console_listener = new ConsoleListener(this, consoleCb);
value = console_listener.getConsoleHostAddress();
+if
(myClassName.equals(DuccManagedReservationSubmit.class.getName())) {
+ value += "?splitstreams"; // Add a query string so APs have
separate streams
+}
} else if (suppress_console_log) {
value = "suppress";
} else {
svn commit: r1715356 - /uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DefaultCallback.java
Author: burn
Date: Fri Nov 20 13:30:07 2015
New Revision: 1715356
URL: http://svn.apache.org/viewvc?rev=1715356&view=rev
Log:
UIMA-4695 Replace callback numbers with an ID of JD or JP or nothing for AP,
and send the 2nd AP stream to stderr
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DefaultCallback.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DefaultCallback.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DefaultCallback.java?rev=1715356&r1=1715355&r2=1715356&view=diff
==
---
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DefaultCallback.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DefaultCallback.java
Fri Nov 20 13:30:07 2015
@@ -21,8 +21,23 @@ package org.apache.uima.ducc.cli;
public class DefaultCallback implements IDuccCallback {
+ // pnum: 0,1 => AP 2 => JD 3,... => JP
public void console(int pnum, String msg) {
-System.out.println("[" + pnum + "] " + msg);
+int jpid = pnum - 2; // get number of JP
+String prefix;
+if (jpid < 0) {
+ prefix = "";
+} else if (jpid == 0) {
+ prefix = "[JD] ";
+} else {
+ prefix = "[JP" + jpid + "] ";
+}
+// Only the AP has split streams
+if (pnum == 1) {
+ System.err.println(prefix + msg);
+} else {
+ System.out.println(prefix + msg);
+}
}
